Merge pull request #634 from runzexia/s2ibin-upload

s2ibin upload

cmd/controller-manager/app/controllers.go

@@ -24,6 +24,9 @@ import (
 	"kubesphere.io/kubesphere/pkg/controller/application"
 	"kubesphere.io/kubesphere/pkg/controller/destinationrule"
 	"kubesphere.io/kubesphere/pkg/controller/job"
+	"kubesphere.io/kubesphere/pkg/controller/s2ibinary"
+
+	"kubesphere.io/kubesphere/pkg/controller/s2irun"
 
 	//"kubesphere.io/kubesphere/pkg/controller/job"
 	"kubesphere.io/kubesphere/pkg/controller/virtualservice"
@@ -36,8 +39,10 @@ import (
 	istioinformers "github.com/knative/pkg/client/informers/externalversions"
 	applicationclientset "github.com/kubernetes-sigs/application/pkg/client/clientset/versioned"
 	applicationinformers "github.com/kubernetes-sigs/application/pkg/client/informers/externalversions"
-	servicemeshclientset "kubesphere.io/kubesphere/pkg/client/clientset/versioned"
-	servicemeshinformers "kubesphere.io/kubesphere/pkg/client/informers/externalversions"
+	s2iclientset "github.com/kubesphere/s2ioperator/pkg/client/clientset/versioned"
+	s2iinformers "github.com/kubesphere/s2ioperator/pkg/client/informers/externalversions"
+	kubesphereclientset "kubesphere.io/kubesphere/pkg/client/clientset/versioned"
+	kubesphereinformers "kubesphere.io/kubesphere/pkg/client/informers/externalversions"
 )
 
 const defaultResync = 600 * time.Second
@@ -62,56 +67,70 @@ func AddControllers(mgr manager.Manager, cfg *rest.Config, stopCh <-chan struct{
 		log.Error(err, "create application client failed")
 		return err
 	}
+	s2iclient, err := s2iclientset.NewForConfig(cfg)
+	if err != nil {
+		log.Error(err, "create s2i client failed")
+		return err
+	}
+	kubesphereclient, err := kubesphereclientset.NewForConfig(cfg)
+	if err != nil {
+		log.Error(err, "create kubesphere client failed")
+		return err
+	}
 
 	informerFactory := informers.NewSharedInformerFactory(kubeClient, defaultResync)
 	istioInformer := istioinformers.NewSharedInformerFactory(istioclient, defaultResync)
 	applicationInformer := applicationinformers.NewSharedInformerFactory(applicationClient, defaultResync)
+	s2iInformer := s2iinformers.NewSharedInformerFactory(s2iclient, defaultResync)
 
-	servicemeshclient, err := servicemeshclientset.NewForConfig(cfg)
-	if err != nil {
-		log.Error(err, "create servicemesh client failed")
-		return err
-	}
-
-	servicemeshInformer := servicemeshinformers.NewSharedInformerFactory(servicemeshclient, defaultResync)
+	kubesphereInformer := kubesphereinformers.NewSharedInformerFactory(kubesphereclient, defaultResync)
 
 	vsController := virtualservice.NewVirtualServiceController(informerFactory.Core().V1().Services(),
 		istioInformer.Networking().V1alpha3().VirtualServices(),
 		istioInformer.Networking().V1alpha3().DestinationRules(),
-		servicemeshInformer.Servicemesh().V1alpha2().Strategies(),
+		kubesphereInformer.Servicemesh().V1alpha2().Strategies(),
 		kubeClient,
 		istioclient,
-		servicemeshclient)
+		kubesphereclient)
 
 	drController := destinationrule.NewDestinationRuleController(informerFactory.Apps().V1().Deployments(),
 		istioInformer.Networking().V1alpha3().DestinationRules(),
 		informerFactory.Core().V1().Services(),
-		servicemeshInformer.Servicemesh().V1alpha2().ServicePolicies(),
+		kubesphereInformer.Servicemesh().V1alpha2().ServicePolicies(),
 		kubeClient,
 		istioclient,
-		servicemeshclient)
+		kubesphereclient)
 
 	apController := application.NewApplicationController(informerFactory.Core().V1().Services(),
 		informerFactory.Apps().V1().Deployments(),
 		informerFactory.Apps().V1().StatefulSets(),
-		servicemeshInformer.Servicemesh().V1alpha2().Strategies(),
-		servicemeshInformer.Servicemesh().V1alpha2().ServicePolicies(),
+		kubesphereInformer.Servicemesh().V1alpha2().Strategies(),
+		kubesphereInformer.Servicemesh().V1alpha2().ServicePolicies(),
 		applicationInformer.App().V1beta1().Applications(),
 		kubeClient,
 		applicationClient)
 
 	jobController := job.NewJobController(informerFactory.Batch().V1().Jobs(), kubeClient)
 
-	servicemeshInformer.Start(stopCh)
+	s2iBinaryController := s2ibinary.NewController(kubesphereclient, kubeClient, kubesphereInformer.Devops().V1alpha1().S2iBinaries())
+
+	s2iRunController := s2irun.NewController(kubesphereclient, s2iclient, kubeClient,
+		kubesphereInformer.Devops().V1alpha1().S2iBinaries(),
+		s2iInformer.Devops().V1alpha1().S2iRuns())
+
+	kubesphereInformer.Start(stopCh)
 	istioInformer.Start(stopCh)
 	informerFactory.Start(stopCh)
 	applicationInformer.Start(stopCh)
+	s2iInformer.Start(stopCh)
 
 	controllers := map[string]manager.Runnable{
 		"virtualservice-controller":  vsController,
 		"destinationrule-controller": drController,
 		"application-controller":     apController,
 		"job-controller":             jobController,
+		"s2ibinary-controller":       s2iBinaryController,
+		"s2irun-controller":          s2iRunController,
 	}
 
 	for name, ctrl := range controllers {

cmd/controller-manager/controller-manager.go

@@ -20,69 +20,64 @@ package main
 
 import (
 	"flag"
-	"k8s.io/client-go/tools/clientcmd"
+	"k8s.io/klog"
 	"kubesphere.io/kubesphere/cmd/controller-manager/app"
 	"kubesphere.io/kubesphere/pkg/apis"
 	"kubesphere.io/kubesphere/pkg/controller"
+	"kubesphere.io/kubesphere/pkg/simple/client/k8s"
 	"os"
 	"sigs.k8s.io/controller-runtime/pkg/manager"
-	logf "sigs.k8s.io/controller-runtime/pkg/runtime/log"
 	"sigs.k8s.io/controller-runtime/pkg/runtime/signals"
 )
 
 var (
 	masterURL   string
-	kubeconfig  string
 	metricsAddr string
 )
 
 func init() {
-	flag.StringVar(&masterURL, "master-url", "", "only need if out of cluster")
-	flag.StringVar(&kubeconfig, "kubeconfig", "", "only need if out of cluster")
 	flag.StringVar(&metricsAddr, "metrics-addr", ":8080", "The address the metric endpoint binds to.")
 }
 
 func main() {
 	flag.Parse()
 
-	logf.SetLogger(logf.ZapLogger(false))
-	log := logf.Log.WithName("controller-manager")
 
-	cfg, err := clientcmd.BuildConfigFromFlags(masterURL, kubeconfig)
+	cfg, err := k8s.Config()
 	if err != nil {
-		log.Error(err, "failed to build kubeconfig")
+		klog.Error(err, "failed to build kubeconfig")
 		os.Exit(1)
 	}
 
 	stopCh := signals.SetupSignalHandler()
 
-	log.Info("setting up manager")
+	klog.Info("setting up manager")
 	mgr, err := manager.New(cfg, manager.Options{})
 	if err != nil {
-		log.Error(err, "unable to set up overall controller manager")
+		klog.Error(err, "unable to set up overall controller manager")
 		os.Exit(1)
 	}
 
-	log.Info("setting up scheme")
+	klog.Info("setting up scheme")
 	if err := apis.AddToScheme(mgr.GetScheme()); err != nil {
-		log.Error(err, "unable add APIs to scheme")
+		klog.Error(err, "unable add APIs to scheme")
 		os.Exit(1)
 	}
 
-	log.Info("Setting up controllers")
+	klog.Info("Setting up controllers")
 	if err := controller.AddToManager(mgr); err != nil {
-		log.Error(err, "unable to register controllers to the manager")
+		klog.Error(err, "unable to register controllers to the manager")
 		os.Exit(1)
 	}
 
 	if err := app.AddControllers(mgr, cfg, stopCh); err != nil {
-		log.Error(err, "unable to register controllers to the manager")
+		klog.Error(err, "unable to register controllers to the manager")
 		os.Exit(1)
 	}
 
-	log.Info("Starting the Cmd.")
+	klog.Info("Starting the Cmd.")
 	if err := mgr.Start(stopCh); err != nil {
-		log.Error(err, "unable to run the manager")
+		klog.Error(err, "unable to run the manager")
 		os.Exit(1)
 	}
 

cmd/ks-apiserver/app/server.go

@@ -190,6 +190,7 @@ func waitForResourceSync() {
 
 	ksInformerFactory := informers.KsSharedInformerFactory()
 	ksInformerFactory.Tenant().V1alpha1().Workspaces().Lister()
+	ksInformerFactory.Devops().V1alpha1().S2iBinaries().Lister()
 
 	ksInformerFactory.Start(stopChan)
 	ksInformerFactory.WaitForCacheSync(stopChan)

config/samples/devops_v1alpha1_s2ibinary.yaml

+apiVersion: devops.kubesphere.io/v1alpha1
+kind: S2iBinary
+metadata:
+  labels:
+    controller-tools.k8s.io: "1.0"
+  name: s2ibinary-sample

go.mod

@@ -4,6 +4,7 @@ go 1.12
 
 require (
 	bitbucket.org/ww/goautoneg v0.0.0-20120707110453-75cd24fc2f2c // indirect
+	code.cloudfoundry.org/bytefmt v0.0.0-20190710193110-1eb035ffe2b6
 	github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78 // indirect
 	github.com/Azure/go-autorest/autorest v0.5.0 // indirect
 	github.com/Microsoft/go-winio v0.4.12 // indirect
@@ -12,6 +13,7 @@ require (
 	github.com/Sirupsen/logrus v1.4.1 // indirect
 	github.com/appscode/jsonpatch v0.0.0-20190108182946-7c0e3b262f30 // indirect
 	github.com/asaskevich/govalidator v0.0.0-20180315120708-ccb8e960c48f
+	github.com/aws/aws-sdk-go v1.22.2
 	github.com/beevik/etree v1.1.0
 	github.com/beorn7/perks v1.0.0 // indirect
 	github.com/cenkalti/backoff v2.2.1+incompatible // indirect

go.sum

@@ -3,6 +3,8 @@ bitbucket.org/ww/goautoneg v0.0.0-20120707110453-75cd24fc2f2c/go.mod h1:1vhO7Mn/
 cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
 cloud.google.com/go v0.34.0 h1:eOI3/cP2VTU6uZLDYAoic+eyzzB9YyGmJ7eIjl8rOPg=
 cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+code.cloudfoundry.org/bytefmt v0.0.0-20190710193110-1eb035ffe2b6 h1:tW+ztA4A9UT9xnco5wUjW1oNi35k22eUEn9tNpPYVwE=
+code.cloudfoundry.org/bytefmt v0.0.0-20190710193110-1eb035ffe2b6/go.mod h1:wN/zk7mhREp/oviagqUXY3EwuHhWyOvAdsn5Y4CzOrc=
 contrib.go.opencensus.io/exporter/ocagent v0.4.12 h1:jGFvw3l57ViIVEPKKEUXPcLYIXJmQxLUh6ey1eJhwyc=
 contrib.go.opencensus.io/exporter/ocagent v0.4.12/go.mod h1:450APlNTSR6FrvC3CTRqYosuDstRB9un7SOx2k/9ckA=
 github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78 h1:w+iIsaOQNcT7OZ575w+acHgRric5iCyQh+xv+KJ4HB8=
@@ -50,6 +52,8 @@ github.com/appscode/jsonpatch v0.0.0-20190108182946-7c0e3b262f30 h1:Kn3rqvbUFqSe
 github.com/appscode/jsonpatch v0.0.0-20190108182946-7c0e3b262f30/go.mod h1:4AJxUpXUhv4N+ziTvIcWWXgeorXpxPZOfk9HdEVr96M=
 github.com/asaskevich/govalidator v0.0.0-20180315120708-ccb8e960c48f h1:y2hSFdXeA1y5z5f0vfNO0Dg5qVY036qzlz3Pds0B92o=
 github.com/asaskevich/govalidator v0.0.0-20180315120708-ccb8e960c48f/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
+github.com/aws/aws-sdk-go v1.22.2 h1:uYP58k2Cd9y1qBy8CxTe5ADmdi4kANm8Ul8ch3kkIcQ=
+github.com/aws/aws-sdk-go v1.22.2/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
 github.com/beevik/etree v1.1.0 h1:T0xke/WvNtMoCqgzPhkX2r4rjY3GDZFi+FjpRZY2Jbs=
 github.com/beevik/etree v1.1.0/go.mod h1:r8Aw8JqVegEf0w2fDnATrX9VpkMcyFeM0FhwO62wh+A=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
@@ -234,6 +238,8 @@ github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i
 github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
 github.com/jimstudt/http-authentication v0.0.0-20140401203705-3eca13d6893a h1:BcF8coBl0QFVhe8vAMMlD+CV8EISiu9MGKLoj6ZEyJA=
 github.com/jimstudt/http-authentication v0.0.0-20140401203705-3eca13d6893a/go.mod h1:wK6yTYYcgjHE1Z1QtXACPDjcFJyBskHEdagmnq3vsP8=
+github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM=
+github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
 github.com/jonboulle/clockwork v0.1.0 h1:VKV+ZcuP6l3yW9doeqz6ziZGgcynBVQO+obU0+0hcPo=
 github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
 github.com/json-iterator/go v1.1.6 h1:MrUvLMLTMxbqFJ9kzlvat/rYZqZnW3u4wkLzWTaFwKs=

pkg/apis/devops/crdinstall/install.go

+/*
+
+ Copyright 2019 The KubeSphere Authors.
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+
+*/
+
+package install
+
+import (
+	k8sruntime "k8s.io/apimachinery/pkg/runtime"
+	urlruntime "k8s.io/apimachinery/pkg/util/runtime"
+	devopsv1alpha1 "kubesphere.io/kubesphere/pkg/apis/devops/v1alpha1"
+)
+
+func Install(scheme *k8sruntime.Scheme) {
+	urlruntime.Must(devopsv1alpha1.AddToScheme(scheme))
+	urlruntime.Must(scheme.SetVersionPriority(devopsv1alpha1.SchemeGroupVersion))
+}

pkg/apis/devops/v1alpha1/s2ibinary_types.go

@@ -20,6 +20,23 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
+const (
+	ResourceKindS2iBinary         = "S2iBinary"
+	ResourceSingularServicePolicy = "s2ibinary"
+	ResourcePluralServicePolicy   = "s2ibinaries"
+)
+
+const (
+	StatusUploading    = "Uploading"
+	StatusReady        = "Ready"
+	StatusUploadFailed = "UploadFailed"
+)
+
+const (
+	S2iBinaryFinalizerName = "s2ibinary.finalizers.kubesphere.io"
+	S2iBinaryLabelKey      = "s2ibinary-name.kubesphere.io"
+)
+
 // EDIT THIS FILE!  THIS IS SCAFFOLDING FOR YOU TO OWN!
 // NOTE: json tags are required.  Any new fields you add must have json tags for the fields to be serialized.
 

pkg/apis/devops/v1alpha2/register.go

@@ -22,6 +22,7 @@ import (
 	"github.com/emicklei/go-restful"
 	"github.com/emicklei/go-restful-openapi"
 	"k8s.io/apimachinery/pkg/runtime/schema"
+	devopsv1alpha1 "kubesphere.io/kubesphere/pkg/apis/devops/v1alpha1"
 	devopsapi "kubesphere.io/kubesphere/pkg/apiserver/devops"
 	"kubesphere.io/kubesphere/pkg/apiserver/runtime"
 	"kubesphere.io/kubesphere/pkg/constants"
@@ -612,6 +613,26 @@ The last one is encrypted info, such as the password of the username-password ty
 		Returns(http.StatusOK, RespOK, devops.Crumb{}).
 		Writes(devops.Crumb{}))
 
+	webservice.Route(webservice.PUT("/namespaces/{namespace}/s2ibinaries/{s2ibinary}/file").
+		To(devopsapi.UploadS2iBinary).
+		Consumes("multipart/form-data").
+		Produces(restful.MIME_JSON).
+		Doc("Upload S2iBinary file").
+		Param(webservice.PathParameter("namespace", "the name of namespaces")).
+		Param(webservice.PathParameter("s2ibinary", "the name of s2ibinary")).
+		Param(webservice.FormParameter("s2ibinary", "file to upload")).
+		Param(webservice.FormParameter("md5", "md5 of file")).
+		Returns(http.StatusOK, RespOK, devopsv1alpha1.S2iBinary{}))
+
+	webservice.Route(webservice.GET("/namespaces/{namespace}/s2ibinaries/{s2ibinary}/file/{file}").
+		To(devopsapi.DownloadS2iBinary).
+		Produces(restful.MIME_OCTET).
+		Doc("Download S2iBinary file").
+		Param(webservice.PathParameter("namespace", "the name of namespaces")).
+		Param(webservice.PathParameter("s2ibinary", "the name of s2ibinary")).
+		Param(webservice.PathParameter("file","the name of binary file")).
+		Returns(http.StatusOK, RespOK, nil))
+
 	// TODO are not used in this version. will be added in 2.1.0
 	//// match /job/init-job/descriptorByName/org.jenkinsci.plugins.workflow.cps.CpsFlowDefinition/checkScriptCompile
 	//webservice.Route(webservice.POST("/devops/check/scriptcompile").

pkg/apiserver/devops/s2ibinary.go

+package devops
+
+import (
+	"code.cloudfoundry.org/bytefmt"
+	"fmt"
+	"github.com/emicklei/go-restful"
+	"k8s.io/klog"
+	"kubesphere.io/kubesphere/pkg/errors"
+	"kubesphere.io/kubesphere/pkg/models/devops"
+	"kubesphere.io/kubesphere/pkg/utils/hashutil"
+	"net/http"
+)
+
+func UploadS2iBinary(req *restful.Request, resp *restful.Response) {
+	ns := req.PathParameter("namespace")
+	name := req.PathParameter("s2ibinary")
+
+	err := req.Request.ParseMultipartForm(bytefmt.MEGABYTE * 20)
+	if err != nil {
+		klog.Errorf("%+v", err)
+		errors.ParseSvcErr(restful.NewError(http.StatusBadRequest, err.Error()), resp)
+		return
+	}
+	if len(req.Request.MultipartForm.File) == 0 {
+		err := restful.NewError(http.StatusBadRequest, "could not get file from form")
+		klog.Errorf("%+v", err)
+		errors.ParseSvcErr(restful.NewError(http.StatusBadRequest, err.Error()), resp)
+		return
+	}
+	if len(req.Request.MultipartForm.File["s2ibinary"]) == 0 {
+		err := restful.NewError(http.StatusBadRequest, "could not get file from form")
+		klog.Errorf("%+v", err)
+		errors.ParseSvcErr(err, resp)
+		return
+	}
+	if len(req.Request.MultipartForm.File["s2ibinary"]) > 1 {
+		err := restful.NewError(http.StatusBadRequest, "s2ibinary should only have one file")
+		klog.Errorf("%+v", err)
+		errors.ParseSvcErr(err, resp)
+		return
+	}
+	defer req.Request.MultipartForm.RemoveAll()
+	file, err := req.Request.MultipartForm.File["s2ibinary"][0].Open()
+	if err != nil {
+		klog.Error(err)
+		errors.ParseSvcErr(err, resp)
+		return
+	}
+	filemd5, err := hashutil.GetMD5(file)
+	if err != nil {
+		klog.Error(err)
+		errors.ParseSvcErr(err, resp)
+		return
+	}
+	md5, ok := req.Request.MultipartForm.Value["md5"]
+	if ok && len(req.Request.MultipartForm.Value["md5"]) > 0 {
+		if md5[0] != filemd5 {
+			err := restful.NewError(http.StatusBadRequest, fmt.Sprintf("md5 not match, origin: %+v, calculate: %+v", md5[0], filemd5))
+			klog.Error(err)
+			errors.ParseSvcErr(err, resp)
+			return
+		}
+	}
+
+	s2ibin, err := devops.UploadS2iBinary(ns, name, filemd5, req.Request.MultipartForm.File["s2ibinary"][0])
+	if err != nil {
+		klog.Errorf("%+v", err)
+		errors.ParseSvcErr(err, resp)
+		return
+	}
+	resp.WriteAsJson(s2ibin)
+
+}
+
+func DownloadS2iBinary(req *restful.Request, resp *restful.Response) {
+	ns := req.PathParameter("namespace")
+	name := req.PathParameter("s2ibinary")
+	fileName := req.PathParameter("file")
+	url, err := devops.DownloadS2iBinary(ns, name, fileName)
+	if err != nil {
+		klog.Errorf("%+v", err)
+		errors.ParseSvcErr(err, resp)
+		return
+	}
+	http.Redirect(resp.ResponseWriter, req.Request, url, http.StatusFound)
+	return
+}

pkg/controller/s2ibinary/OWNERS

+approvers:
+    - runzexia
+    - soulseen
+
+reviewers:
+    - runzexia
+    - soulseen
+
+labels:
+    - area/controller
+    - area/devops

pkg/controller/s2ibinary/s2ibinary_controller.go

+package s2ibinary
+
+import (
+	"fmt"
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/awserr"
+	"github.com/aws/aws-sdk-go/service/s3"
+	v1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/errors"
+	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
+	"k8s.io/apimachinery/pkg/util/wait"
+	clientset "k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/kubernetes/scheme"
+	v1core "k8s.io/client-go/kubernetes/typed/core/v1"
+	"k8s.io/client-go/tools/cache"
+	"k8s.io/client-go/tools/record"
+	"k8s.io/client-go/util/workqueue"
+	"k8s.io/klog"
+	"k8s.io/kubernetes/pkg/util/metrics"
+	"kubesphere.io/kubesphere/pkg/simple/client/s2is3"
+	"kubesphere.io/kubesphere/pkg/utils/sliceutil"
+	"time"
+
+	devopsv1alpha1 "kubesphere.io/kubesphere/pkg/apis/devops/v1alpha1"
+	devopsclient "kubesphere.io/kubesphere/pkg/client/clientset/versioned"
+	devopsinformers "kubesphere.io/kubesphere/pkg/client/informers/externalversions/devops/v1alpha1"
+	devopslisters "kubesphere.io/kubesphere/pkg/client/listers/devops/v1alpha1"
+)
+
+type S2iBinaryController struct {
+	client       clientset.Interface
+	devopsClient devopsclient.Interface
+
+	eventBroadcaster record.EventBroadcaster
+	eventRecorder    record.EventRecorder
+
+	s2iBinaryLister devopslisters.S2iBinaryLister
+	s2iBinarySynced cache.InformerSynced
+
+	workqueue workqueue.RateLimitingInterface
+
+	workerLoopPeriod time.Duration
+}
+
+func NewController(devopsclientset devopsclient.Interface,
+	client clientset.Interface,
+	s2ibinInformer devopsinformers.S2iBinaryInformer) *S2iBinaryController {
+
+	broadcaster := record.NewBroadcaster()
+	broadcaster.StartLogging(func(format string, args ...interface{}) {
+		klog.Info(fmt.Sprintf(format, args))
+	})
+	broadcaster.StartRecordingToSink(&v1core.EventSinkImpl{Interface: client.CoreV1().Events("")})
+	recorder := broadcaster.NewRecorder(scheme.Scheme, v1.EventSource{Component: "s2ibinary-controller"})
+
+	if client != nil && client.CoreV1().RESTClient().GetRateLimiter() != nil {
+		metrics.RegisterMetricAndTrackRateLimiterUsage("s2ibinary_controller", client.CoreV1().RESTClient().GetRateLimiter())
+	}
+
+	v := &S2iBinaryController{
+		client:           client,
+		devopsClient:     devopsclientset,
+		workqueue:        workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "s2ibinary"),
+		s2iBinaryLister:  s2ibinInformer.Lister(),
+		s2iBinarySynced:  s2ibinInformer.Informer().HasSynced,
+		workerLoopPeriod: time.Second,
+	}
+
+	v.eventBroadcaster = broadcaster
+	v.eventRecorder = recorder
+
+	s2ibinInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
+		AddFunc: v.enqueueS2iBinary,
+		UpdateFunc: func(oldObj, newObj interface{}) {
+			old := oldObj.(*devopsv1alpha1.S2iBinary)
+			new := newObj.(*devopsv1alpha1.S2iBinary)
+			if old.ResourceVersion == new.ResourceVersion {
+				return
+			}
+			v.enqueueS2iBinary(newObj)
+		},
+		DeleteFunc: v.enqueueS2iBinary,
+	})
+	return v
+}
+
+// enqueueS2iBinary takes a Foo resource and converts it into a namespace/name
+// string which is then put onto the work workqueue. This method should *not* be
+// passed resources of any type other than S2iBinary.
+func (c *S2iBinaryController) enqueueS2iBinary(obj interface{}) {
+	var key string
+	var err error
+	if key, err = cache.MetaNamespaceKeyFunc(obj); err != nil {
+		utilruntime.HandleError(err)
+		return
+	}
+	c.workqueue.Add(key)
+}
+
+func (c *S2iBinaryController) processNextWorkItem() bool {
+	obj, shutdown := c.workqueue.Get()
+
+	if shutdown {
+		return false
+	}
+
+	err := func(obj interface{}) error {
+		defer c.workqueue.Done(obj)
+		var key string
+		var ok bool
+
+		if key, ok = obj.(string); !ok {
+			c.workqueue.Forget(obj)
+			utilruntime.HandleError(fmt.Errorf("expected string in workqueue but got %#v", obj))
+			return nil
+		}
+		if err := c.syncHandler(key); err != nil {
+			c.workqueue.AddRateLimited(key)
+			return fmt.Errorf("error syncing '%s': %s, requeuing", key, err.Error())
+		}
+		c.workqueue.Forget(obj)
+		klog.V(5).Infof("Successfully synced '%s'", key)
+		return nil
+	}(obj)
+
+	if err != nil {
+		klog.Error(err, "could not reconcile s2ibinary")
+		utilruntime.HandleError(err)
+		return true
+	}
+
+	return true
+}
+
+func (c *S2iBinaryController) worker() {
+
+	for c.processNextWorkItem() {
+	}
+}
+
+func (c *S2iBinaryController) Start(stopCh <-chan struct{}) error {
+	return c.Run(1, stopCh)
+}
+
+func (c *S2iBinaryController) Run(workers int, stopCh <-chan struct{}) error {
+	defer utilruntime.HandleCrash()
+	defer c.workqueue.ShutDown()
+
+	klog.Info("starting s2ibinary controller")
+	defer klog.Info("shutting down s2ibinary controller")
+
+	if !cache.WaitForCacheSync(stopCh, c.s2iBinarySynced) {
+		return fmt.Errorf("failed to wait for caches to sync")
+	}
+
+	for i := 0; i < workers; i++ {
+		go wait.Until(c.worker, c.workerLoopPeriod, stopCh)
+	}
+
+	<-stopCh
+	return nil
+}
+
+// syncHandler compares the actual state with the desired, and attempts to
+// converge the two. It then updates the Status block of the Foo resource
+// with the current status of the resource.
+func (c *S2iBinaryController) syncHandler(key string) error {
+	namespace, name, err := cache.SplitMetaNamespaceKey(key)
+	if err != nil {
+		klog.Error(err, fmt.Sprintf("could not split s2ibin meta %s ", key))
+		return nil
+	}
+	s2ibin, err := c.s2iBinaryLister.S2iBinaries(namespace).Get(name)
+	if err != nil {
+		if errors.IsNotFound(err) {
+			klog.Info(fmt.Sprintf("s2ibin '%s' in work queue no longer exists ", key))
+			return nil
+		}
+		klog.Error(err, fmt.Sprintf("could not get s2ibin %s ", key))
+		return err
+	}
+	if s2ibin.ObjectMeta.DeletionTimestamp.IsZero() {
+		if !sliceutil.HasString(s2ibin.ObjectMeta.Finalizers, devopsv1alpha1.S2iBinaryFinalizerName) {
+			s2ibin.ObjectMeta.Finalizers = append(s2ibin.ObjectMeta.Finalizers, devopsv1alpha1.S2iBinaryFinalizerName)
+			_, err := c.devopsClient.DevopsV1alpha1().S2iBinaries(namespace).Update(s2ibin)
+			if err != nil {
+				klog.Error(err, fmt.Sprintf("failed to update s2ibin %s ", key))
+				return err
+			}
+		}
+
+	} else {
+		if sliceutil.HasString(s2ibin.ObjectMeta.Finalizers, devopsv1alpha1.S2iBinaryFinalizerName) {
+			if err := c.deleteBinaryInS3(s2ibin); err != nil {
+				klog.Error(err, fmt.Sprintf("failed to delete resource %s in s3", key))
+				return err
+			}
+			s2ibin.ObjectMeta.Finalizers = sliceutil.RemoveString(s2ibin.ObjectMeta.Finalizers, func(item string) bool {
+				return item == devopsv1alpha1.S2iBinaryFinalizerName
+			})
+			_, err := c.devopsClient.DevopsV1alpha1().S2iBinaries(namespace).Update(s2ibin)
+			if err != nil {
+				klog.Error(err, fmt.Sprintf("failed to update s2ibin %s ", key))
+				return err
+			}
+		}
+	}
+
+	return nil
+}
+
+func (c *S2iBinaryController) deleteBinaryInS3(s2ibin *devopsv1alpha1.S2iBinary) error {
+	s3client := s2is3.Client()
+	input := &s3.DeleteObjectInput{
+		Bucket: s2is3.Bucket(),
+		Key:    aws.String(fmt.Sprintf("%s-%s", s2ibin.Namespace, s2ibin.Name)),
+	}
+	_, err := s3client.DeleteObject(input)
+	if err != nil {
+		if aerr, ok := err.(awserr.Error); ok {
+			switch aerr.Code() {
+			case s3.ErrCodeNoSuchKey:
+				return nil
+			default:
+				klog.Error(err, fmt.Sprintf("failed to delete s2ibin %s/%s in s3", s2ibin.Namespace, s2ibin.Name))
+				return err
+			}
+		} else {
+			klog.Error(err, fmt.Sprintf("failed to delete s2ibin %s/%s in s3", s2ibin.Namespace, s2ibin.Name))
+			return err
+		}
+	}
+	return nil
+}

pkg/controller/s2irun/OWNERS

+approvers:
+    - runzexia
+    - soulseen
+
+reviewers:
+    - runzexia
+    - soulseen
+
+labels:
+    - area/controller
+    - area/devops

pkg/controller/s2irun/s2irun_controller.go

+package s2irun
+
+import (
+	"fmt"
+	v1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/errors"
+	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
+	"k8s.io/apimachinery/pkg/util/wait"
+	clientset "k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/kubernetes/scheme"
+	v1core "k8s.io/client-go/kubernetes/typed/core/v1"
+	"k8s.io/client-go/tools/cache"
+	"k8s.io/client-go/tools/record"
+	"k8s.io/client-go/util/workqueue"
+	"k8s.io/klog"
+	"k8s.io/kubernetes/pkg/util/metrics"
+	"kubesphere.io/kubesphere/pkg/utils/sliceutil"
+	"time"
+
+	s2iv1alpha1 "github.com/kubesphere/s2ioperator/pkg/apis/devops/v1alpha1"
+	s2iclient "github.com/kubesphere/s2ioperator/pkg/client/clientset/versioned"
+	s2iinformers "github.com/kubesphere/s2ioperator/pkg/client/informers/externalversions/devops/v1alpha1"
+	s2ilisters "github.com/kubesphere/s2ioperator/pkg/client/listers/devops/v1alpha1"
+	devopsv1alpha1 "kubesphere.io/kubesphere/pkg/apis/devops/v1alpha1"
+	devopsclient "kubesphere.io/kubesphere/pkg/client/clientset/versioned"
+	devopsinformers "kubesphere.io/kubesphere/pkg/client/informers/externalversions/devops/v1alpha1"
+	devopslisters "kubesphere.io/kubesphere/pkg/client/listers/devops/v1alpha1"
+)
+
+type S2iRunController struct {
+	client    clientset.Interface
+	s2iClient s2iclient.Interface
+
+	devopsClient devopsclient.Interface
+
+	eventBroadcaster record.EventBroadcaster
+	eventRecorder    record.EventRecorder
+
+	s2iRunLister s2ilisters.S2iRunLister
+	s2iRunSynced cache.InformerSynced
+
+	s2iBinaryLister devopslisters.S2iBinaryLister
+	s2iBinarySynced cache.InformerSynced
+
+	workqueue workqueue.RateLimitingInterface
+
+	workerLoopPeriod time.Duration
+}
+
+func NewController(devopsclientset devopsclient.Interface, s2iclientset s2iclient.Interface,
+	client clientset.Interface,
+	s2ibinInformer devopsinformers.S2iBinaryInformer, s2iRunInformer s2iinformers.S2iRunInformer) *S2iRunController {
+
+	broadcaster := record.NewBroadcaster()
+	broadcaster.StartLogging(func(format string, args ...interface{}) {
+		klog.Info(fmt.Sprintf(format, args))
+	})
+	broadcaster.StartRecordingToSink(&v1core.EventSinkImpl{Interface: client.CoreV1().Events("")})
+	recorder := broadcaster.NewRecorder(scheme.Scheme, v1.EventSource{Component: "s2irun-controller"})
+
+	if client != nil && client.CoreV1().RESTClient().GetRateLimiter() != nil {
+		metrics.RegisterMetricAndTrackRateLimiterUsage("s2irun_controller", client.CoreV1().RESTClient().GetRateLimiter())
+	}
+
+	v := &S2iRunController{
+		client:           client,
+		devopsClient:     devopsclientset,
+		s2iClient:        s2iclientset,
+		workqueue:        workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "s2irun"),
+		s2iBinaryLister:  s2ibinInformer.Lister(),
+		s2iBinarySynced:  s2ibinInformer.Informer().HasSynced,
+		s2iRunLister:     s2iRunInformer.Lister(),
+		s2iRunSynced:     s2iRunInformer.Informer().HasSynced,
+		workerLoopPeriod: time.Second,
+	}
+
+	v.eventBroadcaster = broadcaster
+	v.eventRecorder = recorder
+
+	s2iRunInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
+		AddFunc: v.enqueueS2iRun,
+		UpdateFunc: func(oldObj, newObj interface{}) {
+			old := oldObj.(*s2iv1alpha1.S2iRun)
+			new := newObj.(*s2iv1alpha1.S2iRun)
+			if old.ResourceVersion == new.ResourceVersion {
+				return
+			}
+			v.enqueueS2iRun(newObj)
+		},
+		DeleteFunc: v.enqueueS2iRun,
+	})
+	return v
+}
+
+// enqueueFoo takes a Foo resource and converts it into a namespace/name
+// string which is then put onto the work workqueue. This method should *not* be
+// passed resources of any type other than Foo.
+func (c *S2iRunController) enqueueS2iRun(obj interface{}) {
+	var key string
+	var err error
+	if key, err = cache.MetaNamespaceKeyFunc(obj); err != nil {
+		utilruntime.HandleError(err)
+		return
+	}
+	c.workqueue.Add(key)
+}
+
+func (c *S2iRunController) processNextWorkItem() bool {
+	obj, shutdown := c.workqueue.Get()
+
+	if shutdown {
+		return false
+	}
+
+	err := func(obj interface{}) error {
+		defer c.workqueue.Done(obj)
+		var key string
+		var ok bool
+		if key, ok = obj.(string); !ok {
+			c.workqueue.Forget(obj)
+			utilruntime.HandleError(fmt.Errorf("expected string in workqueue but got %#v", obj))
+			return nil
+		}
+		if err := c.syncHandler(key); err != nil {
+			c.workqueue.AddRateLimited(key)
+			return fmt.Errorf("error syncing '%s': %s, requeuing", key, err.Error())
+		}
+		c.workqueue.Forget(obj)
+		klog.V(5).Infof("Successfully synced '%s'", key)
+		return nil
+	}(obj)
+
+	if err != nil {
+		klog.Error(err, "could not reconcile s2irun")
+		utilruntime.HandleError(err)
+		return true
+	}
+
+	return true
+}
+
+func (c *S2iRunController) worker() {
+
+	for c.processNextWorkItem() {
+	}
+}
+
+func (c *S2iRunController) Start(stopCh <-chan struct{}) error {
+	return c.Run(1, stopCh)
+}
+
+func (c *S2iRunController) Run(workers int, stopCh <-chan struct{}) error {
+	defer utilruntime.HandleCrash()
+	defer c.workqueue.ShutDown()
+
+	klog.Info("starting s2irun controller")
+	defer klog.Info("shutting down s2irun controller")
+
+	if !cache.WaitForCacheSync(stopCh, c.s2iBinarySynced) {
+		return fmt.Errorf("failed to wait for caches to sync")
+	}
+
+	for i := 0; i < workers; i++ {
+		go wait.Until(c.worker, c.workerLoopPeriod, stopCh)
+	}
+
+	<-stopCh
+	return nil
+}
+
+// syncHandler compares the actual state with the desired, and attempts to
+// converge the two. It then updates the Status block of the Foo resource
+// with the current status of the resource.
+func (c *S2iRunController) syncHandler(key string) error {
+	namespace, name, err := cache.SplitMetaNamespaceKey(key)
+	if err != nil {
+		klog.Error(err, fmt.Sprintf("could not split s2irun meta %s ", key))
+		return nil
+	}
+	s2irun, err := c.s2iRunLister.S2iRuns(namespace).Get(name)
+	if err != nil {
+		if errors.IsNotFound(err) {
+			klog.Info(fmt.Sprintf("s2irun '%s' in work queue no longer exists ", key))
+			return nil
+		}
+		klog.Error(err, fmt.Sprintf("could not get s2irun %s ", key))
+		return err
+	}
+	if s2irun.Labels != nil {
+		_, ok := s2irun.Labels[devopsv1alpha1.S2iBinaryLabelKey]
+		if ok {
+			if s2irun.ObjectMeta.DeletionTimestamp.IsZero() {
+				if !sliceutil.HasString(s2irun.ObjectMeta.Finalizers, devopsv1alpha1.S2iBinaryFinalizerName) {
+					s2irun.ObjectMeta.Finalizers = append(s2irun.ObjectMeta.Finalizers, devopsv1alpha1.S2iBinaryFinalizerName)
+					_, err := c.s2iClient.DevopsV1alpha1().S2iRuns(namespace).Update(s2irun)
+					if err != nil {
+						klog.Error(err, fmt.Sprintf("failed to update s2irun %s", key))
+						return err
+					}
+				}
+
+			} else {
+				if sliceutil.HasString(s2irun.ObjectMeta.Finalizers, devopsv1alpha1.S2iBinaryFinalizerName) {
+					if err := c.DeleteS2iBinary(s2irun); err != nil {
+						klog.Error(err, fmt.Sprintf("failed to delete s2ibin %s in", key))
+						return err
+					}
+					s2irun.ObjectMeta.Finalizers = sliceutil.RemoveString(s2irun.ObjectMeta.Finalizers, func(item string) bool {
+						return item == devopsv1alpha1.S2iBinaryFinalizerName
+					})
+					_, err := c.s2iClient.DevopsV1alpha1().S2iRuns(namespace).Update(s2irun)
+					if err != nil {
+						klog.Error(err, fmt.Sprintf("failed to update s2irun %s ", key))
+						return err
+					}
+				}
+			}
+		}
+	}
+
+	return nil
+}
+
+func (c *S2iRunController) DeleteS2iBinary(s2irun *s2iv1alpha1.S2iRun) error {
+	s2iBinName := s2irun.Labels[devopsv1alpha1.S2iBinaryLabelKey]
+	s2iBin, err := c.s2iBinaryLister.S2iBinaries(s2irun.Namespace).Get(s2iBinName)
+	if err != nil {
+		if errors.IsNotFound(err) {
+			klog.Info(fmt.Sprintf("s2ibin '%s/%s' has been delted ", s2irun.Namespace, s2iBinName))
+			return nil
+		}
+		klog.Error(err, fmt.Sprintf("failed to get s2ibin %s/%s ", s2irun.Namespace, s2iBinName))
+		return err
+	}
+	err = c.devopsClient.DevopsV1alpha1().S2iBinaries(s2iBin.Namespace).Delete(s2iBinName, nil)
+	if err != nil {
+		if errors.IsNotFound(err) {
+			klog.Info(fmt.Sprintf("s2ibin '%s/%s' has been delted ", s2irun.Namespace, s2iBinName))
+			return nil
+		}
+		klog.Error(err, fmt.Sprintf("failed to delete s2ibin %s/%s ", s2irun.Namespace, s2iBinName))
+		return err
+	}
+
+	return nil
+}

pkg/models/devops/s2ibinary_handler.go

+package devops
+
+import (
+	"code.cloudfoundry.org/bytefmt"
+	"fmt"
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/awserr"
+	"github.com/aws/aws-sdk-go/service/s3"
+	"github.com/aws/aws-sdk-go/service/s3/s3manager"
+	"github.com/emicklei/go-restful"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/util/retry"
+	"k8s.io/klog"
+	"kubesphere.io/kubesphere/pkg/apis/devops/v1alpha1"
+	"kubesphere.io/kubesphere/pkg/informers"
+	"kubesphere.io/kubesphere/pkg/simple/client/k8s"
+	"kubesphere.io/kubesphere/pkg/simple/client/s2is3"
+	"mime/multipart"
+	"net/http"
+	"reflect"
+	"time"
+)
+
+const (
+	GetS2iBinaryURL = "kapis/devops.kubesphere.io/v1alpha2/namespaces/%s/s2ibinaries/%s/file/%s"
+)
+
+func UploadS2iBinary(namespace, name, md5 string, fileHeader *multipart.FileHeader) (*v1alpha1.S2iBinary, error) {
+	binFile, err := fileHeader.Open()
+	if err != nil {
+		klog.Errorf("%+v", err)
+		return nil, err
+	}
+	defer binFile.Close()
+
+	origin, err := informers.KsSharedInformerFactory().Devops().V1alpha1().S2iBinaries().Lister().S2iBinaries(namespace).Get(name)
+	if err != nil {
+		klog.Errorf("%+v", err)
+		return nil, err
+	}
+	//Check file is uploading
+	if origin.Status.Phase == v1alpha1.StatusUploading {
+		err := restful.NewError(http.StatusConflict, "file is uploading, please try later")
+		klog.Error(err)
+		return nil, err
+	}
+	copy := origin.DeepCopy()
+	copy.Spec.MD5 = md5
+	copy.Spec.Size = bytefmt.ByteSize(uint64(fileHeader.Size))
+	copy.Spec.FileName = fileHeader.Filename
+	copy.Spec.DownloadURL = fmt.Sprintf(GetS2iBinaryURL, namespace, name, copy.Spec.FileName)
+	if origin.Status.Phase == v1alpha1.StatusReady && reflect.DeepEqual(origin, copy) {
+		return origin, nil
+	}
+
+	//Set status Uploading to lock resource
+	uploading, err := SetS2iBinaryStatus(copy, v1alpha1.StatusUploading)
+	if err != nil {
+		err := restful.NewError(http.StatusConflict, fmt.Sprintf("could not set status: %+v", err))
+		klog.Error(err)
+		return nil, err
+	}
+
+	copy = uploading.DeepCopy()
+	copy.Spec.MD5 = md5
+	copy.Spec.Size = bytefmt.ByteSize(uint64(fileHeader.Size))
+	copy.Spec.FileName = fileHeader.Filename
+	copy.Spec.DownloadURL = fmt.Sprintf(GetS2iBinaryURL, namespace, name, copy.Spec.FileName)
+	s3session := s2is3.Session()
+	if s3session == nil {
+		err := fmt.Errorf("could not connect to s2i s3")
+		klog.Error(err)
+		_, serr := SetS2iBinaryStatusWithRetry(copy, origin.Status.Phase)
+		if serr != nil {
+			klog.Error(serr)
+			return nil, err
+		}
+		return nil, err
+	}
+	uploader := s3manager.NewUploader(s3session, func(uploader *s3manager.Uploader) {
+		uploader.PartSize = 5 * bytefmt.MEGABYTE
+		uploader.LeavePartsOnError = true
+	})
+	_, err = uploader.Upload(&s3manager.UploadInput{
+		Bucket:             s2is3.Bucket(),
+		Key:                aws.String(fmt.Sprintf("%s-%s", namespace, name)),
+		Body:               binFile,
+		ContentMD5:         aws.String(md5),
+		ContentDisposition: aws.String(fmt.Sprintf("attachment; filename=\"%s\"", copy.Spec.FileName)),
+	})
+
+	if err != nil {
+		if aerr, ok := err.(awserr.Error); ok {
+			switch aerr.Code() {
+			case s3.ErrCodeNoSuchBucket:
+				klog.Error(err)
+				_, serr := SetS2iBinaryStatusWithRetry(copy, origin.Status.Phase)
+				if serr != nil {
+					klog.Error(serr)
+				}
+				return nil, err
+			default:
+				klog.Error(err)
+				_, serr := SetS2iBinaryStatusWithRetry(copy, v1alpha1.StatusUploadFailed)
+				if serr != nil {
+					klog.Error(serr)
+				}
+				return nil, err
+			}
+		}
+		klog.Error(err)
+		return nil, err
+	}
+
+	if copy.Spec.UploadTimeStamp == nil {
+		copy.Spec.UploadTimeStamp = new(metav1.Time)
+	}
+	*copy.Spec.UploadTimeStamp = metav1.Now()
+	copy, err = k8s.KsClient().DevopsV1alpha1().S2iBinaries(namespace).Update(copy)
+	if err != nil {
+		klog.Error(err)
+		return nil, err
+	}
+
+	copy, err = SetS2iBinaryStatusWithRetry(copy, v1alpha1.StatusReady)
+	if err != nil {
+		klog.Error(err)
+		return nil, err
+	}
+	return copy, nil
+}
+
+func DownloadS2iBinary(namespace, name, fileName string) (string, error) {
+	origin, err := informers.KsSharedInformerFactory().Devops().V1alpha1().S2iBinaries().Lister().S2iBinaries(namespace).Get(name)
+	if err != nil {
+		klog.Errorf("%+v", err)
+		return "", err
+	}
+	if origin.Spec.FileName != fileName {
+		err := fmt.Errorf("could not fould file %s", fileName)
+		klog.Error(err)
+		return "", err
+	}
+	if origin.Status.Phase != v1alpha1.StatusReady {
+		err := restful.NewError(http.StatusBadRequest, "file is not ready, please try later")
+		klog.Error(err)
+		return "", err
+	}
+	s3Client := s2is3.Client()
+	if s3Client == nil {
+		err := fmt.Errorf("could not get s3 client")
+		klog.Error(err)
+		return "", err
+	}
+	req, _ := s3Client.GetObjectRequest(&s3.GetObjectInput{
+		Bucket:                     s2is3.Bucket(),
+		Key:                        aws.String(fmt.Sprintf("%s-%s", namespace, name)),
+		ResponseContentDisposition: aws.String(fmt.Sprintf("attachment; filename=\"%s\"", origin.Spec.FileName)),
+	})
+	url, err := req.Presign(5 * time.Minute)
+	if err != nil {
+		klog.Error(err)
+		return "", err
+	}
+	return url, nil
+
+}
+
+func SetS2iBinaryStatus(s2ibin *v1alpha1.S2iBinary, status string) (*v1alpha1.S2iBinary, error) {
+	copy := s2ibin.DeepCopy()
+	copy.Status.Phase = status
+	copy, err := k8s.KsClient().DevopsV1alpha1().S2iBinaries(s2ibin.Namespace).Update(copy)
+	if err != nil {
+		klog.Error(err)
+		return nil, err
+	}
+	return copy, nil
+}
+
+func SetS2iBinaryStatusWithRetry(s2ibin *v1alpha1.S2iBinary, status string) (*v1alpha1.S2iBinary, error) {
+
+	var bin *v1alpha1.S2iBinary
+	var err error
+	err = retry.RetryOnConflict(retry.DefaultRetry, func() error {
+		bin, err = informers.KsSharedInformerFactory().Devops().V1alpha1().S2iBinaries().Lister().S2iBinaries(s2ibin.Namespace).Get(s2ibin.Name)
+		if err != nil {
+			klog.Error(err)
+			return err
+		}
+		bin.Status.Phase = status
+		bin, err = k8s.KsClient().DevopsV1alpha1().S2iBinaries(s2ibin.Namespace).Update(bin)
+		if err != nil {
+			klog.Error(err)
+			return err
+		}
+		return nil
+	})
+	if err != nil {
+		klog.Error(err)
+		return nil, err
+	}
+
+	return bin, nil
+}

pkg/simple/client/k8s/k8sclient.go

@@ -38,7 +38,7 @@ var (
 )
 
 func init() {
-	flag.StringVar(&kubeConfigFile, "kubeconfig", "", "path to kubeconfig file")
+	flag.StringVar(&kubeConfigFile, "kubeconfig-path", "", "path to kubeconfig file")
 	flag.StringVar(&MasterURL, "master-url", "", "kube-apiserver url, only needed when out of cluster")
 }
 

pkg/simple/client/s2is3/s3.go

+package s2is3
+
+import (
+	"flag"
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/credentials"
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/s3"
+	"k8s.io/klog"
+	"sync"
+)
+
+var (
+	s3Region          string
+	s3Endpoint        string
+	s3DisableSSL      bool
+	s3ForcePathStyle  bool
+	s3AccessKeyID     string
+	s3SecretAccessKey string
+	s3SessionToken    string
+	s3Bucket          string
+)
+var (
+	s2iS3            *s3.S3
+	s2iS3Session     *session.Session
+	sessionInitMutex sync.Mutex
+	clientInitMutex  sync.Mutex
+)
+
+func init() {
+	flag.StringVar(&s3Region, "s2i-s3-region", "us-east-1", "region of s2i s3")
+	flag.StringVar(&s3Endpoint, "s2i-s3-endpoint", "http://ks-minio.kubesphere-system.svc", "endpoint of s2i s3")
+	flag.StringVar(&s3AccessKeyID, "s2i-s3-access-key-id", "AKIAIOSFODNN7EXAMPLE", "access key of s2i s3")
+	flag.StringVar(&s3SecretAccessKey, "s2i-s3-secret-access-key", "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", "secret access key of s2i s3")
+	flag.StringVar(&s3SessionToken, "s2i-s3-session-token", "", "session token of s2i s3")
+	flag.StringVar(&s3Bucket, "s2i-s3-bucket", "s2i-binaries", "bucket name of s2i s3")
+	flag.BoolVar(&s3DisableSSL, "s2i-s3-disable-SSL", true, "disable ssl")
+	flag.BoolVar(&s3ForcePathStyle, "s2i-s3-force-path-style", true, "force path style")
+}
+
+func Client() *s3.S3 {
+	if s2iS3 != nil {
+		return s2iS3
+	}
+	clientInitMutex.Lock()
+	defer clientInitMutex.Unlock()
+	if s2iS3Session == nil {
+		if sess := Session(); sess != nil {
+			klog.Error("failed to connect to s2i s3")
+			return nil
+		}
+	}
+	s2iS3 = s3.New(s2iS3Session)
+	return s2iS3
+}
+func Session() *session.Session {
+	if s2iS3Session != nil {
+		return s2iS3Session
+	}
+	sessionInitMutex.Lock()
+	defer sessionInitMutex.Unlock()
+	creds := credentials.NewStaticCredentials(
+		s3AccessKeyID, s3SecretAccessKey, s3SessionToken,
+	)
+	config := &aws.Config{
+		Region:           aws.String(s3Region),
+		Endpoint:         aws.String(s3Endpoint),
+		DisableSSL:       aws.Bool(s3DisableSSL),
+		S3ForcePathStyle: aws.Bool(s3ForcePathStyle),
+		Credentials:      creds,
+	}
+	sess, err := session.NewSession(config)
+	if err != nil {
+		klog.Errorf("failed to connect to s2i s3: %+v", err)
+		return nil
+	}
+	s2iS3Session = sess
+	return s2iS3Session
+}
+
+func Bucket() *string {
+	return aws.String(s3Bucket)
+}

pkg/utils/hashutil/MD5.go

+package hashutil
+
+import (
+	"code.cloudfoundry.org/bytefmt"
+	"encoding/hex"
+	"github.com/golang/glog"
+	"io"
+
+	"kubesphere.io/kubesphere/pkg/utils/readerutils"
+)
+
+func GetMD5(reader io.ReadCloser) (string, error) {
+	md5reader := readerutils.NewMD5Reader(reader)
+	data := make([]byte, bytefmt.KILOBYTE)
+	for {
+		_, err := md5reader.Read(data)
+		if err != nil {
+			if err == io.EOF {
+				break
+			}
+			glog.Error(err)
+			return "", err
+		}
+	}
+	err := reader.Close()
+	if err != nil {
+		return "", err
+	}
+
+	return hex.EncodeToString(md5reader.MD5()), nil
+}

pkg/utils/readerutils/MD5Reader.go

+package readerutils
+
+import (
+	"crypto/md5"
+	"hash"
+	"io"
+)
+
+type MD5Reader struct {
+	md5  hash.Hash
+	body io.Reader
+}
+
+func (reader *MD5Reader) Read(b []byte) (int, error) {
+	n, err := reader.body.Read(b)
+	if err != nil {
+		return n, err
+	}
+	return reader.md5.Write(b[:n])
+}
+
+func (reader *MD5Reader) MD5() []byte {
+	return reader.md5.Sum(nil)
+}
+
+func NewMD5Reader(reader io.Reader) *MD5Reader {
+	return &MD5Reader{
+		md5:  md5.New(),
+		body: reader,
+	}
+}

tools/cmd/crd-doc-gen/main.go

@@ -17,6 +17,8 @@ import (
 	"k8s.io/apimachinery/pkg/runtime/serializer"
 	urlruntime "k8s.io/apimachinery/pkg/util/runtime"
 	"k8s.io/kube-openapi/pkg/common"
+	devopsinstall "kubesphere.io/kubesphere/pkg/apis/devops/crdinstall"
+	devopsv1alpha1 "kubesphere.io/kubesphere/pkg/apis/devops/v1alpha1"
 	networkinstall "kubesphere.io/kubesphere/pkg/apis/network/crdinstall"
 	networkv1alpha1 "kubesphere.io/kubesphere/pkg/apis/network/v1alpha1"
 	servicemeshinstall "kubesphere.io/kubesphere/pkg/apis/servicemesh/crdinstall"
@@ -45,6 +47,7 @@ func main() {
 	servicemeshinstall.Install(Scheme)
 	tenantinstall.Install(Scheme)
 	networkinstall.Install(Scheme)
+	devopsinstall.Install(Scheme)
 
 	mapper := meta.NewDefaultRESTMapper(nil)
 
@@ -71,6 +74,9 @@ func main() {
 	mapper.AddSpecific(s2iv1alpha1.SchemeGroupVersion.WithKind(s2iv1alpha1.ResourceKindS2iRun),
 		s2iv1alpha1.SchemeGroupVersion.WithResource(s2iv1alpha1.ResourcePluralS2iRun),
 		s2iv1alpha1.SchemeGroupVersion.WithResource(s2iv1alpha1.ResourceSingularS2iRun), meta.RESTScopeRoot)
+	mapper.AddSpecific(devopsv1alpha1.SchemeGroupVersion.WithKind(devopsv1alpha1.ResourceKindS2iBinary),
+		devopsv1alpha1.SchemeGroupVersion.WithResource(devopsv1alpha1.ResourceSingularServicePolicy),
+		devopsv1alpha1.SchemeGroupVersion.WithResource(devopsv1alpha1.ResourcePluralServicePolicy), meta.RESTScopeRoot)
 
 	mapper.AddSpecific(networkv1alpha1.SchemeGroupVersion.WithKind(networkv1alpha1.ResourceKindWorkspaceNetworkPolicy),
 		networkv1alpha1.SchemeGroupVersion.WithResource(networkv1alpha1.ResourcePluralWorkspaceNetworkPolicy),
@@ -97,6 +103,7 @@ func main() {
 			tenantv1alpha1.GetOpenAPIDefinitions,
 			s2iv1alpha1.GetOpenAPIDefinitions,
 			networkv1alpha1.GetOpenAPIDefinitions,
+			devopsv1alpha1.GetOpenAPIDefinitions,
 		},
 		Resources: []schema.GroupVersionResource{
 			//TODO（runzexia） At present, the document generation requires the openapi structure of the go language,
@@ -109,6 +116,7 @@ func main() {
 			s2iv1alpha1.SchemeGroupVersion.WithResource(s2iv1alpha1.ResourcePluralS2iBuilderTemplate),
 			s2iv1alpha1.SchemeGroupVersion.WithResource(s2iv1alpha1.ResourcePluralS2iBuilder),
 			networkv1alpha1.SchemeGroupVersion.WithResource(networkv1alpha1.ResourcePluralWorkspaceNetworkPolicy),
+			devopsv1alpha1.SchemeGroupVersion.WithResource(devopsv1alpha1.ResourceKindS2iBinary),
 		},
 		Mapper: mapper,
 	})

tools/cmd/doc-gen/main.go

@@ -43,7 +43,7 @@ import (
 var output string
 
 func init() {
-	flag.StringVar(&output, "output", "./api.json", "--output=./api.json")
+	flag.StringVar(&output, "output", "./api/ks-openapi-spec/swagger.json", "--output=./api.json")
 }
 
 func main() {

vendor/code.cloudfoundry.org/bytefmt/LICENSE

+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
\ No newline at end of file

vendor/code.cloudfoundry.org/bytefmt/NOTICE

+Copyright (c) 2015-Present CloudFoundry.org Foundation, Inc. All Rights Reserved.
+
+This project contains software that is Copyright (c) 2013-2015 Pivotal Software, Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+This project may include a number of subcomponents with separate
+copyright notices and license terms. Your use of these subcomponents
+is subject to the terms and conditions of each subcomponent's license,
+as noted in the LICENSE file.

vendor/code.cloudfoundry.org/bytefmt/README.md

+bytefmt
+=======
+
+**Note**: This repository should be imported as `code.cloudfoundry.org/bytefmt`.
+
+Human-readable byte formatter.
+
+Example:
+
+```go
+bytefmt.ByteSize(100.5*bytefmt.MEGABYTE) // returns "100.5M"
+bytefmt.ByteSize(uint64(1024)) // returns "1K"
+```
+
+For documentation, please see http://godoc.org/code.cloudfoundry.org/bytefmt

vendor/code.cloudfoundry.org/bytefmt/bytes.go

+// Package bytefmt contains helper methods and constants for converting to and from a human-readable byte format.
+//
+//	bytefmt.ByteSize(100.5*bytefmt.MEGABYTE) // "100.5M"
+//	bytefmt.ByteSize(uint64(1024)) // "1K"
+//
+package bytefmt
+
+import (
+	"errors"
+	"strconv"
+	"strings"
+	"unicode"
+)
+
+const (
+	BYTE = 1 << (10 * iota)
+	KILOBYTE
+	MEGABYTE
+	GIGABYTE
+	TERABYTE
+	PETABYTE
+	EXABYTE
+)
+
+var invalidByteQuantityError = errors.New("byte quantity must be a positive integer with a unit of measurement like M, MB, MiB, G, GiB, or GB")
+
+// ByteSize returns a human-readable byte string of the form 10M, 12.5K, and so forth.  The following units are available:
+//	E: Exabyte
+//	P: Petabyte
+//	T: Terabyte
+//	G: Gigabyte
+//	M: Megabyte
+//	K: Kilobyte
+//	B: Byte
+// The unit that results in the smallest number greater than or equal to 1 is always chosen.
+func ByteSize(bytes uint64) string {
+	unit := ""
+	value := float64(bytes)
+
+	switch {
+	case bytes >= EXABYTE:
+		unit = "E"
+		value = value / EXABYTE
+	case bytes >= PETABYTE:
+		unit = "P"
+		value = value / PETABYTE
+	case bytes >= TERABYTE:
+		unit = "T"
+		value = value / TERABYTE
+	case bytes >= GIGABYTE:
+		unit = "G"
+		value = value / GIGABYTE
+	case bytes >= MEGABYTE:
+		unit = "M"
+		value = value / MEGABYTE
+	case bytes >= KILOBYTE:
+		unit = "K"
+		value = value / KILOBYTE
+	case bytes >= BYTE:
+		unit = "B"
+	case bytes == 0:
+		return "0"
+	}
+
+	result := strconv.FormatFloat(value, 'f', 1, 64)
+	result = strings.TrimSuffix(result, ".0")
+	return result + unit
+}
+
+// ToMegabytes parses a string formatted by ByteSize as megabytes.
+func ToMegabytes(s string) (uint64, error) {
+	bytes, err := ToBytes(s)
+	if err != nil {
+		return 0, err
+	}
+
+	return bytes / MEGABYTE, nil
+}
+
+// ToBytes parses a string formatted by ByteSize as bytes. Note binary-prefixed and SI prefixed units both mean a base-2 units
+// KB = K = KiB	= 1024
+// MB = M = MiB = 1024 * K
+// GB = G = GiB = 1024 * M
+// TB = T = TiB = 1024 * G
+// PB = P = PiB = 1024 * T
+// EB = E = EiB = 1024 * P
+func ToBytes(s string) (uint64, error) {
+	s = strings.TrimSpace(s)
+	s = strings.ToUpper(s)
+
+	i := strings.IndexFunc(s, unicode.IsLetter)
+
+	if i == -1 {
+		return 0, invalidByteQuantityError
+	}
+
+	bytesString, multiple := s[:i], s[i:]
+	bytes, err := strconv.ParseFloat(bytesString, 64)
+	if err != nil || bytes <= 0 {
+		return 0, invalidByteQuantityError
+	}
+
+	switch multiple {
+	case "E", "EB", "EIB":
+		return uint64(bytes * EXABYTE), nil
+	case "P", "PB", "PIB":
+		return uint64(bytes * PETABYTE), nil
+	case "T", "TB", "TIB":
+		return uint64(bytes * TERABYTE), nil
+	case "G", "GB", "GIB":
+		return uint64(bytes * GIGABYTE), nil
+	case "M", "MB", "MIB":
+		return uint64(bytes * MEGABYTE), nil
+	case "K", "KB", "KIB":
+		return uint64(bytes * KILOBYTE), nil
+	case "B":
+		return uint64(bytes), nil
+	default:
+		return 0, invalidByteQuantityError
+	}
+}

vendor/code.cloudfoundry.org/bytefmt/package.go

+package bytefmt // import "code.cloudfoundry.org/bytefmt"

vendor/github.com/aws/aws-sdk-go/LICENSE.txt

+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

vendor/github.com/aws/aws-sdk-go/NOTICE.txt

+AWS SDK for Go
+Copyright 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+Copyright 2014-2015 Stripe, Inc.

vendor/github.com/aws/aws-sdk-go/aws/awserr/error.go

+// Package awserr represents API error interface accessors for the SDK.
+package awserr
+
+// An Error wraps lower level errors with code, message and an original error.
+// The underlying concrete error type may also satisfy other interfaces which
+// can be to used to obtain more specific information about the error.
+//
+// Calling Error() or String() will always include the full information about
+// an error based on its underlying type.
+//
+// Example:
+//
+//     output, err := s3manage.Upload(svc, input, opts)
+//     if err != nil {
+//         if awsErr, ok := err.(awserr.Error); ok {
+//             // Get error details
+//             log.Println("Error:", awsErr.Code(), awsErr.Message())
+//
+//             // Prints out full error message, including original error if there was one.
+//             log.Println("Error:", awsErr.Error())
+//
+//             // Get original error
+//             if origErr := awsErr.OrigErr(); origErr != nil {
+//                 // operate on original error.
+//             }
+//         } else {
+//             fmt.Println(err.Error())
+//         }
+//     }
+//
+type Error interface {
+	// Satisfy the generic error interface.
+	error
+
+	// Returns the short phrase depicting the classification of the error.
+	Code() string
+
+	// Returns the error details message.
+	Message() string
+
+	// Returns the original error if one was set.  Nil is returned if not set.
+	OrigErr() error
+}
+
+// BatchError is a batch of errors which also wraps lower level errors with
+// code, message, and original errors. Calling Error() will include all errors
+// that occurred in the batch.
+//
+// Deprecated: Replaced with BatchedErrors. Only defined for backwards
+// compatibility.
+type BatchError interface {
+	// Satisfy the generic error interface.
+	error
+
+	// Returns the short phrase depicting the classification of the error.
+	Code() string
+
+	// Returns the error details message.
+	Message() string
+
+	// Returns the original error if one was set.  Nil is returned if not set.
+	OrigErrs() []error
+}
+
+// BatchedErrors is a batch of errors which also wraps lower level errors with
+// code, message, and original errors. Calling Error() will include all errors
+// that occurred in the batch.
+//
+// Replaces BatchError
+type BatchedErrors interface {
+	// Satisfy the base Error interface.
+	Error
+
+	// Returns the original error if one was set.  Nil is returned if not set.
+	OrigErrs() []error
+}
+
+// New returns an Error object described by the code, message, and origErr.
+//
+// If origErr satisfies the Error interface it will not be wrapped within a new
+// Error object and will instead be returned.
+func New(code, message string, origErr error) Error {
+	var errs []error
+	if origErr != nil {
+		errs = append(errs, origErr)
+	}
+	return newBaseError(code, message, errs)
+}
+
+// NewBatchError returns an BatchedErrors with a collection of errors as an
+// array of errors.
+func NewBatchError(code, message string, errs []error) BatchedErrors {
+	return newBaseError(code, message, errs)
+}
+
+// A RequestFailure is an interface to extract request failure information from
+// an Error such as the request ID of the failed request returned by a service.
+// RequestFailures may not always have a requestID value if the request failed
+// prior to reaching the service such as a connection error.
+//
+// Example:
+//
+//     output, err := s3manage.Upload(svc, input, opts)
+//     if err != nil {
+//         if reqerr, ok := err.(RequestFailure); ok {
+//             log.Println("Request failed", reqerr.Code(), reqerr.Message(), reqerr.RequestID())
+//         } else {
+//             log.Println("Error:", err.Error())
+//         }
+//     }
+//
+// Combined with awserr.Error:
+//
+//    output, err := s3manage.Upload(svc, input, opts)
+//    if err != nil {
+//        if awsErr, ok := err.(awserr.Error); ok {
+//            // Generic AWS Error with Code, Message, and original error (if any)
+//            fmt.Println(awsErr.Code(), awsErr.Message(), awsErr.OrigErr())
+//
+//            if reqErr, ok := err.(awserr.RequestFailure); ok {
+//                // A service error occurred
+//                fmt.Println(reqErr.StatusCode(), reqErr.RequestID())
+//            }
+//        } else {
+//            fmt.Println(err.Error())
+//        }
+//    }
+//
+type RequestFailure interface {
+	Error
+
+	// The status code of the HTTP response.
+	StatusCode() int
+
+	// The request ID returned by the service for a request failure. This will
+	// be empty if no request ID is available such as the request failed due
+	// to a connection error.
+	RequestID() string
+}
+
+// NewRequestFailure returns a wrapped error with additional information for
+// request status code, and service requestID.
+//
+// Should be used to wrap all request which involve service requests. Even if
+// the request failed without a service response, but had an HTTP status code
+// that may be meaningful.
+func NewRequestFailure(err Error, statusCode int, reqID string) RequestFailure {
+	return newRequestError(err, statusCode, reqID)
+}
+
+// UnmarshalError provides the interface for the SDK failing to unmarshal data.
+type UnmarshalError interface {
+	awsError
+	Bytes() []byte
+}
+
+// NewUnmarshalError returns an initialized UnmarshalError error wrapper adding
+// the bytes that fail to unmarshal to the error.
+func NewUnmarshalError(err error, msg string, bytes []byte) UnmarshalError {
+	return &unmarshalError{
+		awsError: New("UnmarshalError", msg, err),
+		bytes:    bytes,
+	}
+}

vendor/github.com/aws/aws-sdk-go/aws/awserr/types.go

+package awserr
+
+import (
+	"encoding/hex"
+	"fmt"
+)
+
+// SprintError returns a string of the formatted error code.
+//
+// Both extra and origErr are optional.  If they are included their lines
+// will be added, but if they are not included their lines will be ignored.
+func SprintError(code, message, extra string, origErr error) string {
+	msg := fmt.Sprintf("%s: %s", code, message)
+	if extra != "" {
+		msg = fmt.Sprintf("%s\n\t%s", msg, extra)
+	}
+	if origErr != nil {
+		msg = fmt.Sprintf("%s\ncaused by: %s", msg, origErr.Error())
+	}
+	return msg
+}
+
+// A baseError wraps the code and message which defines an error. It also
+// can be used to wrap an original error object.
+//
+// Should be used as the root for errors satisfying the awserr.Error. Also
+// for any error which does not fit into a specific error wrapper type.
+type baseError struct {
+	// Classification of error
+	code string
+
+	// Detailed information about error
+	message string
+
+	// Optional original error this error is based off of. Allows building
+	// chained errors.
+	errs []error
+}
+
+// newBaseError returns an error object for the code, message, and errors.
+//
+// code is a short no whitespace phrase depicting the classification of
+// the error that is being created.
+//
+// message is the free flow string containing detailed information about the
+// error.
+//
+// origErrs is the error objects which will be nested under the new errors to
+// be returned.
+func newBaseError(code, message string, origErrs []error) *baseError {
+	b := &baseError{
+		code:    code,
+		message: message,
+		errs:    origErrs,
+	}
+
+	return b
+}
+
+// Error returns the string representation of the error.
+//
+// See ErrorWithExtra for formatting.
+//
+// Satisfies the error interface.
+func (b baseError) Error() string {
+	size := len(b.errs)
+	if size > 0 {
+		return SprintError(b.code, b.message, "", errorList(b.errs))
+	}
+
+	return SprintError(b.code, b.message, "", nil)
+}
+
+// String returns the string representation of the error.
+// Alias for Error to satisfy the stringer interface.
+func (b baseError) String() string {
+	return b.Error()
+}
+
+// Code returns the short phrase depicting the classification of the error.
+func (b baseError) Code() string {
+	return b.code
+}
+
+// Message returns the error details message.
+func (b baseError) Message() string {
+	return b.message
+}
+
+// OrigErr returns the original error if one was set. Nil is returned if no
+// error was set. This only returns the first element in the list. If the full
+// list is needed, use BatchedErrors.
+func (b baseError) OrigErr() error {
+	switch len(b.errs) {
+	case 0:
+		return nil
+	case 1:
+		return b.errs[0]
+	default:
+		if err, ok := b.errs[0].(Error); ok {
+			return NewBatchError(err.Code(), err.Message(), b.errs[1:])
+		}
+		return NewBatchError("BatchedErrors",
+			"multiple errors occurred", b.errs)
+	}
+}
+
+// OrigErrs returns the original errors if one was set. An empty slice is
+// returned if no error was set.
+func (b baseError) OrigErrs() []error {
+	return b.errs
+}
+
+// So that the Error interface type can be included as an anonymous field
+// in the requestError struct and not conflict with the error.Error() method.
+type awsError Error
+
+// A requestError wraps a request or service error.
+//
+// Composed of baseError for code, message, and original error.
+type requestError struct {
+	awsError
+	statusCode int
+	requestID  string
+	bytes      []byte
+}
+
+// newRequestError returns a wrapped error with additional information for
+// request status code, and service requestID.
+//
+// Should be used to wrap all request which involve service requests. Even if
+// the request failed without a service response, but had an HTTP status code
+// that may be meaningful.
+//
+// Also wraps original errors via the baseError.
+func newRequestError(err Error, statusCode int, requestID string) *requestError {
+	return &requestError{
+		awsError:   err,
+		statusCode: statusCode,
+		requestID:  requestID,
+	}
+}
+
+// Error returns the string representation of the error.
+// Satisfies the error interface.
+func (r requestError) Error() string {
+	extra := fmt.Sprintf("status code: %d, request id: %s",
+		r.statusCode, r.requestID)
+	return SprintError(r.Code(), r.Message(), extra, r.OrigErr())
+}
+
+// String returns the string representation of the error.
+// Alias for Error to satisfy the stringer interface.
+func (r requestError) String() string {
+	return r.Error()
+}
+
+// StatusCode returns the wrapped status code for the error
+func (r requestError) StatusCode() int {
+	return r.statusCode
+}
+
+// RequestID returns the wrapped requestID
+func (r requestError) RequestID() string {
+	return r.requestID
+}
+
+// OrigErrs returns the original errors if one was set. An empty slice is
+// returned if no error was set.
+func (r requestError) OrigErrs() []error {
+	if b, ok := r.awsError.(BatchedErrors); ok {
+		return b.OrigErrs()
+	}
+	return []error{r.OrigErr()}
+}
+
+type unmarshalError struct {
+	awsError
+	bytes []byte
+}
+
+// Error returns the string representation of the error.
+// Satisfies the error interface.
+func (e unmarshalError) Error() string {
+	extra := hex.Dump(e.bytes)
+	return SprintError(e.Code(), e.Message(), extra, e.OrigErr())
+}
+
+// String returns the string representation of the error.
+// Alias for Error to satisfy the stringer interface.
+func (e unmarshalError) String() string {
+	return e.Error()
+}
+
+// Bytes returns the bytes that failed to unmarshal.
+func (e unmarshalError) Bytes() []byte {
+	return e.bytes
+}
+
+// An error list that satisfies the golang interface
+type errorList []error
+
+// Error returns the string representation of the error.
+//
+// Satisfies the error interface.
+func (e errorList) Error() string {
+	msg := ""
+	// How do we want to handle the array size being zero
+	if size := len(e); size > 0 {
+		for i := 0; i < size; i++ {
+			msg += e[i].Error()
+			// We check the next index to see if it is within the slice.
+			// If it is, then we append a newline. We do this, because unit tests
+			// could be broken with the additional '\n'
+			if i+1 < size {
+				msg += "\n"
+			}
+		}
+	}
+	return msg
+}

vendor/github.com/aws/aws-sdk-go/aws/awsutil/copy.go

+package awsutil
+
+import (
+	"io"
+	"reflect"
+	"time"
+)
+
+// Copy deeply copies a src structure to dst. Useful for copying request and
+// response structures.
+//
+// Can copy between structs of different type, but will only copy fields which
+// are assignable, and exist in both structs. Fields which are not assignable,
+// or do not exist in both structs are ignored.
+func Copy(dst, src interface{}) {
+	dstval := reflect.ValueOf(dst)
+	if !dstval.IsValid() {
+		panic("Copy dst cannot be nil")
+	}
+
+	rcopy(dstval, reflect.ValueOf(src), true)
+}
+
+// CopyOf returns a copy of src while also allocating the memory for dst.
+// src must be a pointer type or this operation will fail.
+func CopyOf(src interface{}) (dst interface{}) {
+	dsti := reflect.New(reflect.TypeOf(src).Elem())
+	dst = dsti.Interface()
+	rcopy(dsti, reflect.ValueOf(src), true)
+	return
+}
+
+// rcopy performs a recursive copy of values from the source to destination.
+//
+// root is used to skip certain aspects of the copy which are not valid
+// for the root node of a object.
+func rcopy(dst, src reflect.Value, root bool) {
+	if !src.IsValid() {
+		return
+	}
+
+	switch src.Kind() {
+	case reflect.Ptr:
+		if _, ok := src.Interface().(io.Reader); ok {
+			if dst.Kind() == reflect.Ptr && dst.Elem().CanSet() {
+				dst.Elem().Set(src)
+			} else if dst.CanSet() {
+				dst.Set(src)
+			}
+		} else {
+			e := src.Type().Elem()
+			if dst.CanSet() && !src.IsNil() {
+				if _, ok := src.Interface().(*time.Time); !ok {
+					dst.Set(reflect.New(e))
+				} else {
+					tempValue := reflect.New(e)
+					tempValue.Elem().Set(src.Elem())
+					// Sets time.Time's unexported values
+					dst.Set(tempValue)
+				}
+			}
+			if src.Elem().IsValid() {
+				// Keep the current root state since the depth hasn't changed
+				rcopy(dst.Elem(), src.Elem(), root)
+			}
+		}
+	case reflect.Struct:
+		t := dst.Type()
+		for i := 0; i < t.NumField(); i++ {
+			name := t.Field(i).Name
+			srcVal := src.FieldByName(name)
+			dstVal := dst.FieldByName(name)
+			if srcVal.IsValid() && dstVal.CanSet() {
+				rcopy(dstVal, srcVal, false)
+			}
+		}
+	case reflect.Slice:
+		if src.IsNil() {
+			break
+		}
+
+		s := reflect.MakeSlice(src.Type(), src.Len(), src.Cap())
+		dst.Set(s)
+		for i := 0; i < src.Len(); i++ {
+			rcopy(dst.Index(i), src.Index(i), false)
+		}
+	case reflect.Map:
+		if src.IsNil() {
+			break
+		}
+
+		s := reflect.MakeMap(src.Type())
+		dst.Set(s)
+		for _, k := range src.MapKeys() {
+			v := src.MapIndex(k)
+			v2 := reflect.New(v.Type()).Elem()
+			rcopy(v2, v, false)
+			dst.SetMapIndex(k, v2)
+		}
+	default:
+		// Assign the value if possible. If its not assignable, the value would
+		// need to be converted and the impact of that may be unexpected, or is
+		// not compatible with the dst type.
+		if src.Type().AssignableTo(dst.Type()) {
+			dst.Set(src)
+		}
+	}
+}

vendor/github.com/aws/aws-sdk-go/aws/awsutil/equal.go

+package awsutil
+
+import (
+	"reflect"
+)
+
+// DeepEqual returns if the two values are deeply equal like reflect.DeepEqual.
+// In addition to this, this method will also dereference the input values if
+// possible so the DeepEqual performed will not fail if one parameter is a
+// pointer and the other is not.
+//
+// DeepEqual will not perform indirection of nested values of the input parameters.
+func DeepEqual(a, b interface{}) bool {
+	ra := reflect.Indirect(reflect.ValueOf(a))
+	rb := reflect.Indirect(reflect.ValueOf(b))
+
+	if raValid, rbValid := ra.IsValid(), rb.IsValid(); !raValid && !rbValid {
+		// If the elements are both nil, and of the same type they are equal
+		// If they are of different types they are not equal
+		return reflect.TypeOf(a) == reflect.TypeOf(b)
+	} else if raValid != rbValid {
+		// Both values must be valid to be equal
+		return false
+	}
+
+	return reflect.DeepEqual(ra.Interface(), rb.Interface())
+}

vendor/github.com/aws/aws-sdk-go/aws/awsutil/path_value.go

+package awsutil
+
+import (
+	"reflect"
+	"regexp"
+	"strconv"
+	"strings"
+
+	"github.com/jmespath/go-jmespath"
+)
+
+var indexRe = regexp.MustCompile(`(.+)\[(-?\d+)?\]$`)
+
+// rValuesAtPath returns a slice of values found in value v. The values
+// in v are explored recursively so all nested values are collected.
+func rValuesAtPath(v interface{}, path string, createPath, caseSensitive, nilTerm bool) []reflect.Value {
+	pathparts := strings.Split(path, "||")
+	if len(pathparts) > 1 {
+		for _, pathpart := range pathparts {
+			vals := rValuesAtPath(v, pathpart, createPath, caseSensitive, nilTerm)
+			if len(vals) > 0 {
+				return vals
+			}
+		}
+		return nil
+	}
+
+	values := []reflect.Value{reflect.Indirect(reflect.ValueOf(v))}
+	components := strings.Split(path, ".")
+	for len(values) > 0 && len(components) > 0 {
+		var index *int64
+		var indexStar bool
+		c := strings.TrimSpace(components[0])
+		if c == "" { // no actual component, illegal syntax
+			return nil
+		} else if caseSensitive && c != "*" && strings.ToLower(c[0:1]) == c[0:1] {
+			// TODO normalize case for user
+			return nil // don't support unexported fields
+		}
+
+		// parse this component
+		if m := indexRe.FindStringSubmatch(c); m != nil {
+			c = m[1]
+			if m[2] == "" {
+				index = nil
+				indexStar = true
+			} else {
+				i, _ := strconv.ParseInt(m[2], 10, 32)
+				index = &i
+				indexStar = false
+			}
+		}
+
+		nextvals := []reflect.Value{}
+		for _, value := range values {
+			// pull component name out of struct member
+			if value.Kind() != reflect.Struct {
+				continue
+			}
+
+			if c == "*" { // pull all members
+				for i := 0; i < value.NumField(); i++ {
+					if f := reflect.Indirect(value.Field(i)); f.IsValid() {
+						nextvals = append(nextvals, f)
+					}
+				}
+				continue
+			}
+
+			value = value.FieldByNameFunc(func(name string) bool {
+				if c == name {
+					return true
+				} else if !caseSensitive && strings.ToLower(name) == strings.ToLower(c) {
+					return true
+				}
+				return false
+			})
+
+			if nilTerm && value.Kind() == reflect.Ptr && len(components[1:]) == 0 {
+				if !value.IsNil() {
+					value.Set(reflect.Zero(value.Type()))
+				}
+				return []reflect.Value{value}
+			}
+
+			if createPath && value.Kind() == reflect.Ptr && value.IsNil() {
+				// TODO if the value is the terminus it should not be created
+				// if the value to be set to its position is nil.
+				value.Set(reflect.New(value.Type().Elem()))
+				value = value.Elem()
+			} else {
+				value = reflect.Indirect(value)
+			}
+
+			if value.Kind() == reflect.Slice || value.Kind() == reflect.Map {
+				if !createPath && value.IsNil() {
+					value = reflect.ValueOf(nil)
+				}
+			}
+
+			if value.IsValid() {
+				nextvals = append(nextvals, value)
+			}
+		}
+		values = nextvals
+
+		if indexStar || index != nil {
+			nextvals = []reflect.Value{}
+			for _, valItem := range values {
+				value := reflect.Indirect(valItem)
+				if value.Kind() != reflect.Slice {
+					continue
+				}
+
+				if indexStar { // grab all indices
+					for i := 0; i < value.Len(); i++ {
+						idx := reflect.Indirect(value.Index(i))
+						if idx.IsValid() {
+							nextvals = append(nextvals, idx)
+						}
+					}
+					continue
+				}
+
+				// pull out index
+				i := int(*index)
+				if i >= value.Len() { // check out of bounds
+					if createPath {
+						// TODO resize slice
+					} else {
+						continue
+					}
+				} else if i < 0 { // support negative indexing
+					i = value.Len() + i
+				}
+				value = reflect.Indirect(value.Index(i))
+
+				if value.Kind() == reflect.Slice || value.Kind() == reflect.Map {
+					if !createPath && value.IsNil() {
+						value = reflect.ValueOf(nil)
+					}
+				}
+
+				if value.IsValid() {
+					nextvals = append(nextvals, value)
+				}
+			}
+			values = nextvals
+		}
+
+		components = components[1:]
+	}
+	return values
+}
+
+// ValuesAtPath returns a list of values at the case insensitive lexical
+// path inside of a structure.
+func ValuesAtPath(i interface{}, path string) ([]interface{}, error) {
+	result, err := jmespath.Search(path, i)
+	if err != nil {
+		return nil, err
+	}
+
+	v := reflect.ValueOf(result)
+	if !v.IsValid() || (v.Kind() == reflect.Ptr && v.IsNil()) {
+		return nil, nil
+	}
+	if s, ok := result.([]interface{}); ok {
+		return s, err
+	}
+	if v.Kind() == reflect.Map && v.Len() == 0 {
+		return nil, nil
+	}
+	if v.Kind() == reflect.Slice {
+		out := make([]interface{}, v.Len())
+		for i := 0; i < v.Len(); i++ {
+			out[i] = v.Index(i).Interface()
+		}
+		return out, nil
+	}
+
+	return []interface{}{result}, nil
+}
+
+// SetValueAtPath sets a value at the case insensitive lexical path inside
+// of a structure.
+func SetValueAtPath(i interface{}, path string, v interface{}) {
+	rvals := rValuesAtPath(i, path, true, false, v == nil)
+	for _, rval := range rvals {
+		if rval.Kind() == reflect.Ptr && rval.IsNil() {
+			continue
+		}
+		setValue(rval, v)
+	}
+}
+
+func setValue(dstVal reflect.Value, src interface{}) {
+	if dstVal.Kind() == reflect.Ptr {
+		dstVal = reflect.Indirect(dstVal)
+	}
+	srcVal := reflect.ValueOf(src)
+
+	if !srcVal.IsValid() { // src is literal nil
+		if dstVal.CanAddr() {
+			// Convert to pointer so that pointer's value can be nil'ed
+			//                     dstVal = dstVal.Addr()
+		}
+		dstVal.Set(reflect.Zero(dstVal.Type()))
+
+	} else if srcVal.Kind() == reflect.Ptr {
+		if srcVal.IsNil() {
+			srcVal = reflect.Zero(dstVal.Type())
+		} else {
+			srcVal = reflect.ValueOf(src).Elem()
+		}
+		dstVal.Set(srcVal)
+	} else {
+		dstVal.Set(srcVal)
+	}
+
+}

vendor/github.com/aws/aws-sdk-go/aws/awsutil/prettify.go

+package awsutil
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"reflect"
+	"strings"
+)
+
+// Prettify returns the string representation of a value.
+func Prettify(i interface{}) string {
+	var buf bytes.Buffer
+	prettify(reflect.ValueOf(i), 0, &buf)
+	return buf.String()
+}
+
+// prettify will recursively walk value v to build a textual
+// representation of the value.
+func prettify(v reflect.Value, indent int, buf *bytes.Buffer) {
+	for v.Kind() == reflect.Ptr {
+		v = v.Elem()
+	}
+
+	switch v.Kind() {
+	case reflect.Struct:
+		strtype := v.Type().String()
+		if strtype == "time.Time" {
+			fmt.Fprintf(buf, "%s", v.Interface())
+			break
+		} else if strings.HasPrefix(strtype, "io.") {
+			buf.WriteString("<buffer>")
+			break
+		}
+
+		buf.WriteString("{\n")
+
+		names := []string{}
+		for i := 0; i < v.Type().NumField(); i++ {
+			name := v.Type().Field(i).Name
+			f := v.Field(i)
+			if name[0:1] == strings.ToLower(name[0:1]) {
+				continue // ignore unexported fields
+			}
+			if (f.Kind() == reflect.Ptr || f.Kind() == reflect.Slice || f.Kind() == reflect.Map) && f.IsNil() {
+				continue // ignore unset fields
+			}
+			names = append(names, name)
+		}
+
+		for i, n := range names {
+			val := v.FieldByName(n)
+			buf.WriteString(strings.Repeat(" ", indent+2))
+			buf.WriteString(n + ": ")
+			prettify(val, indent+2, buf)
+
+			if i < len(names)-1 {
+				buf.WriteString(",\n")
+			}
+		}
+
+		buf.WriteString("\n" + strings.Repeat(" ", indent) + "}")
+	case reflect.Slice:
+		strtype := v.Type().String()
+		if strtype == "[]uint8" {
+			fmt.Fprintf(buf, "<binary> len %d", v.Len())
+			break
+		}
+
+		nl, id, id2 := "", "", ""
+		if v.Len() > 3 {
+			nl, id, id2 = "\n", strings.Repeat(" ", indent), strings.Repeat(" ", indent+2)
+		}
+		buf.WriteString("[" + nl)
+		for i := 0; i < v.Len(); i++ {
+			buf.WriteString(id2)
+			prettify(v.Index(i), indent+2, buf)
+
+			if i < v.Len()-1 {
+				buf.WriteString("," + nl)
+			}
+		}
+
+		buf.WriteString(nl + id + "]")
+	case reflect.Map:
+		buf.WriteString("{\n")
+
+		for i, k := range v.MapKeys() {
+			buf.WriteString(strings.Repeat(" ", indent+2))
+			buf.WriteString(k.String() + ": ")
+			prettify(v.MapIndex(k), indent+2, buf)
+
+			if i < v.Len()-1 {
+				buf.WriteString(",\n")
+			}
+		}
+
+		buf.WriteString("\n" + strings.Repeat(" ", indent) + "}")
+	default:
+		if !v.IsValid() {
+			fmt.Fprint(buf, "<invalid value>")
+			return
+		}
+		format := "%v"
+		switch v.Interface().(type) {
+		case string:
+			format = "%q"
+		case io.ReadSeeker, io.Reader:
+			format = "buffer(%p)"
+		}
+		fmt.Fprintf(buf, format, v.Interface())
+	}
+}

vendor/github.com/aws/aws-sdk-go/aws/awsutil/string_value.go

+package awsutil
+
+import (
+	"bytes"
+	"fmt"
+	"reflect"
+	"strings"
+)
+
+// StringValue returns the string representation of a value.
+func StringValue(i interface{}) string {
+	var buf bytes.Buffer
+	stringValue(reflect.ValueOf(i), 0, &buf)
+	return buf.String()
+}
+
+func stringValue(v reflect.Value, indent int, buf *bytes.Buffer) {
+	for v.Kind() == reflect.Ptr {
+		v = v.Elem()
+	}
+
+	switch v.Kind() {
+	case reflect.Struct:
+		buf.WriteString("{\n")
+
+		for i := 0; i < v.Type().NumField(); i++ {
+			ft := v.Type().Field(i)
+			fv := v.Field(i)
+
+			if ft.Name[0:1] == strings.ToLower(ft.Name[0:1]) {
+				continue // ignore unexported fields
+			}
+			if (fv.Kind() == reflect.Ptr || fv.Kind() == reflect.Slice) && fv.IsNil() {
+				continue // ignore unset fields
+			}
+
+			buf.WriteString(strings.Repeat(" ", indent+2))
+			buf.WriteString(ft.Name + ": ")
+
+			if tag := ft.Tag.Get("sensitive"); tag == "true" {
+				buf.WriteString("<sensitive>")
+			} else {
+				stringValue(fv, indent+2, buf)
+			}
+
+			buf.WriteString(",\n")
+		}
+
+		buf.WriteString("\n" + strings.Repeat(" ", indent) + "}")
+	case reflect.Slice:
+		nl, id, id2 := "", "", ""
+		if v.Len() > 3 {
+			nl, id, id2 = "\n", strings.Repeat(" ", indent), strings.Repeat(" ", indent+2)
+		}
+		buf.WriteString("[" + nl)
+		for i := 0; i < v.Len(); i++ {
+			buf.WriteString(id2)
+			stringValue(v.Index(i), indent+2, buf)
+
+			if i < v.Len()-1 {
+				buf.WriteString("," + nl)
+			}
+		}
+
+		buf.WriteString(nl + id + "]")
+	case reflect.Map:
+		buf.WriteString("{\n")
+
+		for i, k := range v.MapKeys() {
+			buf.WriteString(strings.Repeat(" ", indent+2))
+			buf.WriteString(k.String() + ": ")
+			stringValue(v.MapIndex(k), indent+2, buf)
+
+			if i < v.Len()-1 {
+				buf.WriteString(",\n")
+			}
+		}
+
+		buf.WriteString("\n" + strings.Repeat(" ", indent) + "}")
+	default:
+		format := "%v"
+		switch v.Interface().(type) {
+		case string:
+			format = "%q"
+		}
+		fmt.Fprintf(buf, format, v.Interface())
+	}
+}

vendor/github.com/aws/aws-sdk-go/aws/client/client.go

+package client
+
+import (
+	"fmt"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/client/metadata"
+	"github.com/aws/aws-sdk-go/aws/request"
+)
+
+// A Config provides configuration to a service client instance.
+type Config struct {
+	Config        *aws.Config
+	Handlers      request.Handlers
+	Endpoint      string
+	SigningRegion string
+	SigningName   string
+
+	// States that the signing name did not come from a modeled source but
+	// was derived based on other data. Used by service client constructors
+	// to determine if the signin name can be overridden based on metadata the
+	// service has.
+	SigningNameDerived bool
+}
+
+// ConfigProvider provides a generic way for a service client to receive
+// the ClientConfig without circular dependencies.
+type ConfigProvider interface {
+	ClientConfig(serviceName string, cfgs ...*aws.Config) Config
+}
+
+// ConfigNoResolveEndpointProvider same as ConfigProvider except it will not
+// resolve the endpoint automatically. The service client's endpoint must be
+// provided via the aws.Config.Endpoint field.
+type ConfigNoResolveEndpointProvider interface {
+	ClientConfigNoResolveEndpoint(cfgs ...*aws.Config) Config
+}
+
+// A Client implements the base client request and response handling
+// used by all service clients.
+type Client struct {
+	request.Retryer
+	metadata.ClientInfo
+
+	Config   aws.Config
+	Handlers request.Handlers
+}
+
+// New will return a pointer to a new initialized service client.
+func New(cfg aws.Config, info metadata.ClientInfo, handlers request.Handlers, options ...func(*Client)) *Client {
+	svc := &Client{
+		Config:     cfg,
+		ClientInfo: info,
+		Handlers:   handlers.Copy(),
+	}
+
+	switch retryer, ok := cfg.Retryer.(request.Retryer); {
+	case ok:
+		svc.Retryer = retryer
+	case cfg.Retryer != nil && cfg.Logger != nil:
+		s := fmt.Sprintf("WARNING: %T does not implement request.Retryer; using DefaultRetryer instead", cfg.Retryer)
+		cfg.Logger.Log(s)
+		fallthrough
+	default:
+		maxRetries := aws.IntValue(cfg.MaxRetries)
+		if cfg.MaxRetries == nil || maxRetries == aws.UseServiceDefaultRetries {
+			maxRetries = 3
+		}
+		svc.Retryer = DefaultRetryer{NumMaxRetries: maxRetries}
+	}
+
+	svc.AddDebugHandlers()
+
+	for _, option := range options {
+		option(svc)
+	}
+
+	return svc
+}
+
+// NewRequest returns a new Request pointer for the service API
+// operation and parameters.
+func (c *Client) NewRequest(operation *request.Operation, params interface{}, data interface{}) *request.Request {
+	return request.New(c.Config, c.ClientInfo, c.Handlers, c.Retryer, operation, params, data)
+}
+
+// AddDebugHandlers injects debug logging handlers into the service to log request
+// debug information.
+func (c *Client) AddDebugHandlers() {
+	if !c.Config.LogLevel.AtLeast(aws.LogDebug) {
+		return
+	}
+
+	c.Handlers.Send.PushFrontNamed(LogHTTPRequestHandler)
+	c.Handlers.Send.PushBackNamed(LogHTTPResponseHandler)
+}

vendor/github.com/aws/aws-sdk-go/aws/client/default_retryer.go

+package client
+
+import (
+	"strconv"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws/request"
+	"github.com/aws/aws-sdk-go/internal/sdkrand"
+)
+
+// DefaultRetryer implements basic retry logic using exponential backoff for
+// most services. If you want to implement custom retry logic, implement the
+// request.Retryer interface or create a structure type that composes this
+// struct and override the specific methods. For example, to override only
+// the MaxRetries method:
+//
+//		type retryer struct {
+//      client.DefaultRetryer
+//    }
+//
+//    // This implementation always has 100 max retries
+//    func (d retryer) MaxRetries() int { return 100 }
+type DefaultRetryer struct {
+	NumMaxRetries int
+}
+
+// MaxRetries returns the number of maximum returns the service will use to make
+// an individual API request.
+func (d DefaultRetryer) MaxRetries() int {
+	return d.NumMaxRetries
+}
+
+// RetryRules returns the delay duration before retrying this request again
+func (d DefaultRetryer) RetryRules(r *request.Request) time.Duration {
+	// Set the upper limit of delay in retrying at ~five minutes
+	minTime := 30
+	throttle := d.shouldThrottle(r)
+	if throttle {
+		if delay, ok := getRetryDelay(r); ok {
+			return delay
+		}
+
+		minTime = 500
+	}
+
+	retryCount := r.RetryCount
+	if throttle && retryCount > 8 {
+		retryCount = 8
+	} else if retryCount > 13 {
+		retryCount = 13
+	}
+
+	delay := (1 << uint(retryCount)) * (sdkrand.SeededRand.Intn(minTime) + minTime)
+	return time.Duration(delay) * time.Millisecond
+}
+
+// ShouldRetry returns true if the request should be retried.
+func (d DefaultRetryer) ShouldRetry(r *request.Request) bool {
+	// If one of the other handlers already set the retry state
+	// we don't want to override it based on the service's state
+	if r.Retryable != nil {
+		return *r.Retryable
+	}
+
+	if r.HTTPResponse.StatusCode >= 500 && r.HTTPResponse.StatusCode != 501 {
+		return true
+	}
+	return r.IsErrorRetryable() || d.shouldThrottle(r)
+}
+
+// ShouldThrottle returns true if the request should be throttled.
+func (d DefaultRetryer) shouldThrottle(r *request.Request) bool {
+	switch r.HTTPResponse.StatusCode {
+	case 429:
+	case 502:
+	case 503:
+	case 504:
+	default:
+		return r.IsErrorThrottle()
+	}
+
+	return true
+}
+
+// This will look in the Retry-After header, RFC 7231, for how long
+// it will wait before attempting another request
+func getRetryDelay(r *request.Request) (time.Duration, bool) {
+	if !canUseRetryAfterHeader(r) {
+		return 0, false
+	}
+
+	delayStr := r.HTTPResponse.Header.Get("Retry-After")
+	if len(delayStr) == 0 {
+		return 0, false
+	}
+
+	delay, err := strconv.Atoi(delayStr)
+	if err != nil {
+		return 0, false
+	}
+
+	return time.Duration(delay) * time.Second, true
+}
+
+// Will look at the status code to see if the retry header pertains to
+// the status code.
+func canUseRetryAfterHeader(r *request.Request) bool {
+	switch r.HTTPResponse.StatusCode {
+	case 429:
+	case 503:
+	default:
+		return false
+	}
+
+	return true
+}

vendor/github.com/aws/aws-sdk-go/aws/client/logger.go

+package client
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"net/http/httputil"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/request"
+)
+
+const logReqMsg = `DEBUG: Request %s/%s Details:
+---[ REQUEST POST-SIGN ]-----------------------------
+%s
+-----------------------------------------------------`
+
+const logReqErrMsg = `DEBUG ERROR: Request %s/%s:
+---[ REQUEST DUMP ERROR ]-----------------------------
+%s
+------------------------------------------------------`
+
+type logWriter struct {
+	// Logger is what we will use to log the payload of a response.
+	Logger aws.Logger
+	// buf stores the contents of what has been read
+	buf *bytes.Buffer
+}
+
+func (logger *logWriter) Write(b []byte) (int, error) {
+	return logger.buf.Write(b)
+}
+
+type teeReaderCloser struct {
+	// io.Reader will be a tee reader that is used during logging.
+	// This structure will read from a body and write the contents to a logger.
+	io.Reader
+	// Source is used just to close when we are done reading.
+	Source io.ReadCloser
+}
+
+func (reader *teeReaderCloser) Close() error {
+	return reader.Source.Close()
+}
+
+// LogHTTPRequestHandler is a SDK request handler to log the HTTP request sent
+// to a service. Will include the HTTP request body if the LogLevel of the
+// request matches LogDebugWithHTTPBody.
+var LogHTTPRequestHandler = request.NamedHandler{
+	Name: "awssdk.client.LogRequest",
+	Fn:   logRequest,
+}
+
+func logRequest(r *request.Request) {
+	logBody := r.Config.LogLevel.Matches(aws.LogDebugWithHTTPBody)
+	bodySeekable := aws.IsReaderSeekable(r.Body)
+
+	b, err := httputil.DumpRequestOut(r.HTTPRequest, logBody)
+	if err != nil {
+		r.Config.Logger.Log(fmt.Sprintf(logReqErrMsg,
+			r.ClientInfo.ServiceName, r.Operation.Name, err))
+		return
+	}
+
+	if logBody {
+		if !bodySeekable {
+			r.SetReaderBody(aws.ReadSeekCloser(r.HTTPRequest.Body))
+		}
+		// Reset the request body because dumpRequest will re-wrap the
+		// r.HTTPRequest's Body as a NoOpCloser and will not be reset after
+		// read by the HTTP client reader.
+		if err := r.Error; err != nil {
+			r.Config.Logger.Log(fmt.Sprintf(logReqErrMsg,
+				r.ClientInfo.ServiceName, r.Operation.Name, err))
+			return
+		}
+	}
+
+	r.Config.Logger.Log(fmt.Sprintf(logReqMsg,
+		r.ClientInfo.ServiceName, r.Operation.Name, string(b)))
+}
+
+// LogHTTPRequestHeaderHandler is a SDK request handler to log the HTTP request sent
+// to a service. Will only log the HTTP request's headers. The request payload
+// will not be read.
+var LogHTTPRequestHeaderHandler = request.NamedHandler{
+	Name: "awssdk.client.LogRequestHeader",
+	Fn:   logRequestHeader,
+}
+
+func logRequestHeader(r *request.Request) {
+	b, err := httputil.DumpRequestOut(r.HTTPRequest, false)
+	if err != nil {
+		r.Config.Logger.Log(fmt.Sprintf(logReqErrMsg,
+			r.ClientInfo.ServiceName, r.Operation.Name, err))
+		return
+	}
+
+	r.Config.Logger.Log(fmt.Sprintf(logReqMsg,
+		r.ClientInfo.ServiceName, r.Operation.Name, string(b)))
+}
+
+const logRespMsg = `DEBUG: Response %s/%s Details:
+---[ RESPONSE ]--------------------------------------
+%s
+-----------------------------------------------------`
+
+const logRespErrMsg = `DEBUG ERROR: Response %s/%s:
+---[ RESPONSE DUMP ERROR ]-----------------------------
+%s
+-----------------------------------------------------`
+
+// LogHTTPResponseHandler is a SDK request handler to log the HTTP response
+// received from a service. Will include the HTTP response body if the LogLevel
+// of the request matches LogDebugWithHTTPBody.
+var LogHTTPResponseHandler = request.NamedHandler{
+	Name: "awssdk.client.LogResponse",
+	Fn:   logResponse,
+}
+
+func logResponse(r *request.Request) {
+	lw := &logWriter{r.Config.Logger, bytes.NewBuffer(nil)}
+
+	if r.HTTPResponse == nil {
+		lw.Logger.Log(fmt.Sprintf(logRespErrMsg,
+			r.ClientInfo.ServiceName, r.Operation.Name, "request's HTTPResponse is nil"))
+		return
+	}
+
+	logBody := r.Config.LogLevel.Matches(aws.LogDebugWithHTTPBody)
+	if logBody {
+		r.HTTPResponse.Body = &teeReaderCloser{
+			Reader: io.TeeReader(r.HTTPResponse.Body, lw),
+			Source: r.HTTPResponse.Body,
+		}
+	}
+
+	handlerFn := func(req *request.Request) {
+		b, err := httputil.DumpResponse(req.HTTPResponse, false)
+		if err != nil {
+			lw.Logger.Log(fmt.Sprintf(logRespErrMsg,
+				req.ClientInfo.ServiceName, req.Operation.Name, err))
+			return
+		}
+
+		lw.Logger.Log(fmt.Sprintf(logRespMsg,
+			req.ClientInfo.ServiceName, req.Operation.Name, string(b)))
+
+		if logBody {
+			b, err := ioutil.ReadAll(lw.buf)
+			if err != nil {
+				lw.Logger.Log(fmt.Sprintf(logRespErrMsg,
+					req.ClientInfo.ServiceName, req.Operation.Name, err))
+				return
+			}
+
+			lw.Logger.Log(string(b))
+		}
+	}
+
+	const handlerName = "awsdk.client.LogResponse.ResponseBody"
+
+	r.Handlers.Unmarshal.SetBackNamed(request.NamedHandler{
+		Name: handlerName, Fn: handlerFn,
+	})
+	r.Handlers.UnmarshalError.SetBackNamed(request.NamedHandler{
+		Name: handlerName, Fn: handlerFn,
+	})
+}
+
+// LogHTTPResponseHeaderHandler is a SDK request handler to log the HTTP
+// response received from a service. Will only log the HTTP response's headers.
+// The response payload will not be read.
+var LogHTTPResponseHeaderHandler = request.NamedHandler{
+	Name: "awssdk.client.LogResponseHeader",
+	Fn:   logResponseHeader,
+}
+
+func logResponseHeader(r *request.Request) {
+	if r.Config.Logger == nil {
+		return
+	}
+
+	b, err := httputil.DumpResponse(r.HTTPResponse, false)
+	if err != nil {
+		r.Config.Logger.Log(fmt.Sprintf(logRespErrMsg,
+			r.ClientInfo.ServiceName, r.Operation.Name, err))
+		return
+	}
+
+	r.Config.Logger.Log(fmt.Sprintf(logRespMsg,
+		r.ClientInfo.ServiceName, r.Operation.Name, string(b)))
+}

vendor/github.com/aws/aws-sdk-go/aws/client/metadata/client_info.go

+package metadata
+
+// ClientInfo wraps immutable data from the client.Client structure.
+type ClientInfo struct {
+	ServiceName   string
+	ServiceID     string
+	APIVersion    string
+	Endpoint      string
+	SigningName   string
+	SigningRegion string
+	JSONVersion   string
+	TargetPrefix  string
+}

vendor/github.com/aws/aws-sdk-go/aws/context_1_5.go

+// +build !go1.9
+
+package aws
+
+import "time"
+
+// Context is an copy of the Go v1.7 stdlib's context.Context interface.
+// It is represented as a SDK interface to enable you to use the "WithContext"
+// API methods with Go v1.6 and a Context type such as golang.org/x/net/context.
+//
+// See https://golang.org/pkg/context on how to use contexts.
+type Context interface {
+	// Deadline returns the time when work done on behalf of this context
+	// should be canceled. Deadline returns ok==false when no deadline is
+	// set. Successive calls to Deadline return the same results.
+	Deadline() (deadline time.Time, ok bool)
+
+	// Done returns a channel that's closed when work done on behalf of this
+	// context should be canceled. Done may return nil if this context can
+	// never be canceled. Successive calls to Done return the same value.
+	Done() <-chan struct{}
+
+	// Err returns a non-nil error value after Done is closed. Err returns
+	// Canceled if the context was canceled or DeadlineExceeded if the
+	// context's deadline passed. No other values for Err are defined.
+	// After Done is closed, successive calls to Err return the same value.
+	Err() error
+
+	// Value returns the value associated with this context for key, or nil
+	// if no value is associated with key. Successive calls to Value with
+	// the same key returns the same result.
+	//
+	// Use context values only for request-scoped data that transits
+	// processes and API boundaries, not for passing optional parameters to
+	// functions.
+	Value(key interface{}) interface{}
+}

vendor/github.com/aws/aws-sdk-go/aws/context_1_9.go

+// +build go1.9
+
+package aws
+
+import "context"
+
+// Context is an alias of the Go stdlib's context.Context interface.
+// It can be used within the SDK's API operation "WithContext" methods.
+//
+// See https://golang.org/pkg/context on how to use contexts.
+type Context = context.Context

vendor/github.com/aws/aws-sdk-go/aws/context_background_1_5.go

+// +build !go1.7
+
+package aws
+
+import "time"
+
+// An emptyCtx is a copy of the Go 1.7 context.emptyCtx type. This is copied to
+// provide a 1.6 and 1.5 safe version of context that is compatible with Go
+// 1.7's Context.
+//
+// An emptyCtx is never canceled, has no values, and has no deadline. It is not
+// struct{}, since vars of this type must have distinct addresses.
+type emptyCtx int
+
+func (*emptyCtx) Deadline() (deadline time.Time, ok bool) {
+	return
+}
+
+func (*emptyCtx) Done() <-chan struct{} {
+	return nil
+}
+
+func (*emptyCtx) Err() error {
+	return nil
+}
+
+func (*emptyCtx) Value(key interface{}) interface{} {
+	return nil
+}
+
+func (e *emptyCtx) String() string {
+	switch e {
+	case backgroundCtx:
+		return "aws.BackgroundContext"
+	}
+	return "unknown empty Context"
+}
+
+var (
+	backgroundCtx = new(emptyCtx)
+)
+
+// BackgroundContext returns a context that will never be canceled, has no
+// values, and no deadline. This context is used by the SDK to provide
+// backwards compatibility with non-context API operations and functionality.
+//
+// Go 1.6 and before:
+// This context function is equivalent to context.Background in the Go stdlib.
+//
+// Go 1.7 and later:
+// The context returned will be the value returned by context.Background()
+//
+// See https://golang.org/pkg/context for more information on Contexts.
+func BackgroundContext() Context {
+	return backgroundCtx
+}

vendor/github.com/aws/aws-sdk-go/aws/context_background_1_7.go

+// +build go1.7
+
+package aws
+
+import "context"
+
+// BackgroundContext returns a context that will never be canceled, has no
+// values, and no deadline. This context is used by the SDK to provide
+// backwards compatibility with non-context API operations and functionality.
+//
+// Go 1.6 and before:
+// This context function is equivalent to context.Background in the Go stdlib.
+//
+// Go 1.7 and later:
+// The context returned will be the value returned by context.Background()
+//
+// See https://golang.org/pkg/context for more information on Contexts.
+func BackgroundContext() Context {
+	return context.Background()
+}

vendor/github.com/aws/aws-sdk-go/aws/context_sleep.go

+package aws
+
+import (
+	"time"
+)
+
+// SleepWithContext will wait for the timer duration to expire, or the context
+// is canceled. Which ever happens first. If the context is canceled the Context's
+// error will be returned.
+//
+// Expects Context to always return a non-nil error if the Done channel is closed.
+func SleepWithContext(ctx Context, dur time.Duration) error {
+	t := time.NewTimer(dur)
+	defer t.Stop()
+
+	select {
+	case <-t.C:
+		break
+	case <-ctx.Done():
+		return ctx.Err()
+	}
+
+	return nil
+}

vendor/github.com/aws/aws-sdk-go/aws/corehandlers/handlers.go

+package corehandlers
+
+import (
+	"bytes"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"net/url"
+	"regexp"
+	"strconv"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/awserr"
+	"github.com/aws/aws-sdk-go/aws/credentials"
+	"github.com/aws/aws-sdk-go/aws/request"
+)
+
+// Interface for matching types which also have a Len method.
+type lener interface {
+	Len() int
+}
+
+// BuildContentLengthHandler builds the content length of a request based on the body,
+// or will use the HTTPRequest.Header's "Content-Length" if defined. If unable
+// to determine request body length and no "Content-Length" was specified it will panic.
+//
+// The Content-Length will only be added to the request if the length of the body
+// is greater than 0. If the body is empty or the current `Content-Length`
+// header is <= 0, the header will also be stripped.
+var BuildContentLengthHandler = request.NamedHandler{Name: "core.BuildContentLengthHandler", Fn: func(r *request.Request) {
+	var length int64
+
+	if slength := r.HTTPRequest.Header.Get("Content-Length"); slength != "" {
+		length, _ = strconv.ParseInt(slength, 10, 64)
+	} else {
+		if r.Body != nil {
+			var err error
+			length, err = aws.SeekerLen(r.Body)
+			if err != nil {
+				r.Error = awserr.New(request.ErrCodeSerialization, "failed to get request body's length", err)
+				return
+			}
+		}
+	}
+
+	if length > 0 {
+		r.HTTPRequest.ContentLength = length
+		r.HTTPRequest.Header.Set("Content-Length", fmt.Sprintf("%d", length))
+	} else {
+		r.HTTPRequest.ContentLength = 0
+		r.HTTPRequest.Header.Del("Content-Length")
+	}
+}}
+
+var reStatusCode = regexp.MustCompile(`^(\d{3})`)
+
+// ValidateReqSigHandler is a request handler to ensure that the request's
+// signature doesn't expire before it is sent. This can happen when a request
+// is built and signed significantly before it is sent. Or significant delays
+// occur when retrying requests that would cause the signature to expire.
+var ValidateReqSigHandler = request.NamedHandler{
+	Name: "core.ValidateReqSigHandler",
+	Fn: func(r *request.Request) {
+		// Unsigned requests are not signed
+		if r.Config.Credentials == credentials.AnonymousCredentials {
+			return
+		}
+
+		signedTime := r.Time
+		if !r.LastSignedAt.IsZero() {
+			signedTime = r.LastSignedAt
+		}
+
+		// 5 minutes to allow for some clock skew/delays in transmission.
+		// Would be improved with aws/aws-sdk-go#423
+		if signedTime.Add(5 * time.Minute).After(time.Now()) {
+			return
+		}
+
+		fmt.Println("request expired, resigning")
+		r.Sign()
+	},
+}
+
+// SendHandler is a request handler to send service request using HTTP client.
+var SendHandler = request.NamedHandler{
+	Name: "core.SendHandler",
+	Fn: func(r *request.Request) {
+		sender := sendFollowRedirects
+		if r.DisableFollowRedirects {
+			sender = sendWithoutFollowRedirects
+		}
+
+		if request.NoBody == r.HTTPRequest.Body {
+			// Strip off the request body if the NoBody reader was used as a
+			// place holder for a request body. This prevents the SDK from
+			// making requests with a request body when it would be invalid
+			// to do so.
+			//
+			// Use a shallow copy of the http.Request to ensure the race condition
+			// of transport on Body will not trigger
+			reqOrig, reqCopy := r.HTTPRequest, *r.HTTPRequest
+			reqCopy.Body = nil
+			r.HTTPRequest = &reqCopy
+			defer func() {
+				r.HTTPRequest = reqOrig
+			}()
+		}
+
+		var err error
+		r.HTTPResponse, err = sender(r)
+		if err != nil {
+			handleSendError(r, err)
+		}
+	},
+}
+
+func sendFollowRedirects(r *request.Request) (*http.Response, error) {
+	return r.Config.HTTPClient.Do(r.HTTPRequest)
+}
+
+func sendWithoutFollowRedirects(r *request.Request) (*http.Response, error) {
+	transport := r.Config.HTTPClient.Transport
+	if transport == nil {
+		transport = http.DefaultTransport
+	}
+
+	return transport.RoundTrip(r.HTTPRequest)
+}
+
+func handleSendError(r *request.Request, err error) {
+	// Prevent leaking if an HTTPResponse was returned. Clean up
+	// the body.
+	if r.HTTPResponse != nil {
+		r.HTTPResponse.Body.Close()
+	}
+	// Capture the case where url.Error is returned for error processing
+	// response. e.g. 301 without location header comes back as string
+	// error and r.HTTPResponse is nil. Other URL redirect errors will
+	// comeback in a similar method.
+	if e, ok := err.(*url.Error); ok && e.Err != nil {
+		if s := reStatusCode.FindStringSubmatch(e.Err.Error()); s != nil {
+			code, _ := strconv.ParseInt(s[1], 10, 64)
+			r.HTTPResponse = &http.Response{
+				StatusCode: int(code),
+				Status:     http.StatusText(int(code)),
+				Body:       ioutil.NopCloser(bytes.NewReader([]byte{})),
+			}
+			return
+		}
+	}
+	if r.HTTPResponse == nil {
+		// Add a dummy request response object to ensure the HTTPResponse
+		// value is consistent.
+		r.HTTPResponse = &http.Response{
+			StatusCode: int(0),
+			Status:     http.StatusText(int(0)),
+			Body:       ioutil.NopCloser(bytes.NewReader([]byte{})),
+		}
+	}
+	// Catch all other request errors.
+	r.Error = awserr.New("RequestError", "send request failed", err)
+	r.Retryable = aws.Bool(true) // network errors are retryable
+
+	// Override the error with a context canceled error, if that was canceled.
+	ctx := r.Context()
+	select {
+	case <-ctx.Done():
+		r.Error = awserr.New(request.CanceledErrorCode,
+			"request context canceled", ctx.Err())
+		r.Retryable = aws.Bool(false)
+	default:
+	}
+}
+
+// ValidateResponseHandler is a request handler to validate service response.
+var ValidateResponseHandler = request.NamedHandler{Name: "core.ValidateResponseHandler", Fn: func(r *request.Request) {
+	if r.HTTPResponse.StatusCode == 0 || r.HTTPResponse.StatusCode >= 300 {
+		// this may be replaced by an UnmarshalError handler
+		r.Error = awserr.New("UnknownError", "unknown error", nil)
+	}
+}}
+
+// AfterRetryHandler performs final checks to determine if the request should
+// be retried and how long to delay.
+var AfterRetryHandler = request.NamedHandler{Name: "core.AfterRetryHandler", Fn: func(r *request.Request) {
+	// If one of the other handlers already set the retry state
+	// we don't want to override it based on the service's state
+	if r.Retryable == nil || aws.BoolValue(r.Config.EnforceShouldRetryCheck) {
+		r.Retryable = aws.Bool(r.ShouldRetry(r))
+	}
+
+	if r.WillRetry() {
+		r.RetryDelay = r.RetryRules(r)
+
+		if sleepFn := r.Config.SleepDelay; sleepFn != nil {
+			// Support SleepDelay for backwards compatibility and testing
+			sleepFn(r.RetryDelay)
+		} else if err := aws.SleepWithContext(r.Context(), r.RetryDelay); err != nil {
+			r.Error = awserr.New(request.CanceledErrorCode,
+				"request context canceled", err)
+			r.Retryable = aws.Bool(false)
+			return
+		}
+
+		// when the expired token exception occurs the credentials
+		// need to be expired locally so that the next request to
+		// get credentials will trigger a credentials refresh.
+		if r.IsErrorExpired() {
+			r.Config.Credentials.Expire()
+		}
+
+		r.RetryCount++
+		r.Error = nil
+	}
+}}
+
+// ValidateEndpointHandler is a request handler to validate a request had the
+// appropriate Region and Endpoint set. Will set r.Error if the endpoint or
+// region is not valid.
+var ValidateEndpointHandler = request.NamedHandler{Name: "core.ValidateEndpointHandler", Fn: func(r *request.Request) {
+	if r.ClientInfo.SigningRegion == "" && aws.StringValue(r.Config.Region) == "" {
+		r.Error = aws.ErrMissingRegion
+	} else if r.ClientInfo.Endpoint == "" {
+		r.Error = aws.ErrMissingEndpoint
+	}
+}}

vendor/github.com/aws/aws-sdk-go/aws/corehandlers/param_validator.go

+package corehandlers
+
+import "github.com/aws/aws-sdk-go/aws/request"
+
+// ValidateParametersHandler is a request handler to validate the input parameters.
+// Validating parameters only has meaning if done prior to the request being sent.
+var ValidateParametersHandler = request.NamedHandler{Name: "core.ValidateParametersHandler", Fn: func(r *request.Request) {
+	if !r.ParamsFilled() {
+		return
+	}
+
+	if v, ok := r.Params.(request.Validator); ok {
+		if err := v.Validate(); err != nil {
+			r.Error = err
+		}
+	}
+}}

vendor/github.com/aws/aws-sdk-go/aws/corehandlers/user_agent.go

+package corehandlers
+
+import (
+	"os"
+	"runtime"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/request"
+)
+
+// SDKVersionUserAgentHandler is a request handler for adding the SDK Version
+// to the user agent.
+var SDKVersionUserAgentHandler = request.NamedHandler{
+	Name: "core.SDKVersionUserAgentHandler",
+	Fn: request.MakeAddToUserAgentHandler(aws.SDKName, aws.SDKVersion,
+		runtime.Version(), runtime.GOOS, runtime.GOARCH),
+}
+
+const execEnvVar = `AWS_EXECUTION_ENV`
+const execEnvUAKey = `exec-env`
+
+// AddHostExecEnvUserAgentHander is a request handler appending the SDK's
+// execution environment to the user agent.
+//
+// If the environment variable AWS_EXECUTION_ENV is set, its value will be
+// appended to the user agent string.
+var AddHostExecEnvUserAgentHander = request.NamedHandler{
+	Name: "core.AddHostExecEnvUserAgentHander",
+	Fn: func(r *request.Request) {
+		v := os.Getenv(execEnvVar)
+		if len(v) == 0 {
+			return
+		}
+
+		request.AddToUserAgent(r, execEnvUAKey+"/"+v)
+	},
+}

vendor/github.com/aws/aws-sdk-go/aws/credentials/example.ini

+[default]
+aws_access_key_id = accessKey
+aws_secret_access_key = secret
+aws_session_token = token
+
+[no_token]
+aws_access_key_id = accessKey
+aws_secret_access_key = secret
+
+[with_colon]
+aws_access_key_id: accessKey
+aws_secret_access_key: secret

vendor/github.com/aws/aws-sdk-go/aws/errors.go

+package aws
+
+import "github.com/aws/aws-sdk-go/aws/awserr"
+
+var (
+	// ErrMissingRegion is an error that is returned if region configuration is
+	// not found.
+	ErrMissingRegion = awserr.New("MissingRegion", "could not find region configuration", nil)
+
+	// ErrMissingEndpoint is an error that is returned if an endpoint cannot be
+	// resolved for a service.
+	ErrMissingEndpoint = awserr.New("MissingEndpoint", "'Endpoint' configuration is required for this service", nil)
+)