diff --git a/pkg/minikube/perf/start.go b/pkg/minikube/perf/start.go
index b587e907c592d52c67205aa9974a3dda1dcfb895..3a9c62bf9b99714ddde917ad6ddb6a756841cb13 100644
--- a/pkg/minikube/perf/start.go
+++ b/pkg/minikube/perf/start.go
@@ -36,9 +36,13 @@ const (
 func CompareMinikubeStart(ctx context.Context, out io.Writer, binaries []*Binary) error {
 	drivers := []string{"kvm2", "docker"}
 	for _, d := range drivers {
+		fmt.Printf("**%s Driver**\n", d)
+		if err := downloadArtifacts(ctx, binaries, d); err != nil {
+			fmt.Printf("error downloading artifacts: %v", err)
+			continue
+		}
 		rm, err := collectResults(ctx, binaries, d)
 		if err != nil {
-			fmt.Printf("**%s Driver**\n", d)
 			fmt.Printf("error collecting results for %s driver: %v\n", d, err)
 			continue
 		}
@@ -76,6 +80,18 @@ func average(nums []float64) float64 {
 	return total / float64(len(nums))
 }
 
+func downloadArtifacts(ctx context.Context, binaries []*Binary, driver string) error {
+	for _, b := range binaries {
+		c := exec.CommandContext(ctx, b.path, "start", fmt.Sprintf("--driver=%s", driver), "--download-only")
+		c.Stderr = os.Stderr
+		log.Printf("Running: %v...", c.Args)
+		if err := c.Run(); err != nil {
+			return errors.Wrap(err, "downloading artifacts")
+		}
+	}
+	return nil
+}
+
 // timeMinikubeStart returns the time it takes to execute `minikube start`
 func timeMinikubeStart(ctx context.Context, binary *Binary, driver string) (*result, error) {
 	startCmd := exec.CommandContext(ctx, binary.path, "start", fmt.Sprintf("--driver=%s", driver))