Merge pull request #9503 from priyawadhwa/schedule-stop-unix

Implement schedule stop for unix

cmd/minikube/cmd/stop.go

@@ -18,6 +18,7 @@ package cmd
 
 import (
 	"os"
+	"runtime"
 	"time"
 
 	"github.com/docker/machine/libmachine"
@@ -36,13 +37,15 @@ import (
 	"k8s.io/minikube/pkg/minikube/out"
 	"k8s.io/minikube/pkg/minikube/out/register"
 	"k8s.io/minikube/pkg/minikube/reason"
+	"k8s.io/minikube/pkg/minikube/schedule"
 	"k8s.io/minikube/pkg/minikube/style"
 	"k8s.io/minikube/pkg/util/retry"
 )
 
 var (
-	stopAll    bool
-	keepActive bool
+	stopAll               bool
+	keepActive            bool
+	scheduledStopDuration time.Duration
 )
 
 // stopCmd represents the stop command
@@ -56,6 +59,10 @@ var stopCmd = &cobra.Command{
 func init() {
 	stopCmd.Flags().BoolVar(&stopAll, "all", false, "Set flag to stop all profiles (clusters)")
 	stopCmd.Flags().BoolVar(&keepActive, "keep-context-active", false, "keep the kube-context active after cluster is stopped. Defaults to false.")
+	stopCmd.Flags().DurationVar(&scheduledStopDuration, "schedule", 0*time.Second, "Set flag to stop cluster after a set amount of time (e.g. --schedule=5m)")
+	if err := stopCmd.Flags().MarkHidden("schedule"); err != nil {
+		klog.Info("unable to mark --schedule flag as hidden")
+	}
 	stopCmd.Flags().StringVarP(&outputFormat, "output", "o", "text", "Format to print stdout in. Options include: [text,json]")
 
 	if err := viper.GetViper().BindPFlags(stopCmd.Flags()); err != nil {
@@ -88,6 +95,20 @@ func runStop(cmd *cobra.Command, args []string) {
 		profilesToStop = append(profilesToStop, cname)
 	}
 
+	// Kill any existing scheduled stops
+	schedule.KillExisting(profilesToStop)
+
+	if scheduledStopDuration != 0 {
+		if runtime.GOOS == "windows" {
+			exit.Message(reason.Usage, "the --schedule flag is currently not supported on windows")
+		}
+		if err := schedule.Daemonize(profilesToStop, scheduledStopDuration); err != nil {
+			exit.Message(reason.DaemonizeError, "unable to daemonize: {{.err}}", out.V{"err": err.Error()})
+		}
+		klog.Infof("sleeping %s before completing stop...", scheduledStopDuration.String())
+		time.Sleep(scheduledStopDuration)
+	}
+
 	stoppedNodes := 0
 	for _, profile := range profilesToStop {
 		stoppedNodes = stopProfile(profile)

go.mod

@@ -8,6 +8,7 @@ require (
 	github.com/Microsoft/go-winio v0.4.15-0.20190919025122-fc70bd9a86b5 // indirect
 	github.com/Parallels/docker-machine-parallels/v2 v2.0.1
 	github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d // indirect
+	github.com/VividCortex/godaemon v0.0.0-20201030160542-15e3f4925a21
 	github.com/blang/semver v3.5.0+incompatible
 	github.com/c4milo/gotoolkit v0.0.0-20170318115440-bcc06269efa9 // indirect
 	github.com/cenkalti/backoff v2.2.1+incompatible

go.sum

@@ -118,6 +118,8 @@ github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d h1:G0m3OIz70MZUW
 github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg=
 github.com/VividCortex/ewma v1.1.1 h1:MnEK4VOv6n0RSY4vtRe3h11qjxL3+t0B8yOL8iMXdcM=
 github.com/VividCortex/ewma v1.1.1/go.mod h1:2Tkkvm3sRDVXaiyucHiACn4cqf7DpdyLvmxzcbUokwA=
+github.com/VividCortex/godaemon v0.0.0-20201030160542-15e3f4925a21 h1:Pgxfz/g+XyfRjYqRjKUFpDh5IciFncmA/Uio6AU/z9g=
+github.com/VividCortex/godaemon v0.0.0-20201030160542-15e3f4925a21/go.mod h1:Y8CJ3IwPIAkMhv/rRUWIlczaeqd9ty9yrl+nc2AbaL4=
 github.com/afbjorklund/go-containerregistry v0.0.0-20200902152226-fbad78ec2813 h1:0tskN1ipU/BBrpoEIy0rdZS9jf5+wdP6IMRak8Iu/YE=
 github.com/afbjorklund/go-containerregistry v0.0.0-20200902152226-fbad78ec2813/go.mod h1:npTSyywOeILcgWqd+rvtzGWflIPPcBQhYoOONaY4ltM=
 github.com/afbjorklund/go-getter v1.4.1-0.20201020145846-c0da14b4bffe h1:TdcuDqk4ArmYI8cbeeL/RM5BPciDOaWpGZoPoT3OziQ=

pkg/minikube/config/types.go

@@ -71,6 +71,7 @@ type ClusterConfig struct {
 	Addons                  map[string]bool
 	VerifyComponents        map[string]bool // map of components to verify and wait for after start.
 	StartHostTimeout        time.Duration
+	ScheduledStop           *ScheduledStopConfig
 	ExposedPorts            []string // Only used by the docker and podman driver
 }
 
@@ -138,3 +139,10 @@ type VersionedExtraOption struct {
 	// flag is applied to
 	GreaterThanOrEqual semver.Version
 }
+
+// ScheduledStopConfig contains information around scheduled stop
+// not yet used, will be used to show status of scheduled stop
+type ScheduledStopConfig struct {
+	InitiationTime int64
+	Duration       time.Duration
+}

pkg/minikube/localpath/localpath.go

@@ -19,6 +19,7 @@ package localpath
 import (
 	"os"
 	"os/exec"
+	"path"
 	"path/filepath"
 	"runtime"
 	"strings"
@@ -86,6 +87,11 @@ func ClientCert(name string) string {
 	return new
 }
 
+// PID returns the path to the pid file used by profile for scheduled stop
+func PID(profile string) string {
+	return path.Join(Profile(profile), "pid")
+}
+
 // ClientKey returns client certificate path, used by kubeconfig
 func ClientKey(name string) string {
 	new := filepath.Join(Profile(name), "client.key")

pkg/minikube/reason/reason.go

@@ -114,6 +114,7 @@ var (
 	InternalYamlMarshal      = Kind{ID: "MK_YAML_MARSHAL", ExitCode: ExProgramError}
 	InternalCredsNotFound    = Kind{ID: "MK_CREDENTIALS_NOT_FOUND", ExitCode: ExProgramNotFound, Style: style.Shrug}
 	InternalSemverParse      = Kind{ID: "MK_SEMVER_PARSE", ExitCode: ExProgramError}
+	DaemonizeError           = Kind{ID: "MK_DAEMONIZE", ExitCode: ExProgramError}
 
 	RsrcInsufficientCores             = Kind{ID: "RSRC_INSUFFICIENT_CORES", ExitCode: ExInsufficientCores, Style: style.UnmetRequirement}
 	RsrcInsufficientDarwinDockerCores = Kind{

pkg/minikube/schedule/daemonize_unix.go

+// +build !windows
+
+/*
+Copyright 2020 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package schedule
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"strconv"
+	"time"
+
+	"github.com/VividCortex/godaemon"
+	"github.com/pkg/errors"
+	"k8s.io/klog/v2"
+	"k8s.io/minikube/pkg/minikube/localpath"
+)
+
+// KillExisting kills existing scheduled stops by looking up the PID
+// of the scheduled stop from the PID file saved for the profile and killing the process
+func KillExisting(profiles []string) {
+	for _, profile := range profiles {
+		if err := killPIDForProfile(profile); err != nil {
+			klog.Errorf("error killng PID for profile %s: %v", profile, err)
+		}
+	}
+}
+
+func killPIDForProfile(profile string) error {
+	file := localpath.PID(profile)
+	f, err := ioutil.ReadFile(file)
+	if os.IsNotExist(err) {
+		return nil
+	}
+	defer func() {
+		if err := os.Remove(file); err != nil {
+			klog.Errorf("error deleting %s: %v, you may have to delete in manually", file, err)
+		}
+	}()
+	if err != nil {
+		return errors.Wrapf(err, "reading %s", file)
+	}
+	pid, err := strconv.Atoi(string(f))
+	if err != nil {
+		return errors.Wrapf(err, "converting %s to int", f)
+	}
+	p, err := os.FindProcess(pid)
+	if err != nil {
+		return errors.Wrap(err, "finding process")
+	}
+	klog.Infof("killing process %v as it is an old scheduled stop", pid)
+	if err := p.Kill(); err != nil {
+		return errors.Wrapf(err, "killing %v", pid)
+	}
+	return nil
+}
+
+func daemonize(profiles []string, duration time.Duration) error {
+	_, _, err := godaemon.MakeDaemon(&godaemon.DaemonAttr{})
+	if err != nil {
+		return err
+	}
+	// now that this process has daemonized, it has a new PID
+	pid := os.Getpid()
+	return savePIDs(pid, profiles)
+}
+
+func savePIDs(pid int, profiles []string) error {
+	for _, p := range profiles {
+		file := localpath.PID(p)
+		if err := ioutil.WriteFile(file, []byte(fmt.Sprintf("%v", pid)), 0600); err != nil {
+			return err
+		}
+	}
+	return nil
+}

pkg/minikube/schedule/daemonize_windows.go

+// +build windows
+
+/*
+Copyright 2020 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package schedule
+
+import (
+	"fmt"
+	"time"
+
+	"k8s.io/klog/v2"
+)
+
+// KillExisting will kill existing scheduled stops
+func KillExisting(profiles []string) {
+	klog.Errorf("not yet implemented for windows")
+}
+
+func daemonize(profiles []string, duration time.Duration) error {
+	return fmt.Errorf("not yet implemented for windows")
+}

pkg/minikube/schedule/schedule.go

+/*
+Copyright 2020 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package schedule
+
+import (
+	"time"
+
+	"github.com/pkg/errors"
+	"k8s.io/minikube/pkg/minikube/config"
+	"k8s.io/minikube/pkg/minikube/driver"
+	"k8s.io/minikube/pkg/minikube/mustload"
+	"k8s.io/minikube/pkg/minikube/out"
+)
+
+// Daemonize daemonizes minikube so that scheduled stop happens as expected
+func Daemonize(profiles []string, duration time.Duration) error {
+	// save current time and expected duration in config
+	scheduledStop := &config.ScheduledStopConfig{
+		InitiationTime: time.Now().Unix(),
+		Duration:       duration,
+	}
+	var daemonizeProfiles []string
+	for _, p := range profiles {
+		_, cc := mustload.Partial(p)
+		if driver.BareMetal(cc.Driver) {
+			out.WarningT("scheduled stop is not supported on the none driver, skipping scheduling")
+			continue
+		}
+		daemonizeProfiles = append(daemonizeProfiles, p)
+		cc.ScheduledStop = scheduledStop
+		if err := config.SaveProfile(p, cc); err != nil {
+			return errors.Wrap(err, "saving profile")
+		}
+	}
+
+	return daemonize(daemonizeProfiles, duration)
+}

test/integration/scheduled_stop_test.go

+// +build integration
+
+/*
+Copyright 2020 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package integration
+
+import (
+	"context"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"strconv"
+	"syscall"
+	"testing"
+	"time"
+
+	"github.com/docker/machine/libmachine/state"
+	"k8s.io/minikube/pkg/minikube/localpath"
+	"k8s.io/minikube/pkg/util/retry"
+)
+
+func TestScheduledStop(t *testing.T) {
+	if NoneDriver() {
+		t.Skip("--schedule does not apply to none driver ")
+	}
+	profile := UniqueProfileName("scheduled-stop")
+	ctx, cancel := context.WithTimeout(context.Background(), Minutes(5))
+	defer CleanupWithLogs(t, profile, cancel)
+	startMinikube(ctx, t, profile)
+
+	// schedule a stop for 5 min from now and make sure PID is created
+	scheduledStopMinikube(ctx, t, profile, "5m")
+	pid := checkPID(t, profile)
+	if !processRunning(t, pid) {
+		t.Fatalf("process %v is not running", pid)
+	}
+
+	// redo scheduled stop to be in 10s
+	scheduledStopMinikube(ctx, t, profile, "10s")
+	if processRunning(t, pid) {
+		t.Fatalf("process %v running but should have been killed on reschedule of stop", pid)
+	}
+	checkPID(t, profile)
+	// wait allotted time to make sure minikube status is "Stopped"
+	checkStatus := func() error {
+		got := Status(ctx, t, Target(), profile, "Host", profile)
+		if got != state.Stopped.String() {
+			return fmt.Errorf("expected post-stop host status to be -%q- but got *%q*", state.Stopped, got)
+		}
+		return nil
+	}
+	if err := retry.Expo(checkStatus, time.Second, 30*time.Second); err != nil {
+		t.Fatalf("error %v", err)
+	}
+}
+
+func startMinikube(ctx context.Context, t *testing.T, profile string) {
+	args := append([]string{"start", "-p", profile}, StartArgs()...)
+	rr, err := Run(t, exec.CommandContext(ctx, Target(), args...))
+	if err != nil {
+		t.Fatalf("starting minikube: %v\n%s", err, rr.Output())
+	}
+}
+
+func scheduledStopMinikube(ctx context.Context, t *testing.T, profile string, stop string) {
+	args := []string{"stop", "-p", profile, "--schedule", stop}
+	rr, err := Run(t, exec.CommandContext(ctx, Target(), args...))
+	if err != nil {
+		t.Fatalf("starting minikube: %v\n%s", err, rr.Output())
+	}
+}
+
+func checkPID(t *testing.T, profile string) string {
+	file := localpath.PID(profile)
+	var contents []byte
+	getContents := func() error {
+		var err error
+		contents, err = ioutil.ReadFile(file)
+		return err
+	}
+	// first, make sure the PID file exists
+	if err := retry.Expo(getContents, 100*time.Microsecond, time.Minute*1); err != nil {
+		t.Fatalf("error reading %s: %v", file, err)
+	}
+	return string(contents)
+}
+
+func processRunning(t *testing.T, pid string) bool {
+	// make sure PID file contains a running process
+	p, err := strconv.Atoi(pid)
+	if err != nil {
+		return false
+	}
+	process, err := os.FindProcess(p)
+	if err != nil {
+		return false
+	}
+	err = process.Signal(syscall.Signal(0))
+	t.Log("signal error was: ", err)
+	return err == nil
+}