diff --git a/cmd/minikube/cmd/stop.go b/cmd/minikube/cmd/stop.go index 16b09882baa1daf349759a0fbf58b00f7ff55dac..a677051b66d49d1ade04e2bc92a63612347d23a5 100644 --- a/cmd/minikube/cmd/stop.go +++ b/cmd/minikube/cmd/stop.go @@ -18,6 +18,7 @@ package cmd import ( "os" + "runtime" "time" "github.com/docker/machine/libmachine" @@ -36,13 +37,15 @@ import ( "k8s.io/minikube/pkg/minikube/out" "k8s.io/minikube/pkg/minikube/out/register" "k8s.io/minikube/pkg/minikube/reason" + "k8s.io/minikube/pkg/minikube/schedule" "k8s.io/minikube/pkg/minikube/style" "k8s.io/minikube/pkg/util/retry" ) var ( - stopAll bool - keepActive bool + stopAll bool + keepActive bool + scheduledStopDuration time.Duration ) // stopCmd represents the stop command @@ -56,6 +59,10 @@ var stopCmd = &cobra.Command{ func init() { stopCmd.Flags().BoolVar(&stopAll, "all", false, "Set flag to stop all profiles (clusters)") stopCmd.Flags().BoolVar(&keepActive, "keep-context-active", false, "keep the kube-context active after cluster is stopped. Defaults to false.") + stopCmd.Flags().DurationVar(&scheduledStopDuration, "schedule", 0*time.Second, "Set flag to stop cluster after a set amount of time (e.g. --schedule=5m)") + if err := stopCmd.Flags().MarkHidden("schedule"); err != nil { + klog.Info("unable to mark --schedule flag as hidden") + } stopCmd.Flags().StringVarP(&outputFormat, "output", "o", "text", "Format to print stdout in. Options include: [text,json]") if err := viper.GetViper().BindPFlags(stopCmd.Flags()); err != nil { @@ -88,6 +95,20 @@ func runStop(cmd *cobra.Command, args []string) { profilesToStop = append(profilesToStop, cname) } + // Kill any existing scheduled stops + schedule.KillExisting(profilesToStop) + + if scheduledStopDuration != 0 { + if runtime.GOOS == "windows" { + exit.Message(reason.Usage, "the --schedule flag is currently not supported on windows") + } + if err := schedule.Daemonize(profilesToStop, scheduledStopDuration); err != nil { + exit.Message(reason.DaemonizeError, "unable to daemonize: {{.err}}", out.V{"err": err.Error()}) + } + klog.Infof("sleeping %s before completing stop...", scheduledStopDuration.String()) + time.Sleep(scheduledStopDuration) + } + stoppedNodes := 0 for _, profile := range profilesToStop { stoppedNodes = stopProfile(profile) diff --git a/go.mod b/go.mod index 8292131c307760a4f4c81703e4c4c53559dc7ebd..d12b9a3feae75d669b6dd7017cab5ce1b0dffb36 100644 --- a/go.mod +++ b/go.mod @@ -8,6 +8,7 @@ require ( github.com/Microsoft/go-winio v0.4.15-0.20190919025122-fc70bd9a86b5 // indirect github.com/Parallels/docker-machine-parallels/v2 v2.0.1 github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d // indirect + github.com/VividCortex/godaemon v0.0.0-20201030160542-15e3f4925a21 github.com/blang/semver v3.5.0+incompatible github.com/c4milo/gotoolkit v0.0.0-20170318115440-bcc06269efa9 // indirect github.com/cenkalti/backoff v2.2.1+incompatible diff --git a/go.sum b/go.sum index ca950495ac023fb2517b5428bad658b3c91bd2e2..22abe6c76e85038e9b63d5c0d9f6033088325930 100644 --- a/go.sum +++ b/go.sum @@ -118,6 +118,8 @@ github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d h1:G0m3OIz70MZUW github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= github.com/VividCortex/ewma v1.1.1 h1:MnEK4VOv6n0RSY4vtRe3h11qjxL3+t0B8yOL8iMXdcM= github.com/VividCortex/ewma v1.1.1/go.mod h1:2Tkkvm3sRDVXaiyucHiACn4cqf7DpdyLvmxzcbUokwA= +github.com/VividCortex/godaemon v0.0.0-20201030160542-15e3f4925a21 h1:Pgxfz/g+XyfRjYqRjKUFpDh5IciFncmA/Uio6AU/z9g= +github.com/VividCortex/godaemon v0.0.0-20201030160542-15e3f4925a21/go.mod h1:Y8CJ3IwPIAkMhv/rRUWIlczaeqd9ty9yrl+nc2AbaL4= github.com/afbjorklund/go-containerregistry v0.0.0-20200902152226-fbad78ec2813 h1:0tskN1ipU/BBrpoEIy0rdZS9jf5+wdP6IMRak8Iu/YE= github.com/afbjorklund/go-containerregistry v0.0.0-20200902152226-fbad78ec2813/go.mod h1:npTSyywOeILcgWqd+rvtzGWflIPPcBQhYoOONaY4ltM= github.com/afbjorklund/go-getter v1.4.1-0.20201020145846-c0da14b4bffe h1:TdcuDqk4ArmYI8cbeeL/RM5BPciDOaWpGZoPoT3OziQ= diff --git a/pkg/minikube/config/types.go b/pkg/minikube/config/types.go index c54be3e4b7a1a000f679f2984966142cfc7bb8ea..1d19604bc4af3bcca2879797c16f1f565b91e31d 100644 --- a/pkg/minikube/config/types.go +++ b/pkg/minikube/config/types.go @@ -71,6 +71,7 @@ type ClusterConfig struct { Addons map[string]bool VerifyComponents map[string]bool // map of components to verify and wait for after start. StartHostTimeout time.Duration + ScheduledStop *ScheduledStopConfig ExposedPorts []string // Only used by the docker and podman driver } @@ -138,3 +139,10 @@ type VersionedExtraOption struct { // flag is applied to GreaterThanOrEqual semver.Version } + +// ScheduledStopConfig contains information around scheduled stop +// not yet used, will be used to show status of scheduled stop +type ScheduledStopConfig struct { + InitiationTime int64 + Duration time.Duration +} diff --git a/pkg/minikube/localpath/localpath.go b/pkg/minikube/localpath/localpath.go index efbd489ca1ed2be8b0c7e9428e402f6a181101c1..6d87c45d10b368014cb520aea03b9b1f145ba30a 100644 --- a/pkg/minikube/localpath/localpath.go +++ b/pkg/minikube/localpath/localpath.go @@ -19,6 +19,7 @@ package localpath import ( "os" "os/exec" + "path" "path/filepath" "runtime" "strings" @@ -86,6 +87,11 @@ func ClientCert(name string) string { return new } +// PID returns the path to the pid file used by profile for scheduled stop +func PID(profile string) string { + return path.Join(Profile(profile), "pid") +} + // ClientKey returns client certificate path, used by kubeconfig func ClientKey(name string) string { new := filepath.Join(Profile(name), "client.key") diff --git a/pkg/minikube/reason/reason.go b/pkg/minikube/reason/reason.go index f90c1d98d86d56bc2a17fff555455c44c253c4fb..9cce8f1af000dc8f1e76641eaae6e77a248af4b3 100644 --- a/pkg/minikube/reason/reason.go +++ b/pkg/minikube/reason/reason.go @@ -114,6 +114,7 @@ var ( InternalYamlMarshal = Kind{ID: "MK_YAML_MARSHAL", ExitCode: ExProgramError} InternalCredsNotFound = Kind{ID: "MK_CREDENTIALS_NOT_FOUND", ExitCode: ExProgramNotFound, Style: style.Shrug} InternalSemverParse = Kind{ID: "MK_SEMVER_PARSE", ExitCode: ExProgramError} + DaemonizeError = Kind{ID: "MK_DAEMONIZE", ExitCode: ExProgramError} RsrcInsufficientCores = Kind{ID: "RSRC_INSUFFICIENT_CORES", ExitCode: ExInsufficientCores, Style: style.UnmetRequirement} RsrcInsufficientDarwinDockerCores = Kind{ diff --git a/pkg/minikube/schedule/daemonize_unix.go b/pkg/minikube/schedule/daemonize_unix.go new file mode 100644 index 0000000000000000000000000000000000000000..022b6de8a6d95153b1d597aab6684e7d3108a251 --- /dev/null +++ b/pkg/minikube/schedule/daemonize_unix.go @@ -0,0 +1,91 @@ +// +build !windows + +/* +Copyright 2020 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package schedule + +import ( + "fmt" + "io/ioutil" + "os" + "strconv" + "time" + + "github.com/VividCortex/godaemon" + "github.com/pkg/errors" + "k8s.io/klog/v2" + "k8s.io/minikube/pkg/minikube/localpath" +) + +// KillExisting kills existing scheduled stops by looking up the PID +// of the scheduled stop from the PID file saved for the profile and killing the process +func KillExisting(profiles []string) { + for _, profile := range profiles { + if err := killPIDForProfile(profile); err != nil { + klog.Errorf("error killng PID for profile %s: %v", profile, err) + } + } +} + +func killPIDForProfile(profile string) error { + file := localpath.PID(profile) + f, err := ioutil.ReadFile(file) + if os.IsNotExist(err) { + return nil + } + defer func() { + if err := os.Remove(file); err != nil { + klog.Errorf("error deleting %s: %v, you may have to delete in manually", file, err) + } + }() + if err != nil { + return errors.Wrapf(err, "reading %s", file) + } + pid, err := strconv.Atoi(string(f)) + if err != nil { + return errors.Wrapf(err, "converting %s to int", f) + } + p, err := os.FindProcess(pid) + if err != nil { + return errors.Wrap(err, "finding process") + } + klog.Infof("killing process %v as it is an old scheduled stop", pid) + if err := p.Kill(); err != nil { + return errors.Wrapf(err, "killing %v", pid) + } + return nil +} + +func daemonize(profiles []string, duration time.Duration) error { + _, _, err := godaemon.MakeDaemon(&godaemon.DaemonAttr{}) + if err != nil { + return err + } + // now that this process has daemonized, it has a new PID + pid := os.Getpid() + return savePIDs(pid, profiles) +} + +func savePIDs(pid int, profiles []string) error { + for _, p := range profiles { + file := localpath.PID(p) + if err := ioutil.WriteFile(file, []byte(fmt.Sprintf("%v", pid)), 0600); err != nil { + return err + } + } + return nil +} diff --git a/pkg/minikube/schedule/daemonize_windows.go b/pkg/minikube/schedule/daemonize_windows.go new file mode 100644 index 0000000000000000000000000000000000000000..ea3c7fa238565be794b6e533756271d6b9acb471 --- /dev/null +++ b/pkg/minikube/schedule/daemonize_windows.go @@ -0,0 +1,35 @@ +// +build windows + +/* +Copyright 2020 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package schedule + +import ( + "fmt" + "time" + + "k8s.io/klog/v2" +) + +// KillExisting will kill existing scheduled stops +func KillExisting(profiles []string) { + klog.Errorf("not yet implemented for windows") +} + +func daemonize(profiles []string, duration time.Duration) error { + return fmt.Errorf("not yet implemented for windows") +} diff --git a/pkg/minikube/schedule/schedule.go b/pkg/minikube/schedule/schedule.go new file mode 100644 index 0000000000000000000000000000000000000000..a016cf00bffa51a3138044f9ff9da3437b08da28 --- /dev/null +++ b/pkg/minikube/schedule/schedule.go @@ -0,0 +1,51 @@ +/* +Copyright 2020 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package schedule + +import ( + "time" + + "github.com/pkg/errors" + "k8s.io/minikube/pkg/minikube/config" + "k8s.io/minikube/pkg/minikube/driver" + "k8s.io/minikube/pkg/minikube/mustload" + "k8s.io/minikube/pkg/minikube/out" +) + +// Daemonize daemonizes minikube so that scheduled stop happens as expected +func Daemonize(profiles []string, duration time.Duration) error { + // save current time and expected duration in config + scheduledStop := &config.ScheduledStopConfig{ + InitiationTime: time.Now().Unix(), + Duration: duration, + } + var daemonizeProfiles []string + for _, p := range profiles { + _, cc := mustload.Partial(p) + if driver.BareMetal(cc.Driver) { + out.WarningT("scheduled stop is not supported on the none driver, skipping scheduling") + continue + } + daemonizeProfiles = append(daemonizeProfiles, p) + cc.ScheduledStop = scheduledStop + if err := config.SaveProfile(p, cc); err != nil { + return errors.Wrap(err, "saving profile") + } + } + + return daemonize(daemonizeProfiles, duration) +} diff --git a/test/integration/scheduled_stop_test.go b/test/integration/scheduled_stop_test.go new file mode 100644 index 0000000000000000000000000000000000000000..ba0341b942319fa4e9a7ad8b39bb5a5ab6f77687 --- /dev/null +++ b/test/integration/scheduled_stop_test.go @@ -0,0 +1,116 @@ +// +build integration + +/* +Copyright 2020 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package integration + +import ( + "context" + "fmt" + "io/ioutil" + "os" + "os/exec" + "strconv" + "syscall" + "testing" + "time" + + "github.com/docker/machine/libmachine/state" + "k8s.io/minikube/pkg/minikube/localpath" + "k8s.io/minikube/pkg/util/retry" +) + +func TestScheduledStop(t *testing.T) { + if NoneDriver() { + t.Skip("--schedule does not apply to none driver ") + } + profile := UniqueProfileName("scheduled-stop") + ctx, cancel := context.WithTimeout(context.Background(), Minutes(5)) + defer CleanupWithLogs(t, profile, cancel) + startMinikube(ctx, t, profile) + + // schedule a stop for 5 min from now and make sure PID is created + scheduledStopMinikube(ctx, t, profile, "5m") + pid := checkPID(t, profile) + if !processRunning(t, pid) { + t.Fatalf("process %v is not running", pid) + } + + // redo scheduled stop to be in 10s + scheduledStopMinikube(ctx, t, profile, "10s") + if processRunning(t, pid) { + t.Fatalf("process %v running but should have been killed on reschedule of stop", pid) + } + checkPID(t, profile) + // wait allotted time to make sure minikube status is "Stopped" + checkStatus := func() error { + got := Status(ctx, t, Target(), profile, "Host", profile) + if got != state.Stopped.String() { + return fmt.Errorf("expected post-stop host status to be -%q- but got *%q*", state.Stopped, got) + } + return nil + } + if err := retry.Expo(checkStatus, time.Second, 30*time.Second); err != nil { + t.Fatalf("error %v", err) + } +} + +func startMinikube(ctx context.Context, t *testing.T, profile string) { + args := append([]string{"start", "-p", profile}, StartArgs()...) + rr, err := Run(t, exec.CommandContext(ctx, Target(), args...)) + if err != nil { + t.Fatalf("starting minikube: %v\n%s", err, rr.Output()) + } +} + +func scheduledStopMinikube(ctx context.Context, t *testing.T, profile string, stop string) { + args := []string{"stop", "-p", profile, "--schedule", stop} + rr, err := Run(t, exec.CommandContext(ctx, Target(), args...)) + if err != nil { + t.Fatalf("starting minikube: %v\n%s", err, rr.Output()) + } +} + +func checkPID(t *testing.T, profile string) string { + file := localpath.PID(profile) + var contents []byte + getContents := func() error { + var err error + contents, err = ioutil.ReadFile(file) + return err + } + // first, make sure the PID file exists + if err := retry.Expo(getContents, 100*time.Microsecond, time.Minute*1); err != nil { + t.Fatalf("error reading %s: %v", file, err) + } + return string(contents) +} + +func processRunning(t *testing.T, pid string) bool { + // make sure PID file contains a running process + p, err := strconv.Atoi(pid) + if err != nil { + return false + } + process, err := os.FindProcess(p) + if err != nil { + return false + } + err = process.Signal(syscall.Signal(0)) + t.Log("signal error was: ", err) + return err == nil +}