/* * Copyright (c) 2019 Huawei Technologies Co., Ltd. * A-Tune is licensed under the Mulan PSL v2. * You can use this software according to the terms and conditions of the Mulan PSL v2. * You may obtain a copy of Mulan PSL v2 at: * http://license.coscl.org.cn/MulanPSL2 * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR * PURPOSE. * See the Mulan PSL v2 for more details. * Create: 2019-10-29 */ package project import ( "fmt" PB "gitee.com/openeuler/A-Tune/api/profile" "gitee.com/openeuler/A-Tune/common/log" "gitee.com/openeuler/A-Tune/common/utils" "os/exec" "strconv" "strings" "time" ) const ( LESS = "less" GREATER = "greater" DEPEND_ON = "depend_on" RELY_ON = "rely_on" MULTIPLE = "multiple" ) const ( BASE_BENCHMARK_VALUE = "baseValue" MIN_BENCHMARK_VALUE = "minValue" ) // Evaluate :store the evaluate object type Evaluate struct { Name string `yaml:"name"` Info EvalInfo `yaml:"info"` } // EvalInfo :store the evaluation object type EvalInfo struct { Get string `yaml:"get"` Type string `yaml:"type"` Weight int64 `yaml:"weight"` Threshold int64 `yaml:"threshold"` } // YamlPrjCli :store the client yaml project type YamlPrjCli struct { Project string `yaml:"project"` Iterations int32 `yaml:"iterations"` RandomStarts int32 `yaml:"random_starts"` Benchmark string `yaml:"benchmark"` Engine string `yaml:"engine"` FeatureFilterEngine string `yaml:"feature_filter_engine"` FeatureFilterCycle int32 `yaml:"feature_filter_cycle"` FeatureFilterIters int32 `yaml:"feature_filter_iters"` SplitCount int32 `yaml:"split_count"` Evaluations []Evaluate `yaml:"evaluations"` StartsTime time.Time `yaml:"-"` TotalTime int64 `yaml:"-"` EvalMin float64 `yaml:"-"` EvalBase float64 `yaml:"-"` EvalCurrent float64 `yaml:"-"` StartIters int32 `yaml:"-"` Params string `yaml:"-"` FeatureFilter bool `yaml:"-"` } // YamlPrjSvr :store the server yaml project type YamlPrjSvr struct { Project string `yaml:"project"` Object []*YamlPrjObj `yaml:"object"` Maxiterations int32 `yaml:"maxiterations"` Startworkload string `yaml:"startworkload"` Stopworkload string `yaml:"stopworkload"` } // YamlObj :yaml Object type YamlObj struct { Name string `yaml:"name"` Desc string `yaml:"desc"` GetScript string `yaml:"get"` SetScript string `yaml:"set"` Needrestart string `yaml:"needrestart"` Skip bool `yaml:"skip"` Type string `yaml:"type"` Step float32 `yaml:"step,omitempty"` Items []float32 `yaml:"items"` Options []string `yaml:"options"` Scope []float32 `yaml:"scope,flow"` Dtype string `yaml:"dtype"` Ref string `yaml:"ref"` } // YamlPrjObj :store the yaml object type YamlPrjObj struct { Name string `yaml:"name"` Info YamlObj `yaml:"info"` Relations []*RelationShip `yaml:"relationships"` } type RelationShip struct { Type string `yaml:"type"` Target string `yaml:"target"` Value string `yaml:"value"` SrcName string `yaml:"src_name"` } // BenchMark method call the benchmark script func (y *YamlPrjCli) BenchMark() (string, error) { benchStr := make([]string, 0) benchOutByte, err := ExecCommand(y.Benchmark) if err != nil { fmt.Println(string(benchOutByte)) return "", fmt.Errorf("failed to run benchmark, err: %v", err) } var sum float64 for _, evaluation := range y.Evaluations { newScript := strings.Replace(evaluation.Info.Get, "$out", string(benchOutByte), -1) bout, err := ExecCommand(newScript) if err != nil { err = fmt.Errorf("failed to exec %s, err: %v", newScript, err) return strings.Join(benchStr, ","), err } floatOut, err := strconv.ParseFloat(strings.Replace(string(bout), "\n", "", -1), 64) if err != nil { err = fmt.Errorf("failed to parse float, err: %v", err) return strings.Join(benchStr, ","), err } out := strconv.FormatFloat(floatOut*float64(evaluation.Info.Weight)/100, 'f', -1, 64) if evaluation.Info.Type == "negative" { out = "-" + out sum += -floatOut } else { sum += floatOut } benchStr = append(benchStr, evaluation.Name+"="+out) } if utils.IsEquals(y.EvalBase, 0.0) { y.EvalBase = sum y.EvalMin = sum } if !y.FeatureFilter && sum < y.EvalMin { y.EvalMin = sum } y.EvalCurrent = sum return strings.Join(benchStr, ","), nil } // Threshold return the threshold, which replace with the benchmark result. // it is used when the parameters is not match with the relations func (y *YamlPrjCli) Threshold() (string, error) { benchStr := make([]string, 0) for _, evaluation := range y.Evaluations { out := strconv.FormatFloat(float64(evaluation.Info.Threshold)*float64(evaluation.Info.Weight)/100, 'f', -1, 64) if evaluation.Info.Type == "negative" { out = "-" + out } benchStr = append(benchStr, evaluation.Name+"="+out) } return strings.Join(benchStr, ","), nil } // SetHistoryEvalBase method call the set the current EvalBase to history baseline func (y *YamlPrjCli) SetHistoryEvalBase(tuningHistory *PB.TuningHistory) { if !utils.IsEquals(y.EvalBase, 0.0) { return } evaluation := strings.Split(tuningHistory.BaseEval, "=") if len(evaluation) != 2 { return } evalBase, _ := strconv.ParseFloat(evaluation[1], 64) y.EvalBase = evalBase minEval := strings.Split(tuningHistory.MinEval, "=") if len(minEval) != 2 { return } y.EvalMin, _ = strconv.ParseFloat(minEval[1], 64) y.TotalTime = tuningHistory.TotalTime y.StartIters = tuningHistory.Starts } // ImproveRateString method return the string format of performance improve rate func (y *YamlPrjCli) ImproveRateString(current float64) string { if y.EvalBase < 0 { return fmt.Sprintf("%.2f", (current-y.EvalBase)/y.EvalBase*100) } return fmt.Sprintf("%.2f", (y.EvalBase-current)/current*100) } // RunSet method call the set script to set the value func (y *YamlPrjSvr) RunSet(optStr string) (error, string) { paraMap := make(map[string]string) paraSlice := strings.Split(optStr, ",") for _, para := range paraSlice { kvs := strings.Split(para, "=") if len(kvs) < 2 { continue } paraMap[kvs[0]] = kvs[1] } log.Infof("before change paraMap: %+v\n", paraMap) scripts := make([]string, 0) for _, obj := range y.Object { if obj.Info.Skip { log.Infof("item %s is skiped", obj.Name) continue } var objName string active := true for _, relation := range obj.Relations { if relation.Type == RELY_ON && paraMap[relation.Target] != relation.Value { active = false break } if relation.Type == DEPEND_ON && paraMap[relation.Target] != relation.Value { continue } if relation.Type == DEPEND_ON { objName = relation.SrcName } if relation.Type == MULTIPLE { targetValue, _ := strconv.ParseFloat(paraMap[relation.Target], 32) objValue, _ := strconv.ParseFloat(paraMap[obj.Name], 32) paraMap[obj.Name] = strconv.Itoa(int(targetValue * objValue)) continue } } if !active { log.Infof("%s value is not match the relations", obj.Name) continue } out, err := ExecCommand(obj.Info.GetScript) if err != nil { return fmt.Errorf("failed to exec %s, err: %v", obj.Info.GetScript, err), "" } if strings.TrimSpace(string(out)) == paraMap[obj.Name] { log.Infof("%s need not be set, value is %s", obj.Name, paraMap[obj.Name]) continue } script := obj.Info.SetScript var newScript string if len(strings.Fields(paraMap[obj.Name])) > 1 { newScript = strings.Replace(script, "$value", "\""+paraMap[obj.Name]+"\"", -1) } else { newScript = strings.Replace(script, "$value", paraMap[obj.Name], -1) } newScript = strings.Replace(newScript, "$name", objName, -1) log.Info("set script:", newScript) _, err = ExecCommand(newScript) if err != nil { return fmt.Errorf("failed to exec %s, err: %v", newScript, err), "" } scripts = append(scripts, newScript) } log.Infof("after change paraMap: %+v\n", paraMap) return nil, strings.Join(scripts, ",") } // RestartProject method call the StartWorkload and StopWorkload script to restart the service func (y *YamlPrjSvr) RestartProject() (error, string) { startWorkload := y.Startworkload stopWorkload := y.Stopworkload needRestart := false for _, obj := range y.Object { if obj.Info.Needrestart == "true" { needRestart = true break } } scripts := make([]string, 0) if needRestart { out, err := ExecCommand(stopWorkload) if err != nil { return fmt.Errorf("failed to exec %s, err: %v", stopWorkload, err), "" } log.Debug(string(out)) scripts = append(scripts, stopWorkload) log.Debugf("start workload script is: %s", startWorkload) out, err = ExecCommand(startWorkload) if err != nil { return fmt.Errorf("failed to exec %s, err: %v", startWorkload, err), "" } log.Debug(string(out)) scripts = append(scripts, startWorkload) } return nil, strings.Join(scripts, ",") } // MergeProject two yaml project to one object func (y *YamlPrjSvr) MergeProject(prj *YamlPrjSvr) error { for _, obj := range prj.Object { y.Object = append(y.Object, obj) } return nil } // MatchRelations method check if the params match the relations // if less or greater is not match, return false, else return true func (y *YamlPrjSvr) MatchRelations(optStr string) bool { paraMap := make(map[string]string) paraSlice := strings.Split(optStr, ",") for _, para := range paraSlice { kvs := strings.Split(para, "=") if len(kvs) < 2 { continue } paraMap[kvs[0]] = kvs[1] } for _, obj := range y.Object { if obj.Info.Skip { log.Infof("item %s is skiped", obj.Name) continue } for _, relation := range obj.Relations { if relation.Type != LESS && relation.Type != GREATER { continue } targetValue, _ := strconv.Atoi(paraMap[relation.Target]) objValue, _ := strconv.Atoi(paraMap[obj.Name]) if relation.Type == LESS && objValue > targetValue { return false } if relation.Type == GREATER && objValue < targetValue { return false } } } return true } //exec command and get result func ExecCommand(script string) ([]byte, error) { cmd := exec.Command("sh", "-c", script) return cmd.CombinedOutput() }