!85 atune: sync dev branch code

Merge pull request !85 from hanxinke/master

common/client/client.go

@@ -137,8 +137,7 @@ func NewClient(addr string, port string, opts ...Opt) (*Client, error) {
 
 func UnixConnect(ctx context.Context, addr string) (net.Conn, error) {
 	unix_addr, _ := net.ResolveUnixAddr("unix", addr)
-	conn, err := net.DialUnix("unix", nil, unix_addr)
-	return conn, err
+	return net.DialUnix("unix", nil, unix_addr)
 }
 
 func newClient(address string, opts ...Opt) (*Client, error) {

common/config/config.go

@@ -97,6 +97,7 @@ const (
 var (
 	TransProtocol     string
 	Address           string
+	Connect           string
 	Port              string
 	RestPort          string
 	TLS               bool
@@ -134,6 +135,7 @@ func (c *Cfg) Load() error {
 	section := cfg.Section("server")
 	TransProtocol = section.Key("protocol").MustString(DefaultProtocol)
 	Address = section.Key("address").MustString(DefaultTgtAddr)
+	Connect = section.Key("connect").MustString("")
 	Port = section.Key("port").MustString(DefaultTgtPort)
 	RestPort = section.Key("rest_port").MustString("8383")
 	TLS = section.Key("tls").MustBool(false)

common/profile/profile.go

@@ -128,7 +128,7 @@ func (p *Profile) Backup() error {
 	backedPath := path.Join(config.DefaultBackupPath, p.name+"_"+timeUnix)
 	err := os.MkdirAll(backedPath, 0750)
 	if err != nil {
-		log.Error(err.Error())
+		log.Error("failed to create backup path")
 		return nil
 	}
 

common/project/projet.go

@@ -108,7 +108,7 @@ func (y *YamlPrjCli) BenchMark() (string, error) {
 }
 
 // RunSet method call the set script to set the value
-func (y *YamlPrjSvr) RunSet(optStr string) error {
+func (y *YamlPrjSvr) RunSet(optStr string) (error, string) {
 	paraMap := make(map[string]string)
 	paraSlice := strings.Split(optStr, ",")
 	for _, para := range paraSlice {
@@ -118,10 +118,11 @@ func (y *YamlPrjSvr) RunSet(optStr string) error {
 		}
 		paraMap[kvs[0]] = kvs[1]
 	}
+	scripts := make([]string, 0)
 	for _, obj := range y.Object {
 		out, err := ExecCommand(obj.Info.GetScript)
 		if err != nil {
-			return fmt.Errorf("failed to exec %s, err: %v", obj.Info.GetScript, err)
+			return fmt.Errorf("failed to exec %s, err: %v", obj.Info.GetScript, err), ""
 		}
 
 		if strings.TrimSpace(string(out)) == paraMap[obj.Name] {
@@ -133,14 +134,15 @@ func (y *YamlPrjSvr) RunSet(optStr string) error {
 		log.Info("set script:", newScript)
 		_, err = ExecCommand(newScript)
 		if err != nil {
-			return fmt.Errorf("failed to exec %s, err: %v", newScript, err)
+			return fmt.Errorf("failed to exec %s, err: %v", newScript, err), ""
 		}
+		scripts = append(scripts, newScript)
 	}
-	return nil
+	return nil, strings.Join(scripts, ",")
 }
 
 // RestartProject method call the StartWorkload and StopWorkload script to restart the service
-func (y *YamlPrjSvr) RestartProject() error {
+func (y *YamlPrjSvr) RestartProject() (error, string) {
 	startWorkload := y.Startworkload
 	stopWorkload := y.Stopworkload
 
@@ -152,26 +154,30 @@ func (y *YamlPrjSvr) RestartProject() error {
 			break
 		}
 	}
+
+	scripts := make([]string, 0)
 	if needRestart {
 		out, err := ExecCommand(stopWorkload)
 		if err != nil {
-			return fmt.Errorf("failed to exec %s, err: %v", stopWorkload, err)
+			return fmt.Errorf("failed to exec %s, err: %v", stopWorkload, err), ""
 		}
 		log.Debug(string(out))
+		scripts = append(scripts, stopWorkload)
 
 		out, err = ExecCommand(startWorkload)
 		if err != nil {
-			return fmt.Errorf("failed to exec %s, err: %v", startWorkload, err)
+			return fmt.Errorf("failed to exec %s, err: %v", startWorkload, err), ""
 		}
 		log.Debug(string(out))
+
+		scripts = append(scripts, startWorkload)
 	}
 
-	return nil
+	return nil, strings.Join(scripts, ",")
 }
 
 //exec command and get result
 func ExecCommand(script string) ([]byte, error) {
 	cmd := exec.Command("sh", "-c", script)
-	out, err := cmd.CombinedOutput()
-	return out, err
+	return cmd.CombinedOutput()
 }

common/service/timer/timer.go

@@ -60,7 +60,7 @@ func (t *Timer) Run() error {
 
 	section, err := t.Cfg.Raw.GetSection("server")
 	if err != nil {
-		log.Errorf("Faild to get section system, error: %v", err)
+		log.Errorf("failed to get section system, error: %v", err)
 		return err
 	}
 	if !section.Haskey("type") {

common/tuning/optimizer.go

@@ -15,6 +15,7 @@ package tuning
 
 import (
 	PB "atune/api/profile"
+	"atune/common/client"
 	"atune/common/config"
 	"atune/common/http"
 	"atune/common/log"
@@ -22,6 +23,8 @@ import (
 	"atune/common/project"
 	"atune/common/utils"
 	"fmt"
+	"golang.org/x/net/context"
+	"io"
 	"io/ioutil"
 	"os"
 	"path"
@@ -57,7 +60,6 @@ func (o *Optimizer) InitTuned(ch chan *PB.AckCheck) error {
 	//dynamic profle setting
 	o.MaxIter = clientIter
 	if o.MaxIter > o.Prj.Maxiterations {
-		o.MaxIter = o.Prj.Maxiterations
 		log.Infof("project:%s max iterations:%d", o.Prj.Project, o.Prj.Maxiterations)
 		ch <- &PB.AckCheck{Name: fmt.Sprintf("server project %s max iterations %d\n",
 			o.Prj.Project, o.Prj.Maxiterations)}
@@ -153,18 +155,30 @@ func (o *Optimizer) DynamicTuned(ch chan *PB.AckCheck) error {
 	}
 
 	log.Infof("setting params is: %s", o.RespPutIns.Param)
-	if err := o.Prj.RunSet(o.RespPutIns.Param); err != nil {
+	err, scripts := o.Prj.RunSet(o.RespPutIns.Param)
+	if err != nil {
 		log.Error(err)
 		return err
 	}
 	log.Info("set the parameter success")
 
-	if err := o.Prj.RestartProject(); err != nil {
+	err = syncConfigToOthers(scripts)
+	if err != nil {
+		return err
+	}
+
+	err, scripts = o.Prj.RestartProject()
+	if err != nil {
 		log.Error(err)
 		return err
 	}
 	log.Info("restart project success")
 
+	err = syncConfigToOthers(scripts)
+	if err != nil {
+		return err
+	}
+
 	o.StartIterTime = time.Now().Format(config.DefaultTimeFormat)
 
 	if o.Iter == o.MaxIter {
@@ -206,11 +220,16 @@ func (o *Optimizer) RestoreConfigTuned(ch chan *PB.AckCheck) error {
 	}
 
 	log.Infof("restoring params is: %s", string(content))
-	if err := o.Prj.RunSet(string(content)); err != nil {
+	err, scripts := o.Prj.RunSet(string(content))
+	if err != nil {
 		log.Error(err)
 		return err
 	}
 
+	if err := syncConfigToOthers(scripts); err != nil {
+		return err
+	}
+
 	result := fmt.Sprintf("restore %s project params success", o.Prj.Project)
 	ch <- &PB.AckCheck{Name: result, Status: utils.SUCCESS}
 	log.Infof(result)
@@ -301,3 +320,80 @@ func CheckServerPrj(data string, optimizer *Optimizer) error {
 
 	return fmt.Errorf("project:%s not found", data)
 }
+
+//sync tuned node
+func (o *Optimizer) SyncTunedNode(ch chan *PB.AckCheck) error {
+	log.Infof("setting params is: %s", string(o.Content))
+	commands := strings.Split(string(o.Content), ",")
+	for _, command := range commands {
+		_, err := project.ExecCommand(command)
+		if err != nil {
+			return fmt.Errorf("failed to exec %s, err: %v", command, err)
+		}
+	}
+
+	log.Info("set the parameter success")
+	ch <- &PB.AckCheck{Status: utils.SUCCESS}
+
+	return nil
+}
+
+//sync config to other nodes in cluster mode
+func syncConfigToOthers(scripts string) error {
+	if config.TransProtocol != "tcp" || scripts == "" {
+		return nil
+	}
+
+	otherServers := strings.Split(strings.TrimSpace(config.Connect), ",")
+	log.Infof("sync other nodes: %s", otherServers)
+
+	for _, server := range otherServers {
+		if server == config.Address || server == "" {
+			continue
+		}
+		if err := syncConfigToNode(server, scripts); err != nil {
+			log.Errorf("server %s failed to sync config, err: %v", server, err)
+			return err
+		}
+	}
+	return nil
+}
+
+//sync config to server node
+func syncConfigToNode(server string, scripts string) error {
+	c, err := client.NewClient(server, config.Port)
+	if err != nil {
+		return err
+	}
+
+	defer c.Close()
+	svc := PB.NewProfileMgrClient(c.Connection())
+	stream, err := svc.Tuning(context.Background())
+	if err != nil {
+		return err
+	}
+
+	defer stream.CloseSend()
+	content := &PB.ProfileInfo{Name: "sync-config", Content: []byte(scripts)}
+	if err := stream.Send(content); err != nil {
+		return fmt.Errorf("sends failure, error: %v", err)
+	}
+
+	for {
+		reply, err := stream.Recv()
+		if err == io.EOF {
+			break
+		}
+
+		if err != nil {
+			return err
+		}
+
+		if reply.Status == utils.SUCCESS {
+			log.Infof("server %s reply status success", server)
+			break
+		}
+	}
+
+	return nil
+}

misc/atuned.cnf

@@ -21,6 +21,10 @@ protocol = unix
 # ranges: /var/run/atuned/atuned.sock or ip address
 address = /var/run/atuned/atuned.sock
 
+# the atune nodes in cluster mode, separated by commas
+# it is valid when protocol is tcp
+# connect = ip01,ip02,ip03
+
 # the atuned grpc listening port
 # the port can be set between 0 to 65535 which not be used
 # port = 60001

modules/server/profile/profile.go

@@ -368,7 +368,7 @@ func (s *ProfileServer) Analysis(message *PB.AnalysisMessage, stream PB.ProfileM
 
 	//3. judge the workload type is exist in the database
 	classProfile := &sqlstore.GetClass{Class: workloadType}
-	if err := sqlstore.GetClasses(classProfile); err != nil {
+	if err = sqlstore.GetClasses(classProfile); err != nil {
 		log.Errorf("inquery workload type table failed %v", err)
 		return fmt.Errorf("inquery workload type table failed %v", err)
 	}
@@ -476,23 +476,31 @@ func (s *ProfileServer) Tuning(stream PB.ProfileMgr_TuningServer) error {
 		data := reply.Name
 		content := reply.Content
 
+		//sync config with other server
+		if data == "sync-config" {
+			optimizer.Content = content
+			if err = optimizer.SyncTunedNode(ch); err != nil {
+				return err
+			}
+			continue
+		}
+
 		// data == "" means in tuning process
 		if data == "" {
 			optimizer.Content = content
-			err := optimizer.DynamicTuned(ch)
-			if err != nil {
+			if err = optimizer.DynamicTuned(ch); err != nil {
 				return err
 			}
 			continue
 		}
 
-		if err := tuning.CheckServerPrj(data, &optimizer); err != nil {
+		if err = tuning.CheckServerPrj(data, &optimizer); err != nil {
 			return err
 		}
 
 		//content == nil means in restore config
 		if content == nil {
-			if err := optimizer.RestoreConfigTuned(ch); err != nil {
+			if err = optimizer.RestoreConfigTuned(ch); err != nil {
 				return err
 			}
 			continue
@@ -575,9 +583,9 @@ func (s *ProfileServer) InfoProfile(profileInfo *PB.ProfileInfo, stream PB.Profi
 	profileNames := strings.Split(profileType, ",")
 	for _, name := range profileNames {
 		name = strings.Trim(name, " ")
-		context, _ := sqlstore.GetContext(name)
-		context = "\n*** " + name + ":\n" + context
-		_ = stream.Send(&PB.ProfileInfo{Name: context})
+		infoName, _ := sqlstore.GetContext(name)
+		infoName = "\n*** " + name + ":\n" + infoName
+		_ = stream.Send(&PB.ProfileInfo{Name: infoName})
 	}
 
 	return nil
@@ -706,8 +714,7 @@ func (s *ProfileServer) Collection(message *PB.CollectFlag, stream PB.ProfileMgr
 	}
 
 	classApps := &sqlstore.GetClassApp{Class: message.GetType()}
-	err = sqlstore.GetClassApps(classApps)
-	if err != nil {
+	if err = sqlstore.GetClassApps(classApps); err != nil {
 		return err
 	}
 	if len(classApps.Result) == 0 {
@@ -722,11 +729,11 @@ func (s *ProfileServer) Collection(message *PB.CollectFlag, stream PB.ProfileMgr
 		return fmt.Errorf("output_path %s is not exist", message.GetOutputPath())
 	}
 
-	if err := utils.InterfaceByName(message.GetNetwork()); err != nil {
+	if err = utils.InterfaceByName(message.GetNetwork()); err != nil {
 		return err
 	}
 
-	if err := utils.DiskByName(message.GetBlock()); err != nil {
+	if err = utils.DiskByName(message.GetBlock()); err != nil {
 		return err
 	}
 

scripts/blockdev/get.sh

+#!/bin/sh
+# Copyright (c) 2020 Huawei Technologies Co., Ltd.
+# A-Tune is licensed under the Mulan PSL v2.
+# You can use this software according to the terms and conditions of the Mulan PSL v2.
+# You may obtain a copy of Mulan PSL v2 at:
+#     http://license.coscl.org.cn/MulanPSL2
+# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+# PURPOSE.
+# See the Mulan PSL v2 for more details.
+# Create: 2020-03-03
+
+disk=$(echo "$@" | awk '{print $1}')
+value=$(/sbin/blockdev --getra /dev/"$disk")
+ret=$?
+[ $ret -ne 0 ] && echo "\033[31m failed to command /sbin/blockdev --getra \033[31m" && exit 1
+echo "$disk $value"

scripts/blockdev/set.sh

+#!/bin/sh
+# Copyright (c) 2020 Huawei Technologies Co., Ltd.
+# A-Tune is licensed under the Mulan PSL v2.
+# You can use this software according to the terms and conditions of the Mulan PSL v2.
+# You may obtain a copy of Mulan PSL v2 at:
+#     http://license.coscl.org.cn/MulanPSL2
+# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+# PURPOSE.
+# See the Mulan PSL v2 for more details.
+# Create: 2020-03-03
+
+disk=$(echo "$@" | awk '{print $1}')
+value=$(echo "$@" | awk '{print $2}')
+/sbin/blockdev --setra "$value" /dev/"$disk"

scripts/ethtool/common.sh

@@ -13,6 +13,8 @@
 declare -A kmap=(["gro"]="generic-receive-offload" ["gso"]="generic-segmentation-offload"
   ["tso"]="tcp-segmentation-offload" ["lro"]="large-receive-offload")
 declare -A cmap=(["adaptive-rx"]="Adaptive" ["adaptive-tx"]="Adaptive")
+declare -A lmap=(["combined"]="Combined")
+declare -A gmap=(["rx"]="RX:" ["tx"]="TX:")
 koption="on off"
 xoption="toeplitz xor crc32"
 coption="on off"
@@ -32,12 +34,18 @@ function get_ethtool_value() {
   network=${para[1]}
   case "$option" in
   "-K")
-    value=$(ethtool -k "$network" | grep -w "${kmap["${para[2]}"]}" | awk '{print $2}')
+    property=$(ethtool -k "$network" | grep -w "${kmap["${para[2]}"]}")
+    value=$(echo "$property" | awk '{print $2}')
     if !(judge_value "$value" $koption); then
       echo "\033[31m the last parameter of ethtool must select in [$koption] \033[31m"
       return 1
     fi
-    echo "-K $network "${para[2]}" $value"
+    affix=$(echo "$property" | awk '{print $3}')
+    if [[ "$affix" == "[fixed]" ]]; then
+      echo "fixed"
+    else
+      echo "-K $network "${para[2]}" $value"
+    fi
     ;;
   "-X")
     for opt in $xoption; do
@@ -65,6 +73,14 @@ function get_ethtool_value() {
     fi
     echo "-C $network "${para[2]}" $value"
     ;;
+  "-L")
+    value=$(ethtool -l "$network" | grep -w "${lmap["${para[2]}"]}" | awk 'NR==2{print $2}')
+    echo "-L $network "${para[2]}" $value"
+    ;;
+  "-G")
+    value=$(ethtool -g "$network" | grep -w "${gmap["${para[2]}"]}" | awk 'NR==2{print $2}')
+    echo "-G $network "${para[2]}" $value"
+    ;;
   *)
     echo "\033[31m this option is not supported \033[31m" && return 1
     ;;
@@ -86,3 +102,20 @@ function judge_value() {
   false
 }
 
+function cmd_conversion() {
+  option=$(echo "$@" | awk '{print $1}')
+  network=$(echo "$@" | awk '{print $2}')
+  case "$option" in
+  "-L")
+    if [[ $(echo "$@" | awk '{print $4}') == "max" ]]; then
+      max_value=$(ethtool -l "$network" | grep -w "Combined" | awk 'NR==1{print $2}')
+      echo "$@" | sed "s/max/$max_value/"
+    else
+      echo "$@"
+    fi
+    ;;
+  *)
+    echo "$@"
+    ;;
+  esac
+}

scripts/ethtool/set.sh

@@ -20,11 +20,12 @@ command -v ethtool >/dev/null 2>&1
 ret=$?
 [ $ret -ne 0 ] && echo "\033[31m command ethtool is not exist \033[31m" && exit 1
 
+fixed="fixed"
+para=$(cmd_conversion "$@")
 value=$(get_ethtool_value "$@")
-if [[ "$value" == $(eval echo "$*") ]]; then
+if [[ "$value" == "$para" ]] || [[ "$value" == "$fixed" ]] || [[ "$para" == "$fixed" ]]; then
   echo "no need to set"
   exit 0
 fi
 
-ethtool "$@"
-
+ethtool $(eval echo "$para")

scripts/hinic/get.sh

+#!/bin/sh
+# Copyright (c) 2020 Huawei Technologies Co., Ltd.
+# A-Tune is licensed under the Mulan PSL v2.
+# You can use this software according to the terms and conditions of the Mulan PSL v2.
+# You may obtain a copy of Mulan PSL v2 at:
+#     http://license.coscl.org.cn/MulanPSL2
+# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+# PURPOSE.
+# See the Mulan PSL v2 for more details.
+# Create: 2020-02-24
+
+lsmod | grep hinic &>/dev/null
+ret=$?
+[ $ret -ne 0 ] && echo "disable" && exit 0
+
+value=$(cat /sys/bus/pci/drivers/hinic/module/parameters/rx_buff)
+echo "$value"

scripts/hinic/set.sh

+#!/bin/sh
+# Copyright (c) 2020 Huawei Technologies Co., Ltd.
+# A-Tune is licensed under the Mulan PSL v2.
+# You can use this software according to the terms and conditions of the Mulan PSL v2.
+# You may obtain a copy of Mulan PSL v2 at:
+#     http://license.coscl.org.cn/MulanPSL2
+# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+# PURPOSE.
+# See the Mulan PSL v2 for more details.
+# Create: 2020-02-24
+
+SCRIPT=$(basename "$0")
+if [ $# != 1 ]; then
+  echo "Usage: ${SCRIPT} number or disable"
+  exit 1
+fi
+
+path="/etc/modprobe.d/hinic.conf"
+case "$1" in
+"disable")
+  lsmod | grep hinic &>/dev/null
+  ret=$?
+  [ $ret -eq 0 ] && rmmod hinic
+  exit 0
+  ;;
+"2" | "4" | "8" | "16")
+  [ ! -f "$path" ] && touch "$path"
+  sed -i '/options hinic rx_buff=/d' "$path"
+  echo "options hinic rx_buff=$1" >>"$path"
+  lsmod | grep hinic &>/dev/null
+  ret=$?
+  [ $ret -eq 0 ] && rmmod hinic
+  modprobe hinic
+  ;;
+*)
+  echo "\033[31m this option is not supported \033[31m" && exit 1
+  ;;
+esac

scripts/logind/get.sh

+#!/bin/sh
+# Copyright (c) 2020 Huawei Technologies Co., Ltd.
+# A-Tune is licensed under the Mulan PSL v2.
+# You can use this software according to the terms and conditions of the Mulan PSL v2.
+# You may obtain a copy of Mulan PSL v2 at:
+#     http://license.coscl.org.cn/MulanPSL2
+# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+# PURPOSE.
+# See the Mulan PSL v2 for more details.
+# Create: 2020-03-02
+
+declare -A default=(["UserTasksMax"]="4096")
+
+keywords=$(echo "$@" | awk '{print $1}')
+value=$(echo "$@" | awk '{print $2}')
+result=$(cat /etc/systemd/logind.conf | grep -w "^$keywords" | awk -F "=" '{print$2}')
+if [[ "$result" == "" ]]; then
+  echo "$keywords ${default["$keywords"]}"
+else
+  echo "$keywords $result"
+fi

scripts/logind/set.sh

+#!/bin/sh
+# Copyright (c) 2020 Huawei Technologies Co., Ltd.
+# A-Tune is licensed under the Mulan PSL v2.
+# You can use this software according to the terms and conditions of the Mulan PSL v2.
+# You may obtain a copy of Mulan PSL v2 at:
+#     http://license.coscl.org.cn/MulanPSL2
+# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+# PURPOSE.
+# See the Mulan PSL v2 for more details.
+# Create: 2020-03-02
+
+keywords=$(echo "$@" | awk '{print $1}')
+value=$(echo "$@" | awk '{print $2}')
+file=/etc/systemd/logind.conf
+
+sed -i "/$keywords=/d" "$file"
+sed -i "/\[Login\]/a$keywords=$value" "$file"