!163 atune: add spark scenarios

Merge pull request !163 from hanxinke/master

common/schedule/schedule.go

@@ -154,6 +154,7 @@ func (s *Scheduler) Active(ch chan *PB.AckCheck, itemKeys []string, items map[st
 				for _, key := range section.Keys() {
 					description := key.Name()
 					sendChanToAdm(ch, key.Value(), utils.SUGGEST, description)
+					log.Infof("tip key name: %s, key value: %s", key.Name(), key.Value())
 				}
 				continue
 			}

database/init.sql

@@ -192,6 +192,7 @@ INSERT INTO tuned_item(property, item) VALUES("selinux", "Bootloader");
 INSERT INTO tuned_item(property, item) VALUES("skew_tick", "Bootloader");
 INSERT INTO tuned_item(property, item) VALUES("numa_spinlock", "Bootloader");
 INSERT INTO tuned_item(property, item) VALUES("scsi_mod.use_blk_mq", "Bootloader");
+INSERT INTO tuned_item(property, item) VALUES("sched_steal_node_limit", "Bootloader");
 
 INSERT INTO tuned_item(property, item) VALUES("openssl_hpre", "Script");
 INSERT INTO tuned_item(property, item) VALUES("hinic", "Script");
@@ -205,6 +206,11 @@ INSERT INTO tuned_item(property, item) VALUES("Support SPCR", "Bios");
 INSERT INTO tuned_item(property, item) VALUES("Max Payload Size", "Bios");
 INSERT INTO tuned_item(property, item) VALUES("Power Policy", "Bios");
 INSERT INTO tuned_item(property, item) VALUES("Rank Interleaving", "Bios");
+INSERT INTO tuned_item(property, item) VALUES("Read Policy", "Bios");
+INSERT INTO tuned_item(property, item) VALUES("Write Policy", "Bios");
+INSERT INTO tuned_item(property, item) VALUES("I/O Policy", "Bios");
+INSERT INTO tuned_item(property, item) VALUES("Access Policy", "Bios");
+INSERT INTO tuned_item(property, item) VALUES("Drive Cache", "Bios");
 
 INSERT INTO tuned_item(property, item) VALUES("vm.nr_hugepages", "Sysctl");
 INSERT INTO tuned_item(property, item) VALUES("vm.swappiness", "Sysctl");

modules/server/profile/profile.go

@@ -282,6 +282,7 @@ func (s *ProfileServer) Profile(profileInfo *PB.ProfileInfo, stream PB.ProfileMg
 		return err
 	}
 
+	time.Sleep(1 * time.Second)
 	return nil
 }
 

profiles/big-data/hadoop-spark/hibench.conf → profiles/big-data/hadoop-spark/bayesian.conf

@@ -7,12 +7,12 @@
 # IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
 # PURPOSE.
 # See the Mulan PSL v2 for more details.
-# Create: 2020-06-10
+# Create: 2020-08-27
 #
-# spark hibench A-Tune configuration
+# spark bayesian A-Tune configuration
 #
 [main]
-include = include-io-intensive
+include = include-spark-common
 
 [kernel_config]
 #TODO CONFIG
@@ -21,31 +21,22 @@ include = include-io-intensive
 #TODO CONFIG
 
 [bootloader.grub2]
-iommu.passthrough = 1
-iommu.strict = 0
+#TODO CONFIG
 
 [sysfs]
-block/{disk}/device/queue_depth = 256
+#TODO CONFIG
 
 [systemctl]
-firewalld = stop
-sysmonitor = stop
-irqbalance = stop
+#TODO CONFIG
 
 [sysctl]
-fs.file-max = 1000000
-fs.nr_open = 2000000
-kernel.shmmax = 68719476736
-kernel.shmall = 4294967296
-vm.zone_reclaim_mode = 0
+#TODO CONFIG
 
 [script]
-ethtool = -K {network} lro on | -G {network} rx 4096 | -G {network} tx 4096
-hinic = 8
+#TODO CONFIG
 
 [ulimit]
-{user}.hard.nofile = 2000000
-{user}.soft.nofile = 1800000
+#TODO CONFIG
 
 [schedule_policy]
 #TODO CONFIG
@@ -54,4 +45,4 @@ hinic = 8
 #TODO CONFIG
 
 [tip]
-#TODO CONFIG
+set --num-executors to 15, --executor-cores to 19, --executor-memory to 44G and --driver-memory to 36G = application

profiles/big-data/hadoop-spark/kmeans.conf

+# Copyright (c) 2020 Huawei Technologies Co., Ltd.
+# A-Tune is licensed under the Mulan PSL v2.
+# You can use this software according to the terms and conditions of the Mulan PSL v2.
+# You may obtain a copy of Mulan PSL v2 at:
+#     http://license.coscl.org.cn/MulanPSL2
+# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+# PURPOSE.
+# See the Mulan PSL v2 for more details.
+# Create: 2020-08-27
+#
+# spark kmeans A-Tune configuration
+#
+[main]
+include = include-spark-common
+
+[kernel_config]
+#TODO CONFIG
+
+[bios]
+#TODO CONFIG
+
+[bootloader.grub2]
+#TODO CONFIG
+
+[sysfs]
+block/{disk}/queue/read_ahead_kb = 4096
+
+[systemctl]
+#TODO CONFIG
+
+[sysctl]
+#TODO CONFIG
+
+[script]
+#TODO CONFIG
+
+[ulimit]
+#TODO CONFIG
+
+[schedule_policy]
+#TODO CONFIG
+
+[check]
+#TODO CONFIG
+
+[tip]
+set --num-executors to 42, --executor-cores to 6, --executor-memory to 15G and --driver-memory to 36G = application
+set hibench.default.map.parallelism to 1000 and hibench.default.shuffle.parallelism to 2500 = application

profiles/big-data/hadoop-spark/sql1.conf

+# Copyright (c) 2020 Huawei Technologies Co., Ltd.
+# A-Tune is licensed under the Mulan PSL v2.
+# You can use this software according to the terms and conditions of the Mulan PSL v2.
+# You may obtain a copy of Mulan PSL v2 at:
+#     http://license.coscl.org.cn/MulanPSL2
+# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+# PURPOSE.
+# See the Mulan PSL v2 for more details.
+# Create: 2020-08-27
+#
+# spark sql1 A-Tune configuration
+#
+[main]
+include = include-spark-common
+
+[kernel_config]
+#TODO CONFIG
+
+[bios]
+#TODO CONFIG
+
+[bootloader.grub2]
+#TODO CONFIG
+
+[sysfs]
+block/{disk}/queue/read_ahead_kb = 512
+block/{disk}/queue/nr_requests = 128
+
+[systemctl]
+#TODO CONFIG
+
+[sysctl]
+vm.dirty_expire_centisecs = 500
+vm.dirty_writeback_centisecs = 100
+
+[script]
+#TODO CONFIG
+
+[ulimit]
+#TODO CONFIG
+
+[schedule_policy]
+#TODO CONFIG
+
+[check]
+#TODO CONFIG
+
+[tip]
+set --num-executors to 15, --executor-cores to 19, --executor-memory to 44G and --driver-memory to 36G = application

profiles/big-data/hadoop-spark/sql10.conf

+# Copyright (c) 2020 Huawei Technologies Co., Ltd.
+# A-Tune is licensed under the Mulan PSL v2.
+# You can use this software according to the terms and conditions of the Mulan PSL v2.
+# You may obtain a copy of Mulan PSL v2 at:
+#     http://license.coscl.org.cn/MulanPSL2
+# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+# PURPOSE.
+# See the Mulan PSL v2 for more details.
+# Create: 2020-08-27
+#
+# spark sql10 A-Tune configuration
+#
+[main]
+include = include-spark-common
+
+[kernel_config]
+#TODO CONFIG
+
+[bios]
+#TODO CONFIG
+
+[bootloader.grub2]
+#TODO CONFIG
+
+[sysfs]
+#TODO CONFIG
+
+[systemctl]
+#TODO CONFIG
+
+[sysctl]
+#TODO CONFIG
+
+[script]
+#TODO CONFIG
+
+[ulimit]
+#TODO CONFIG
+
+[schedule_policy]
+#TODO CONFIG
+
+[check]
+#TODO CONFIG
+
+[tip]
+set --num-executors to 15, --executor-cores to 19, --executor-memory to 44G and --driver-memory to 36G = application

profiles/big-data/hadoop-spark/sql2.conf

+# Copyright (c) 2020 Huawei Technologies Co., Ltd.
+# A-Tune is licensed under the Mulan PSL v2.
+# You can use this software according to the terms and conditions of the Mulan PSL v2.
+# You may obtain a copy of Mulan PSL v2 at:
+#     http://license.coscl.org.cn/MulanPSL2
+# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+# PURPOSE.
+# See the Mulan PSL v2 for more details.
+# Create: 2020-08-27
+#
+# spark sql2 A-Tune configuration
+#
+[main]
+include = include-spark-common
+
+[kernel_config]
+#TODO CONFIG
+
+[bios]
+#TODO CONFIG
+
+[bootloader.grub2]
+#TODO CONFIG
+
+[sysfs]
+#TODO CONFIG
+
+[systemctl]
+#TODO CONFIG
+
+[sysctl]
+#TODO CONFIG
+
+[script]
+#TODO CONFIG
+
+[ulimit]
+#TODO CONFIG
+
+[schedule_policy]
+#TODO CONFIG
+
+[check]
+#TODO CONFIG
+
+[tip]
+set --num-executors to 42, --executor-cores to 6, --executor-memory to 15G and --driver-memory to 36G = application

profiles/big-data/hadoop-spark/sql3.conf

+# Copyright (c) 2020 Huawei Technologies Co., Ltd.
+# A-Tune is licensed under the Mulan PSL v2.
+# You can use this software according to the terms and conditions of the Mulan PSL v2.
+# You may obtain a copy of Mulan PSL v2 at:
+#     http://license.coscl.org.cn/MulanPSL2
+# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+# PURPOSE.
+# See the Mulan PSL v2 for more details.
+# Create: 2020-08-27
+#
+# spark sql3 A-Tune configuration
+#
+[main]
+include = include-spark-common
+
+[kernel_config]
+#TODO CONFIG
+
+[bios]
+#TODO CONFIG
+
+[bootloader.grub2]
+#TODO CONFIG
+
+[sysfs]
+block/{disk}/queue/read_ahead_kb = 4096
+
+[systemctl]
+#TODO CONFIG
+
+[sysctl]
+#TODO CONFIG
+
+[script]
+#TODO CONFIG
+
+[ulimit]
+#TODO CONFIG
+
+[schedule_policy]
+#TODO CONFIG
+
+[check]
+#TODO CONFIG
+
+[tip]
+set --num-executors to 30, --executor-cores to 6, --executor-memory to 24G and --driver-memory to 36G = application

profiles/big-data/hadoop-spark/sql4.conf

+# Copyright (c) 2020 Huawei Technologies Co., Ltd.
+# A-Tune is licensed under the Mulan PSL v2.
+# You can use this software according to the terms and conditions of the Mulan PSL v2.
+# You may obtain a copy of Mulan PSL v2 at:
+#     http://license.coscl.org.cn/MulanPSL2
+# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+# PURPOSE.
+# See the Mulan PSL v2 for more details.
+# Create: 2020-08-27
+#
+# spark sql4 A-Tune configuration
+#
+[main]
+include = include-spark-common
+
+[kernel_config]
+#TODO CONFIG
+
+[bios]
+#TODO CONFIG
+
+[bootloader.grub2]
+#TODO CONFIG
+
+[sysfs]
+block/{disk}/queue/read_ahead_kb = 4096
+
+[systemctl]
+#TODO CONFIG
+
+[sysctl]
+#TODO CONFIG
+
+[script]
+#TODO CONFIG
+
+[ulimit]
+#TODO CONFIG
+
+[schedule_policy]
+#TODO CONFIG
+
+[check]
+#TODO CONFIG
+
+[tip]
+set --num-executors to 42, --executor-cores to 6, --executor-memory to 15G and --driver-memory to 36G = application

profiles/big-data/hadoop-spark/sql5.conf

+# Copyright (c) 2020 Huawei Technologies Co., Ltd.
+# A-Tune is licensed under the Mulan PSL v2.
+# You can use this software according to the terms and conditions of the Mulan PSL v2.
+# You may obtain a copy of Mulan PSL v2 at:
+#     http://license.coscl.org.cn/MulanPSL2
+# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+# PURPOSE.
+# See the Mulan PSL v2 for more details.
+# Create: 2020-08-27
+#
+# spark sql5 A-Tune configuration
+#
+[main]
+include = include-spark-common
+
+[kernel_config]
+#TODO CONFIG
+
+[bios]
+#TODO CONFIG
+
+[bootloader.grub2]
+#TODO CONFIG
+
+[sysfs]
+#TODO CONFIG
+
+[systemctl]
+#TODO CONFIG
+
+[sysctl]
+#TODO CONFIG
+
+[script]
+#TODO CONFIG
+
+[ulimit]
+#TODO CONFIG
+
+[schedule_policy]
+#TODO CONFIG
+
+[check]
+#TODO CONFIG
+
+[tip]
+set --num-executors to 15, --executor-cores to 19, --executor-memory to 44G and --driver-memory to 36G = application

profiles/big-data/hadoop-spark/sql6.conf

+# Copyright (c) 2020 Huawei Technologies Co., Ltd.
+# A-Tune is licensed under the Mulan PSL v2.
+# You can use this software according to the terms and conditions of the Mulan PSL v2.
+# You may obtain a copy of Mulan PSL v2 at:
+#     http://license.coscl.org.cn/MulanPSL2
+# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+# PURPOSE.
+# See the Mulan PSL v2 for more details.
+# Create: 2020-08-27
+#
+# spark sql6 A-Tune configuration
+#
+[main]
+include = include-spark-common
+
+[kernel_config]
+#TODO CONFIG
+
+[bios]
+#TODO CONFIG
+
+[bootloader.grub2]
+#TODO CONFIG
+
+[sysfs]
+#TODO CONFIG
+
+[systemctl]
+#TODO CONFIG
+
+[sysctl]
+#TODO CONFIG
+
+[script]
+#TODO CONFIG
+
+[ulimit]
+#TODO CONFIG
+
+[schedule_policy]
+#TODO CONFIG
+
+[check]
+#TODO CONFIG
+
+[tip]
+set --num-executors to 15, --executor-cores to 19, --executor-memory to 44G and --driver-memory to 36G = application

profiles/big-data/hadoop-spark/sql7.conf

+# Copyright (c) 2020 Huawei Technologies Co., Ltd.
+# A-Tune is licensed under the Mulan PSL v2.
+# You can use this software according to the terms and conditions of the Mulan PSL v2.
+# You may obtain a copy of Mulan PSL v2 at:
+#     http://license.coscl.org.cn/MulanPSL2
+# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+# PURPOSE.
+# See the Mulan PSL v2 for more details.
+# Create: 2020-08-27
+#
+# spark sql7 A-Tune configuration
+#
+[main]
+include = include-spark-common
+
+[kernel_config]
+#TODO CONFIG
+
+[bios]
+#TODO CONFIG
+
+[bootloader.grub2]
+#TODO CONFIG
+
+[sysfs]
+#TODO CONFIG
+
+[systemctl]
+#TODO CONFIG
+
+[sysctl]
+#TODO CONFIG
+
+[script]
+#TODO CONFIG
+
+[ulimit]
+#TODO CONFIG
+
+[schedule_policy]
+#TODO CONFIG
+
+[check]
+#TODO CONFIG
+
+[tip]
+set --num-executors to 42, --executor-cores to 6, --executor-memory to 15G and --driver-memory to 36G = application

profiles/big-data/hadoop-spark/sql8.conf

+# Copyright (c) 2020 Huawei Technologies Co., Ltd.
+# A-Tune is licensed under the Mulan PSL v2.
+# You can use this software according to the terms and conditions of the Mulan PSL v2.
+# You may obtain a copy of Mulan PSL v2 at:
+#     http://license.coscl.org.cn/MulanPSL2
+# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+# PURPOSE.
+# See the Mulan PSL v2 for more details.
+# Create: 2020-08-27
+#
+# spark sql8 A-Tune configuration
+#
+[main]
+include = include-spark-common
+
+[kernel_config]
+#TODO CONFIG
+
+[bios]
+#TODO CONFIG
+
+[bootloader.grub2]
+#TODO CONFIG
+
+[sysfs]
+#TODO CONFIG
+
+[systemctl]
+#TODO CONFIG
+
+[sysctl]
+#TODO CONFIG
+
+[script]
+#TODO CONFIG
+
+[ulimit]
+#TODO CONFIG
+
+[schedule_policy]
+#TODO CONFIG
+
+[check]
+#TODO CONFIG
+
+[tip]
+set --num-executors to 15, --executor-cores to 19, --executor-memory to 44G and --driver-memory to 36G = application

profiles/big-data/hadoop-spark/sql9.conf

+# Copyright (c) 2020 Huawei Technologies Co., Ltd.
+# A-Tune is licensed under the Mulan PSL v2.
+# You can use this software according to the terms and conditions of the Mulan PSL v2.
+# You may obtain a copy of Mulan PSL v2 at:
+#     http://license.coscl.org.cn/MulanPSL2
+# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+# PURPOSE.
+# See the Mulan PSL v2 for more details.
+# Create: 2020-08-27
+#
+# spark sql9 A-Tune configuration
+#
+[main]
+include = include-spark-common
+
+[kernel_config]
+#TODO CONFIG
+
+[bios]
+#TODO CONFIG
+
+[bootloader.grub2]
+#TODO CONFIG
+
+[sysfs]
+#TODO CONFIG
+
+[systemctl]
+#TODO CONFIG
+
+[sysctl]
+#TODO CONFIG
+
+[script]
+#TODO CONFIG
+
+[ulimit]
+#TODO CONFIG
+
+[schedule_policy]
+#TODO CONFIG
+
+[check]
+#TODO CONFIG
+
+[tip]
+set --num-executors to 15, --executor-cores to 19, --executor-memory to 44G and --driver-memory to 36G = application

profiles/big-data/hadoop-spark/tersort.conf

+# Copyright (c) 2020 Huawei Technologies Co., Ltd.
+# A-Tune is licensed under the Mulan PSL v2.
+# You can use this software according to the terms and conditions of the Mulan PSL v2.
+# You may obtain a copy of Mulan PSL v2 at:
+#     http://license.coscl.org.cn/MulanPSL2
+# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+# PURPOSE.
+# See the Mulan PSL v2 for more details.
+# Create: 2020-08-27
+#
+# spark tersort A-Tune configuration
+#
+[main]
+include = include-spark-common
+
+[kernel_config]
+#TODO CONFIG
+
+[bios]
+#TODO CONFIG
+
+[bootloader.grub2]
+#TODO CONFIG
+
+[sysfs]
+block/{disk}/queue/scheduler = cfq
+block/{disk}/queue/read_ahead_kb = 8192
+block/{disk}/queue/nr_requests = 512
+block/{disk}/queue/iosched/slice_idle = 4
+
+[systemctl]
+#TODO CONFIG
+
+[sysctl]
+vm.dirty_expire_centisecs = 500
+vm.dirty_writeback_centisecs = 100
+
+[script]
+#TODO CONFIG
+
+[ulimit]
+#TODO CONFIG
+
+[schedule_policy]
+#TODO CONFIG
+
+[check]
+#TODO CONFIG
+
+[tip]
+set --num-executors to 27, --executor-cores to 7, --executor-memory to 25G and --driver-memory to 36G = application
+set hibench.default.map.parallelism to 1000 and hibench.default.shuffle.parallelism to 2000 = application

profiles/big-data/hadoop-spark/wordcount.conf

+# Copyright (c) 2020 Huawei Technologies Co., Ltd.
+# A-Tune is licensed under the Mulan PSL v2.
+# You can use this software according to the terms and conditions of the Mulan PSL v2.
+# You may obtain a copy of Mulan PSL v2 at:
+#     http://license.coscl.org.cn/MulanPSL2
+# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+# PURPOSE.
+# See the Mulan PSL v2 for more details.
+# Create: 2020-08-27
+#
+# spark wordcount A-Tune configuration
+#
+[main]
+include = include-spark-common
+
+[kernel_config]
+#TODO CONFIG
+
+[bios]
+#TODO CONFIG
+
+[bootloader.grub2]
+#TODO CONFIG
+
+[sysfs]
+block/{disk}/queue/scheduler = mq-deadline
+block/{disk}/queue/read_ahead_kb = 8192
+block/{disk}/queue/nr_requests = 512
+
+[systemctl]
+#TODO CONFIG
+
+[sysctl]
+vm.dirty_expire_centisecs = 500
+vm.dirty_writeback_centisecs = 100
+vm.dirty_ratio = 60
+vm.dirty_background_ratio = 5
+
+[script]
+#TODO CONFIG
+
+[ulimit]
+#TODO CONFIG
+
+[schedule_policy]
+#TODO CONFIG
+
+[check]
+#TODO CONFIG
+
+[tip]
+set --num-executors to 51, --executor-cores to 6, --executor-memory to 13G and --driver-memory to 36G = application
+set hibench.default.map.parallelism to 300 and hibench.default.shuffle.parallelism to 600 = application

profiles/include/mysql-2p-sysbench.conf

@@ -23,6 +23,7 @@ include = default-default
 [bootloader.grub2]
 iommu.passthrough = 1
 iommu.strict = 0
+sched_steal_node_limit = 4
 
 [sysfs]
 block/{disk}/queue/read_ahead_kb = 64
@@ -30,6 +31,7 @@ block/{disk}/queue/nr_requests = 2048
 block/{disk}/device/queue_depth = 256
 kernel/mm/transparent_hugepage/defrag = never
 kernel/mm/transparent_hugepage/enabled = never
+kernel/debug/sched_features = NO_WAKEUP_PREEMPTION
 
 [systemctl]
 sysmonitor = stop
@@ -46,6 +48,8 @@ kernel.sched_latency_ns = 16110000
 kernel.sched_min_granularity_ns = 8250000
 kernel.sched_nr_migrate = 53
 kernel.sched_wakeup_granularity_ns = 50410000
+kernel.sched_autogroup_enabled = 0
+kernel.numa_balancing = 0
 
 # network core
 net.core.rmem_default = 21299200

profiles/include/spark-common.conf

+# Copyright (c) 2020 Huawei Technologies Co., Ltd.
+# A-Tune is licensed under the Mulan PSL v2.
+# You can use this software according to the terms and conditions of the Mulan PSL v2.
+# You may obtain a copy of Mulan PSL v2 at:
+#     http://license.coscl.org.cn/MulanPSL2
+# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+# PURPOSE.
+# See the Mulan PSL v2 for more details.
+# Create: 2020-08-27
+#
+# spark common A-Tune configuration
+#
+[main]
+include = default-default
+
+[kernel_config]
+#TODO CONFIG
+
+[bios]
+Read Policy = Read Ahead
+Write Policy = Write Through
+I/O Policy = Direct
+Access Policy = Read/Write
+Drive Cache = Unchanged
+
+[bootloader.grub2]
+#TODO CONFIG
+
+[sysfs]
+kernel/mm/transparent_hugepage/defrag = never
+kernel/mm/transparent_hugepage/enabled = never
+block/{disk}/queue/read_ahead_kb = 2048
+block/{disk}/device/queue_depth = 32
+block/{disk}/queue/max_sectors_kb = 1024
+block/{disk}/queue/nr_requests = 256
+block/{disk}/queue/scheduler = mq-deadline
+block/{disk}/queue/rq_affinity = 2
+block/{disk}/queue/nomerges = 0
+
+[systemctl]
+#TODO CONFIG
+
+[sysctl]
+vm.dirty_expire_centisecs = 3000
+vm.dirty_writeback_centisecs = 500
+vm.min_free_kbytes = 11264
+vm.dirty_ratio = 60
+vm.dirty_background_ratio = 5
+kernel.sched_min_granularity_ns = 10000000
+kernel.sched_wakeup_granularity_ns = 15000000
+kernel.sched_autogroup_enabled = 0
+kernel.numa_balancing = 0
+
+[script]
+#TODO CONFIG
+
+[ulimit]
+#TODO CONFIG
+
+[schedule_policy]
+#TODO CONFIG
+
+[check]
+#TODO CONFIG
+
+[tip]
+enable BGI in raid configuration = block