alinux: sched: Add cgroup-level blocked time histograms

to #28739709

This patch measures time that tasks in cpuacct cgroup blocks. There
are two types: blocked due to IO, and others like locks. And they
are exported in"cpuacct.ioblock_latency" and "cpuacct.block_latency"
respectively.

According to histogram, we know the detailed distribution of the
duration. And according to total(ms), we know the percentage of time
tasks spent off rq, waiting for resources:

(△ioblock_latency.total(ms) + △block_latency.total(ms)) / △wall_time

The interface output format is identical to cpuacct.wait_latency.
Signed-off-by: NYihao Wu <wuyihao@linux.alibaba.com>
Acked-by: NXunlei Pang <xlpang@linux.alibaba.com>
Reviewed-by: NShanpei Chen <shanpeic@linux.alibaba.com>
Acked-by: NMichael Wang <yun.wang@linux.alibaba.com>

kernel/sched/cpuacct.c

@@ -35,6 +35,8 @@ struct cpuacct_alistats {
 
 enum sched_lat_stat_item {
 	SCHED_LAT_WAIT,
+	SCHED_LAT_BLOCK,
+	SCHED_LAT_IOBLOCK,
 	SCHED_LAT_NR_STAT
 };
 
@@ -229,6 +231,30 @@ void task_ca_increase_nr_migrations(struct task_struct *tsk)
 	rcu_read_unlock();
 }
 
+void task_ca_update_block(struct task_struct *tsk, u64 runtime)
+{
+	int idx;
+	enum sched_lat_stat_item s;
+	struct cpuacct *ca;
+	unsigned int msecs;
+
+	if (static_branch_likely(&cpuacct_no_sched_lat))
+		return;
+
+	rcu_read_lock();
+	ca = task_ca(tsk);
+	if (tsk->in_iowait)
+		s = SCHED_LAT_IOBLOCK;
+	else
+		s = SCHED_LAT_BLOCK;
+
+	msecs = runtime >> 20; /* Proximately to speed up */
+	idx = get_sched_lat_count_idx(msecs);
+	this_cpu_inc(ca->lat_stat_cpu->item[s][idx]);
+	this_cpu_add(ca->lat_stat_cpu->item[s][SCHED_LAT_TOTAL], runtime);
+	rcu_read_unlock();
+}
+
 void cpuacct_update_latency(struct task_struct *tsk, u64 delta)
 {
 	enum sched_lat_count_t idx;
@@ -780,9 +806,13 @@ static void smp_write_##name(void *info)				\
 }									\
 
 SCHED_LAT_STAT_SMP_WRITE(sched_wait_latency, SCHED_LAT_WAIT);
+SCHED_LAT_STAT_SMP_WRITE(sched_block_latency, SCHED_LAT_BLOCK);
+SCHED_LAT_STAT_SMP_WRITE(sched_ioblock_latency, SCHED_LAT_IOBLOCK);
 
 smp_call_func_t smp_sched_lat_write_funcs[] = {
-	smp_write_sched_wait_latency
+	smp_write_sched_wait_latency,
+	smp_write_sched_block_latency,
+	smp_write_sched_ioblock_latency
 };
 
 static int sched_lat_stat_write(struct cgroup_subsys_state *css,
@@ -892,6 +922,18 @@ static struct cftype files[] = {
 		.write_u64 = sched_lat_stat_write,
 		.seq_show = sched_lat_stat_show
 	},
+	{
+		.name = "block_latency",
+		.private = SCHED_LAT_BLOCK,
+		.write_u64 = sched_lat_stat_write,
+		.seq_show = sched_lat_stat_show
+	},
+	{
+		.name = "ioblock_latency",
+		.private = SCHED_LAT_IOBLOCK,
+		.write_u64 = sched_lat_stat_write,
+		.seq_show = sched_lat_stat_show
+	},
 #endif
 	{ }	/* terminate */
 };

kernel/sched/fair.c

@@ -945,6 +945,7 @@ update_stats_enqueue_sleeper(struct cfs_rq *cfs_rq, struct sched_entity *se)
 		__schedstat_add(se->statistics.sum_sleep_runtime, delta);
 
 		if (tsk) {
+			task_ca_update_block(tsk, delta);
 			if (tsk->in_iowait) {
 				__schedstat_add(se->statistics.iowait_sum, delta);
 				__schedstat_inc(se->statistics.iowait_count);

kernel/sched/sched.h

@@ -2283,10 +2283,13 @@ extern u64 get_idle_time(int cpu);
 extern u64 get_iowait_time(int cpu);
 extern void task_ca_increase_nr_migrations(struct task_struct *tsk);
 void cpuacct_update_latency(struct task_struct *tsk, u64 delta);
+void task_ca_update_block(struct task_struct *tsk, u64 runtime);
 #else
 static inline void task_ca_increase_nr_migrations(struct task_struct *tsk) { }
 static inline void cpuacct_update_latency(struct task_struct *tsk,
 		u64 delta) { }
+static inline void task_ca_update_block(struct task_struct *tsk,
+		u64 runtime) { }
 #endif
 
 #ifdef CONFIG_PSI