alinux: mm, memcg: record latency of direct compact in every memcg

to #26424368

Probe and calculate the latency of direct compact, and then group into
the latency histogram in struct mem_cgroup.

Note that the latency in each memcg is aggregated from all child memcgs.

Usage:

$ cat memory.direct_compact_latency
0-1ms:  1176
1-5ms:  259
5-10ms:         17
10-100ms:       10
100-500ms:      0
500-1000ms:     0
>=1000ms:       0
total(ms):      921

Each line is the count of direct compact within the appropriate latency
range.

To clear the latency histogram:

$ echo 0 > memory.direct_compact_latency
$ cat memory.direct_compact_latency
0-1ms:  0
1-5ms:  0
5-10ms:         0
10-100ms:       0
100-500ms:      0
500-1000ms:     0
>=1000ms:       0
total(ms):      0
Signed-off-by: NXu Yu <xuyu@linux.alibaba.com>
Reviewed-by: NXunlei Pang <xlpang@linux.alibaba.com>

include/linux/memcontrol.h

@@ -91,6 +91,7 @@ struct alloc_context;
 enum mem_lat_stat_item {
 	GLOBAL_DIRECT_RECLAIM,      /* global direct reclaim latency */
 	MEMCG_DIRECT_RECLAIM,       /* memcg direct reclaim latency */
+	DIRECT_COMPACT,             /* direct compact latency */
 	MEM_LAT_NR_STAT,
 };
 

mm/memcontrol.c

@@ -4531,9 +4531,20 @@ static void smp_memcg_direct_reclaim_write(void *info)
 		this_cpu_write(memcg->lat_stat_cpu->item[idx][i], 0);
 }
 
+static void smp_direct_compact_write(void *info)
+{
+	struct mem_cgroup *memcg = (struct mem_cgroup *)info;
+	int idx = DIRECT_COMPACT;
+	int i;
+
+	for (i = MEM_LAT_0_1; i < MEM_LAT_NR_COUNT; i++)
+		this_cpu_write(memcg->lat_stat_cpu->item[idx][i], 0);
+}
+
 smp_call_func_t smp_memcg_lat_write_funcs[] = {
 	smp_global_direct_reclaim_write,
 	smp_memcg_direct_reclaim_write,
+	smp_direct_compact_write,
 };
 
 static int memcg_lat_stat_write(struct cgroup_subsys_state *css,
@@ -4566,6 +4577,16 @@ static int memcg_direct_reclaim_latency_write(struct cgroup_subsys_state *css,
 	return memcg_lat_stat_write(css, MEMCG_DIRECT_RECLAIM);
 }
 
+static int memcg_direct_compact_latency_write(
+			struct cgroup_subsys_state *css,
+			struct cftype *cft, u64 val)
+{
+	if (val != 0)
+		return -EINVAL;
+
+	return memcg_lat_stat_write(css, DIRECT_COMPACT);
+}
+
 static u64 memcg_lat_stat_gather(struct mem_cgroup *memcg,
 				 enum mem_lat_stat_item sidx,
 				 enum mem_lat_count_t cidx)
@@ -4615,6 +4636,13 @@ static int memcg_direct_reclaim_latency_show(struct seq_file *m, void *v)
 	return 0;
 }
 
+static int memcg_direct_compact_latency_show(struct seq_file *m, void *v)
+{
+	memcg_lat_stat_show(m, DIRECT_COMPACT);
+
+	return 0;
+}
+
 enum mem_lat_count_t get_mem_lat_count_idx(u64 duration)
 {
 	enum mem_lat_count_t idx;
@@ -5566,6 +5594,11 @@ static struct cftype mem_cgroup_legacy_files[] = {
 		.write_u64 = memcg_direct_reclaim_latency_write,
 		.seq_show =  memcg_direct_reclaim_latency_show,
 	},
+	{
+		.name = "direct_compact_latency",
+		.write_u64 = memcg_direct_compact_latency_write,
+		.seq_show =  memcg_direct_compact_latency_show,
+	},
 	{
 		.name = "force_empty",
 		.write = mem_cgroup_force_empty_write,

mm/page_alloc.c

@@ -3720,17 +3720,20 @@ __alloc_pages_direct_compact(gfp_t gfp_mask, unsigned int order,
 	struct page *page;
 	unsigned long pflags;
 	unsigned int noreclaim_flag;
+	u64 start;
 
 	if (!order)
 		return NULL;
 
 	psi_memstall_enter(&pflags);
+	start = ktime_get_ns();
 	noreclaim_flag = memalloc_noreclaim_save();
 
 	*compact_result = try_to_compact_pages(gfp_mask, order, alloc_flags, ac,
 									prio);
 
 	memalloc_noreclaim_restore(noreclaim_flag);
+	memcg_lat_stat_update(DIRECT_COMPACT, (ktime_get_ns() - start));
 	psi_memstall_leave(&pflags);
 
 	if (*compact_result <= COMPACT_INACTIVE)