alinux: mm, memcg: record latency of swapout and swapin in every memcg

to #26424368

Probe and calculate the latency of global swapout, memcg swapout and
swapin respectively, and then group into the latency histogram in struct
mem_cgroup.

Note that the latency in each memcg is aggregated from all child memcgs.

Usage:

$ cat memory.direct_swapout_global_latency
0-1ms:  98313
1-5ms:  0
5-10ms:         0
10-100ms:       0
100-500ms:      0
500-1000ms:     0
>=1000ms:       0
total(ms):      52

Each line is the count of global swapout within the appropriate latency
range.

To clear the latency histogram:

$ echo 0 > memory.direct_swapout_global_latency
$ cat memory.direct_swapout_global_latency
0-1ms:  0
1-5ms:  0
5-10ms:         0
10-100ms:       0
100-500ms:      0
500-1000ms:     0
>=1000ms:       0
total(ms):      0

The usage of memory.direct_swapout_memcg_latency and
memory.direct_swapin_latency is the same as
memory.direct_swapout_global_latency.
Signed-off-by: NXu Yu <xuyu@linux.alibaba.com>
Reviewed-by: NYang Shi <yang.shi@linux.alibaba.com>
Reviewed-by: NXunlei Pang <xlpang@linux.alibaba.com>

include/linux/memcontrol.h

@@ -92,6 +92,9 @@ enum mem_lat_stat_item {
 	MEM_LAT_GLOBAL_DIRECT_RECLAIM,	/* global direct reclaim latency */
 	MEM_LAT_MEMCG_DIRECT_RECLAIM,	/* memcg direct reclaim latency */
 	MEM_LAT_DIRECT_COMPACT,		/* direct compact latency */
+	MEM_LAT_GLOBAL_DIRECT_SWAPOUT,	/* global direct swapout latency */
+	MEM_LAT_MEMCG_DIRECT_SWAPOUT,	/* memcg direct swapout latency */
+	MEM_LAT_DIRECT_SWAPIN,		/* direct swapin latency */
 	MEM_LAT_NR_STAT,
 };
 

mm/memcontrol.c

@@ -4532,11 +4532,17 @@ static void smp_write_##name(void *info)				\
 MEMCG_LAT_STAT_SMP_WRITE(global_direct_reclaim, MEM_LAT_GLOBAL_DIRECT_RECLAIM);
 MEMCG_LAT_STAT_SMP_WRITE(memcg_direct_reclaim, MEM_LAT_MEMCG_DIRECT_RECLAIM);
 MEMCG_LAT_STAT_SMP_WRITE(direct_compact, MEM_LAT_DIRECT_COMPACT);
+MEMCG_LAT_STAT_SMP_WRITE(global_direct_swapout, MEM_LAT_GLOBAL_DIRECT_SWAPOUT);
+MEMCG_LAT_STAT_SMP_WRITE(memcg_direct_swapout, MEM_LAT_MEMCG_DIRECT_SWAPOUT);
+MEMCG_LAT_STAT_SMP_WRITE(direct_swapin, MEM_LAT_DIRECT_SWAPIN);
 
 smp_call_func_t smp_memcg_lat_write_funcs[] = {
 	smp_write_global_direct_reclaim,
 	smp_write_memcg_direct_reclaim,
 	smp_write_direct_compact,
+	smp_write_global_direct_swapout,
+	smp_write_memcg_direct_swapout,
+	smp_write_direct_swapin,
 };
 
 static int memcg_lat_stat_write(struct cgroup_subsys_state *css,
@@ -5552,6 +5558,24 @@ static struct cftype mem_cgroup_legacy_files[] = {
 		.write_u64 = memcg_lat_stat_write,
 		.seq_show =  memcg_lat_stat_show,
 	},
+	{
+		.name = "direct_swapout_global_latency",
+		.private = MEM_LAT_GLOBAL_DIRECT_SWAPOUT,
+		.write_u64 = memcg_lat_stat_write,
+		.seq_show =  memcg_lat_stat_show,
+	},
+	{
+		.name = "direct_swapout_memcg_latency",
+		.private = MEM_LAT_MEMCG_DIRECT_SWAPOUT,
+		.write_u64 = memcg_lat_stat_write,
+		.seq_show =  memcg_lat_stat_show,
+	},
+	{
+		.name = "direct_swapin_latency",
+		.private = MEM_LAT_DIRECT_SWAPIN,
+		.write_u64 = memcg_lat_stat_write,
+		.seq_show =  memcg_lat_stat_show,
+	},
 	{
 		.name = "force_empty",
 		.write = mem_cgroup_force_empty_write,

mm/memory.c

@@ -4041,8 +4041,14 @@ static vm_fault_t handle_pte_fault(struct vm_fault *vmf)
 			return do_fault(vmf);
 	}
 
-	if (!pte_present(vmf->orig_pte))
-		return do_swap_page(vmf);
+	if (!pte_present(vmf->orig_pte)) {
+		u64 start = ktime_get_ns();
+		vm_fault_t retval = do_swap_page(vmf);
+
+		memcg_lat_stat_update(MEM_LAT_DIRECT_SWAPIN,
+				      (ktime_get_ns() - start));
+		return retval;
+	}
 
 	if (pte_protnone(vmf->orig_pte) && vma_is_accessible(vmf->vma))
 		return do_numa_page(vmf);

mm/shmem.c

@@ -1641,6 +1641,7 @@ static int shmem_getpage_gfp(struct inode *inode, pgoff_t index,
 	int error;
 	int once = 0;
 	int alloced = 0;
+	u64 start;
 
 	if (index > (MAX_LFS_FILESIZE >> PAGE_SHIFT))
 		return -EFBIG;
@@ -1694,7 +1695,10 @@ static int shmem_getpage_gfp(struct inode *inode, pgoff_t index,
 				count_memcg_event_mm(charge_mm, PGMAJFAULT);
 			}
 			/* Here we actually start the io */
+			start = ktime_get_ns();
 			page = shmem_swapin(swap, gfp, info, index);
+			memcg_lat_stat_update(MEM_LAT_DIRECT_SWAPIN,
+					      (ktime_get_ns() - start));
 			if (!page) {
 				error = -ENOMEM;
 				goto failed;

mm/vmscan.c

@@ -905,6 +905,7 @@ static pageout_t pageout(struct page *page, struct address_space *mapping,
 
 	if (clear_page_dirty_for_io(page)) {
 		int res;
+		u64 start = 0;
 		struct writeback_control wbc = {
 			.sync_mode = WB_SYNC_NONE,
 			.nr_to_write = SWAP_CLUSTER_MAX,
@@ -914,7 +915,14 @@ static pageout_t pageout(struct page *page, struct address_space *mapping,
 		};
 
 		SetPageReclaim(page);
+		if (!current_is_kswapd())
+			start = ktime_get_ns();
 		res = mapping->a_ops->writepage(page, &wbc);
+		if (!current_is_kswapd())
+			memcg_lat_stat_update(global_reclaim(sc) ?
+					      MEM_LAT_GLOBAL_DIRECT_SWAPOUT :
+					      MEM_LAT_MEMCG_DIRECT_SWAPOUT,
+					      (ktime_get_ns() - start));
 		if (res < 0)
 			handle_write_error(mapping, page, res);
 		if (res == AOP_WRITEPAGE_ACTIVATE) {