alinux: mm, memcg: rework memsli interfaces

to #26424368

This reworks memsli "start", "end", "update" interfaces to make it more
clear and symmetrical, by merging "update" action into "end", just like
what psi_memstall_{enter, leave} does.

Now the latency probe pattern of memsli is as follows:

memcg_lat_stat_start(&start);
/* kernel codes being probed */
memcg_lat_stat_end(MEM_LAT_XXX, start);

This also formats the codes and fixes the warning(s) produced when
CONFIG_MEMSLI is not set.
Signed-off-by: NXu Yu <xuyu@linux.alibaba.com>
Reviewed-by: NXunlei Pang <xlpang@linux.alibaba.com>
Reviewed-by: NYang Shi <yang.shi@linux.alibaba.com>

include/linux/memcontrol.h

@@ -1288,22 +1288,15 @@ static inline void memcg_check_wmark_min_adj(struct task_struct *curr,
 #endif /* CONFIG_MEMCG */
 
 #ifdef CONFIG_MEMSLI
-extern void memcg_lat_stat_update(enum mem_lat_stat_item sidx, u64 duration);
 extern void memcg_lat_stat_start(u64 *start);
-extern u64 memcg_lat_stat_end(u64 start);
+extern void memcg_lat_stat_end(enum mem_lat_stat_item sidx, u64 start);
 #else
-static inline void memcg_lat_stat_update(enum mem_lat_stat_item sidx,
-					 u64 duration)
-{
-}
-
 static inline void memcg_lat_stat_start(u64 *start)
 {
 }
 
-static inline u64 memcg_lat_stat_end(u64 start)
+static inline void memcg_lat_stat_end(enum mem_lat_stat_item sidx, u64 start)
 {
-	return 0;
 }
 #endif /* CONFIG_MEMSLI */
 

mm/memcontrol.c

@@ -90,8 +90,10 @@ static bool cgroup_memory_nosocket;
 /* Kernel memory accounting disabled? */
 static bool cgroup_memory_nokmem;
 
+#ifdef CONFIG_MEMSLI
 /* Cgroup memory SLI disabled? */
 static DEFINE_STATIC_KEY_FALSE(cgroup_memory_nosli);
+#endif /* CONFIG_MEMSLI */
 
 /* Whether the swap controller is active */
 #ifdef CONFIG_MEMCG_SWAP
@@ -2535,8 +2537,7 @@ void mem_cgroup_handle_over_high(void)
 	psi_memstall_leave(&pflags);
 
 out:
-	memcg_lat_stat_update(MEM_LAT_MEMCG_DIRECT_RECLAIM,
-			      memcg_lat_stat_end(start));
+	memcg_lat_stat_end(MEM_LAT_MEMCG_DIRECT_RECLAIM, start);
 	css_put(&memcg->css);
 }
 
@@ -2615,8 +2616,7 @@ static int try_charge(struct mem_cgroup *memcg, gfp_t gfp_mask,
 	memcg_lat_stat_start(&start);
 	nr_reclaimed = try_to_free_mem_cgroup_pages(mem_over_limit, nr_pages,
 						    gfp_mask, may_swap);
-	memcg_lat_stat_update(MEM_LAT_MEMCG_DIRECT_RECLAIM,
-			      memcg_lat_stat_end(start));
+	memcg_lat_stat_end(MEM_LAT_MEMCG_DIRECT_RECLAIM, start);
 
 	if (mem_cgroup_margin(mem_over_limit) >= nr_pages)
 		goto retry;
@@ -4604,7 +4604,7 @@ static int memcg_lat_stat_show(struct seq_file *m, void *v)
 	return 0;
 }
 
-enum mem_lat_count_t get_mem_lat_count_idx(u64 duration)
+static enum mem_lat_count_t get_mem_lat_count_idx(u64 duration)
 {
 	enum mem_lat_count_t idx;
 
@@ -4627,19 +4627,30 @@ enum mem_lat_count_t get_mem_lat_count_idx(u64 duration)
 	return idx;
 }
 
-void memcg_lat_stat_update(enum mem_lat_stat_item sidx, u64 duration)
+void memcg_lat_stat_start(u64 *start)
+{
+	if (!static_branch_unlikely(&cgroup_memory_nosli) &&
+	    !mem_cgroup_disabled())
+		*start = ktime_get_ns();
+	else
+		*start = 0;
+}
+
+void memcg_lat_stat_end(enum mem_lat_stat_item sidx, u64 start)
 {
 	struct mem_cgroup *memcg, *iter;
 	enum mem_lat_count_t cidx;
+	u64 duration;
 
-	if (static_branch_unlikely(&cgroup_memory_nosli))
+	if (static_branch_unlikely(&cgroup_memory_nosli) ||
+	    mem_cgroup_disabled())
 		return;
 
-	if (mem_cgroup_disabled())
+	if (start == 0)
 		return;
 
+	duration = ktime_get_ns() - start;
 	cidx = get_mem_lat_count_idx(duration);
-
 	memcg = get_mem_cgroup_from_mm(current->mm);
 	for (iter = memcg; iter; iter = parent_mem_cgroup(iter)) {
 		this_cpu_inc(iter->lat_stat_cpu->item[sidx][cidx]);
@@ -4648,22 +4659,6 @@ void memcg_lat_stat_update(enum mem_lat_stat_item sidx, u64 duration)
 	}
 	css_put(&memcg->css);
 }
-
-void memcg_lat_stat_start(u64 *start)
-{
-	if (!static_branch_unlikely(&cgroup_memory_nosli) &&
-	    !mem_cgroup_disabled())
-		*start = ktime_get_ns();
-}
-
-u64 memcg_lat_stat_end(u64 start)
-{
-	if (!static_branch_unlikely(&cgroup_memory_nosli) &&
-	    !mem_cgroup_disabled())
-		return ktime_get_ns() - start;
-	else
-		return 0;
-}
 #endif /* CONFIG_MEMSLI */
 
 static void __mem_cgroup_threshold(struct mem_cgroup *memcg, bool swap)
@@ -7844,16 +7839,9 @@ static const struct file_operations memsli_enabled_fops = {
 static int __init mem_cgroup_init(void)
 {
 	int cpu, node;
+#ifdef CONFIG_MEMSLI
 	struct proc_dir_entry *memsli_dir, *memsli_enabled_file;
 
-	memcg_wmark_wq = alloc_workqueue("memcg_wmark", WQ_MEM_RECLAIM |
-				WQ_UNBOUND | WQ_FREEZABLE,
-				WQ_UNBOUND_MAX_ACTIVE);
-
-	if (!memcg_wmark_wq)
-		return -ENOMEM;
-
-#ifdef CONFIG_MEMSLI
 	memsli_dir = proc_mkdir("memsli", NULL);
 	if (!memsli_dir)
 		return -ENOMEM;
@@ -7866,6 +7854,13 @@ static int __init mem_cgroup_init(void)
 	}
 #endif /* CONFIG_MEMSLI */
 
+	memcg_wmark_wq = alloc_workqueue("memcg_wmark", WQ_MEM_RECLAIM |
+				WQ_UNBOUND | WQ_FREEZABLE,
+				WQ_UNBOUND_MAX_ACTIVE);
+
+	if (!memcg_wmark_wq)
+		return -ENOMEM;
+
 #ifdef CONFIG_MEMCG_KMEM
 	/*
 	 * Kmem cache creation is mostly done with the slab_mutex held,

mm/memory.c

@@ -4047,8 +4047,7 @@ static vm_fault_t handle_pte_fault(struct vm_fault *vmf)
 
 		memcg_lat_stat_start(&start);
 		retval = do_swap_page(vmf);
-		memcg_lat_stat_update(MEM_LAT_DIRECT_SWAPIN,
-				      memcg_lat_stat_end(start));
+		memcg_lat_stat_end(MEM_LAT_DIRECT_SWAPIN, start);
 		return retval;
 	}
 

mm/page_alloc.c

@@ -3733,8 +3733,7 @@ __alloc_pages_direct_compact(gfp_t gfp_mask, unsigned int order,
 									prio);
 
 	memalloc_noreclaim_restore(noreclaim_flag);
-	memcg_lat_stat_update(MEM_LAT_DIRECT_COMPACT,
-			      memcg_lat_stat_end(start));
+	memcg_lat_stat_end(MEM_LAT_DIRECT_COMPACT, start);
 	psi_memstall_leave(&pflags);
 
 	if (*compact_result <= COMPACT_INACTIVE)
@@ -3954,8 +3953,7 @@ __perform_reclaim(gfp_t gfp_mask, unsigned int order,
 	current->reclaim_state = NULL;
 	memalloc_noreclaim_restore(noreclaim_flag);
 	fs_reclaim_release(gfp_mask);
-	memcg_lat_stat_update(MEM_LAT_GLOBAL_DIRECT_RECLAIM,
-			      memcg_lat_stat_end(start));
+	memcg_lat_stat_end(MEM_LAT_GLOBAL_DIRECT_RECLAIM, start);
 	psi_memstall_leave(&pflags);
 
 	cond_resched();

mm/shmem.c

@@ -1697,8 +1697,7 @@ static int shmem_getpage_gfp(struct inode *inode, pgoff_t index,
 			/* Here we actually start the io */
 			memcg_lat_stat_start(&start);
 			page = shmem_swapin(swap, gfp, info, index);
-			memcg_lat_stat_update(MEM_LAT_DIRECT_SWAPIN,
-					      memcg_lat_stat_end(start));
+			memcg_lat_stat_end(MEM_LAT_DIRECT_SWAPIN, start);
 			if (!page) {
 				error = -ENOMEM;
 				goto failed;

mm/vmscan.c

@@ -919,10 +919,10 @@ static pageout_t pageout(struct page *page, struct address_space *mapping,
 			memcg_lat_stat_start(&start);
 		res = mapping->a_ops->writepage(page, &wbc);
 		if (!current_is_kswapd())
-			memcg_lat_stat_update(global_reclaim(sc) ?
+			memcg_lat_stat_end(global_reclaim(sc) ?
 					      MEM_LAT_GLOBAL_DIRECT_SWAPOUT :
 					      MEM_LAT_MEMCG_DIRECT_SWAPOUT,
-					      memcg_lat_stat_end(start));
+					      start);
 		if (res < 0)
 			handle_write_error(mapping, page, res);
 		if (res == AOP_WRITEPAGE_ACTIVATE) {