mm/dynamic_hugetlb: fix list corruption in hpool_merge_page()

hulk inclusion
category: bugfix
bugzilla: https://gitee.com/openeuler/kernel/issues/I66OCA
CVE: NA

--------------------------------

The percpu pool will be cleared by clear_percpu_pools(), and then check
whether all pages are already freed. If some pages are not freed, we will
firstly isolate the freed pages and then migrate the used pages.
Since we missed to get lock of percpu_pool, the used pages can be free to
percpu_pool while the isolation is going. In such case, the list operation
will be unreliable.

To fix this problem, we need to get all related locks sequentially and
clear the perpcu_pool again before isolate the freed pages.

Fixes: cdbeee51 ("mm/dynamic_hugetlb: add migration function")
Signed-off-by: NLiu Shixin <liushixin2@huawei.com>
Reviewed-by: NTong Tiangen <tongtiangen@huawei.com>
Signed-off-by: NZheng Zengkai <zhengzengkai@huawei.com>

mm/dynamic_hugetlb.c

@@ -182,25 +182,6 @@ static void reclaim_pages_from_percpu_pool(struct dhugetlb_pool *hpool,
 	}
 }
 
-static void clear_percpu_pools(struct dhugetlb_pool *hpool)
-{
-	struct percpu_pages_pool *percpu_pool;
-	int i;
-
-	lockdep_assert_held(&hpool->lock);
-
-	spin_unlock(&hpool->lock);
-	for (i = 0; i < NR_PERCPU_POOL; i++)
-		spin_lock(&hpool->percpu_pool[i].lock);
-	spin_lock(&hpool->lock);
-	for (i = 0; i < NR_PERCPU_POOL; i++) {
-		percpu_pool = &hpool->percpu_pool[i];
-		reclaim_pages_from_percpu_pool(hpool, percpu_pool, percpu_pool->free_pages);
-	}
-	for (i = 0; i < NR_PERCPU_POOL; i++)
-		spin_unlock(&hpool->percpu_pool[i].lock);
-}
-
 /* We only try 5 times to reclaim pages */
 #define	HPOOL_RECLAIM_RETRIES	5
 
@@ -210,6 +191,7 @@ static int hpool_merge_page(struct dhugetlb_pool *hpool, int hpages_pool_idx, bo
 	struct split_hugepage *split_page, *split_next;
 	unsigned long nr_pages, block_size;
 	struct page *page, *next, *p;
+	struct percpu_pages_pool *percpu_pool;
 	bool need_migrate = false, need_initial = false;
 	int i, try;
 	LIST_HEAD(wait_page_list);
@@ -241,7 +223,22 @@ static int hpool_merge_page(struct dhugetlb_pool *hpool, int hpages_pool_idx, bo
 		try = 0;
 
 merge:
-		clear_percpu_pools(hpool);
+		/*
+		 * If we are merging 4K page to 2M page, we need to get
+		 * lock of percpu pool sequentially and clear percpu pool.
+		 */
+		if (hpages_pool_idx == HUGE_PAGES_POOL_2M) {
+			spin_unlock(&hpool->lock);
+			for (i = 0; i < NR_PERCPU_POOL; i++)
+				spin_lock(&hpool->percpu_pool[i].lock);
+			spin_lock(&hpool->lock);
+			for (i = 0; i < NR_PERCPU_POOL; i++) {
+				percpu_pool = &hpool->percpu_pool[i];
+				reclaim_pages_from_percpu_pool(hpool, percpu_pool,
+							       percpu_pool->free_pages);
+			}
+		}
+
 		page = pfn_to_page(split_page->start_pfn);
 		for (i = 0; i < nr_pages; i+= block_size) {
 			p = pfn_to_page(split_page->start_pfn + i);
@@ -252,6 +249,14 @@ static int hpool_merge_page(struct dhugetlb_pool *hpool, int hpages_pool_idx, bo
 					goto migrate;
 			}
 		}
+		if (hpages_pool_idx == HUGE_PAGES_POOL_2M) {
+			/*
+			 * All target 4K page are in src_hpages_pool, we
+			 * can unlock percpu pool.
+			 */
+			for (i = 0; i < NR_PERCPU_POOL; i++)
+				spin_unlock(&hpool->percpu_pool[i].lock);
+		}
 
 		list_del(&split_page->head_pages);
 		hpages_pool->split_normal_pages--;
@@ -284,8 +289,14 @@ static int hpool_merge_page(struct dhugetlb_pool *hpool, int hpages_pool_idx, bo
 		trace_dynamic_hugetlb_split_merge(hpool, page, DHUGETLB_MERGE, page_size(page));
 		return 0;
 next:
+		if (hpages_pool_idx == HUGE_PAGES_POOL_2M) {
+			/* Unlock percpu pool before try next */
+			for (i = 0; i < NR_PERCPU_POOL; i++)
+				spin_unlock(&hpool->percpu_pool[i].lock);
+		}
 		continue;
 migrate:
+		/* page migration only used for HUGE_PAGES_POOL_2M */
 		if (try++ >= HPOOL_RECLAIM_RETRIES)
 			goto next;
 
@@ -300,7 +311,10 @@ static int hpool_merge_page(struct dhugetlb_pool *hpool, int hpages_pool_idx, bo
 		}
 
 		/* Unlock and try migration. */
+		for (i = 0; i < NR_PERCPU_POOL; i++)
+			spin_unlock(&hpool->percpu_pool[i].lock);
 		spin_unlock(&hpool->lock);
+
 		for (i = 0; i < nr_pages; i+= block_size) {
 			p = pfn_to_page(split_page->start_pfn + i);
 			if (PagePool(p))
@@ -312,6 +326,10 @@ static int hpool_merge_page(struct dhugetlb_pool *hpool, int hpages_pool_idx, bo
 		}
 		spin_lock(&hpool->lock);
 
+		/*
+		 * Move all isolate pages to src_hpages_pool and then try
+		 * merge again.
+		 */
 		list_for_each_entry_safe(page, next, &wait_page_list, lru) {
 			list_move_tail(&page->lru, &src_hpages_pool->hugepage_freelists);
 			src_hpages_pool->free_normal_pages++;