Auto gen cmake (#45614)

.pre-commit-config.yaml

@@ -64,6 +64,13 @@ repos:
             (?x)^(
                 paddle/utils/.*
             )$
+-   repo: local
+    hooks:
+    -   id: auto-generate-cmakelists
+        name: auto-generate-cmakelists
+        entry: bash ./tools/gen_ut_cmakelists.hook
+        language: system
+        files: testslist.csv$
 -   repo: https://github.com/cheshirekow/cmake-format-precommit
     rev: v0.6.13
     hooks:

python/paddle/fluid/tests/unittests/collective/CMakeLists.txt

@@ -272,7 +272,7 @@ if((WITH_GPU
   )
   set_tests_properties(test_gen_nccl_id_op PROPERTIES RUN_SERIAL 1)
 endif()
-if((WITH_GPU) AND (LINUX))
+if((WITH_GPU OR WITH_ROCM) AND (LINUX))
   py_test_modules(
     test_communication_stream_allreduce_api MODULES
     test_communication_stream_allreduce_api ENVS

tools/gen_ut_cmakelists.hook

+
+set -e
+lists=`python tools/gen_ut_cmakelists.py -f $* |grep 'modified/new:'|cut -f 2 -d :`
+git add $lists

tools/gen_ut_cmakelists.py

@@ -342,14 +342,20 @@ class CMakeGenerator():
         self.processed_dirs = set()
         self.port_manager = DistUTPortManager(ignore_dirs)
         self.current_dirs = _norm_dirs(current_dirs)
+        self.modified_or_created_files = []
 
     def prepare_dist_ut_port(self):
         for c in self._find_root_dirs():
             self.port_manager.parse_assigned_dist_ut_ports(c, depth=0)
 
     def parse_csvs(self):
+        '''
+        parse csv files, return the lists of craeted or modified files
+        '''
+        self.modified_or_created_files = []
         for c in self.current_dirs:
             self._gen_cmakelists(c)
+        return self.modified_or_created_files
 
     def _find_root_dirs(self):
         root_dirs = []
@@ -449,7 +455,6 @@ class CMakeGenerator():
     def _gen_cmakelists(self, current_work_dir, depth=0):
         if depth == 0:
             self.processed_dirs.clear()
-        print("procfessing dir:", current_work_dir)
         if current_work_dir == "":
             current_work_dir = "."
 
@@ -490,9 +495,20 @@ class CMakeGenerator():
 
         for sub in sub_dirs:
             cmds += f"add_subdirectory({sub})\n"
-        print(cmds, end="")
-        with open(f"{current_work_dir}/CMakeLists.txt", "w") as cmake_file:
-            print(cmds, end="", file=cmake_file)
+
+        # check whether the generated file are thge same with the existing file, ignoring the blank chars
+        # if the are same, skip the weiting process
+        with open(f"{current_work_dir}/CMakeLists.txt", "r") as old_cmake_file:
+            char_seq = old_cmake_file.read().split()
+        char_seq = "".join(char_seq)
+
+        if char_seq != "".join(cmds.split()):
+            assert f"{current_work_dir}/CMakeLists.txt" not in self.modified_or_created_files, \
+                f"the file {current_work_dir}/CMakeLists.txt are modified twice, which may cause some error"
+            self.modified_or_created_files.append(
+                f"{current_work_dir}/CMakeLists.txt")
+            with open(f"{current_work_dir}/CMakeLists.txt", "w") as cmake_file:
+                print(cmds, end="", file=cmake_file)
 
 
 if __name__ == "__main__":
@@ -544,4 +560,8 @@ if __name__ == "__main__":
 
     cmake_generator = CMakeGenerator(current_work_dirs, args.ignore_cmake_dirs)
     cmake_generator.prepare_dist_ut_port()
-    cmake_generator.parse_csvs()
+    created = cmake_generator.parse_csvs()
+
+    # summary the modified files
+    for f in created:
+        print("modified/new:", f)