diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 77bf882a312f9d00631e78ae7a7aa74882c55e0f..f4c0f1d97cbdd2c169f3743dd8d3d30ce7593345 100755 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -64,6 +64,13 @@ repos: (?x)^( paddle/utils/.* )$ +- repo: local + hooks: + - id: auto-generate-cmakelists + name: auto-generate-cmakelists + entry: bash ./tools/gen_ut_cmakelists.hook + language: system + files: testslist.csv$ - repo: https://github.com/cheshirekow/cmake-format-precommit rev: v0.6.13 hooks: diff --git a/python/paddle/fluid/tests/unittests/collective/CMakeLists.txt b/python/paddle/fluid/tests/unittests/collective/CMakeLists.txt index a299de42cb4daa8378845a7606c00942c4818c69..93eaad262bfacc1f85cc2cd27916c0a3b84c52ec 100644 --- a/python/paddle/fluid/tests/unittests/collective/CMakeLists.txt +++ b/python/paddle/fluid/tests/unittests/collective/CMakeLists.txt @@ -272,7 +272,7 @@ if((WITH_GPU ) set_tests_properties(test_gen_nccl_id_op PROPERTIES RUN_SERIAL 1) endif() -if((WITH_GPU) AND (LINUX)) +if((WITH_GPU OR WITH_ROCM) AND (LINUX)) py_test_modules( test_communication_stream_allreduce_api MODULES test_communication_stream_allreduce_api ENVS diff --git a/tools/gen_ut_cmakelists.hook b/tools/gen_ut_cmakelists.hook new file mode 100644 index 0000000000000000000000000000000000000000..196054a511183a0d643cc0f7bf197f95bde02e70 --- /dev/null +++ b/tools/gen_ut_cmakelists.hook @@ -0,0 +1,4 @@ + +set -e +lists=`python tools/gen_ut_cmakelists.py -f $* |grep 'modified/new:'|cut -f 2 -d :` +git add $lists diff --git a/tools/gen_ut_cmakelists.py b/tools/gen_ut_cmakelists.py index 1ba5f563f75a25f49c07a5fb170f42dfec3423c6..ef5ec56538a6c4713e63184c3303800d0c0c6381 100644 --- a/tools/gen_ut_cmakelists.py +++ b/tools/gen_ut_cmakelists.py @@ -342,14 +342,20 @@ class CMakeGenerator(): self.processed_dirs = set() self.port_manager = DistUTPortManager(ignore_dirs) self.current_dirs = _norm_dirs(current_dirs) + self.modified_or_created_files = [] def prepare_dist_ut_port(self): for c in self._find_root_dirs(): self.port_manager.parse_assigned_dist_ut_ports(c, depth=0) def parse_csvs(self): + ''' + parse csv files, return the lists of craeted or modified files + ''' + self.modified_or_created_files = [] for c in self.current_dirs: self._gen_cmakelists(c) + return self.modified_or_created_files def _find_root_dirs(self): root_dirs = [] @@ -449,7 +455,6 @@ class CMakeGenerator(): def _gen_cmakelists(self, current_work_dir, depth=0): if depth == 0: self.processed_dirs.clear() - print("procfessing dir:", current_work_dir) if current_work_dir == "": current_work_dir = "." @@ -490,9 +495,20 @@ class CMakeGenerator(): for sub in sub_dirs: cmds += f"add_subdirectory({sub})\n" - print(cmds, end="") - with open(f"{current_work_dir}/CMakeLists.txt", "w") as cmake_file: - print(cmds, end="", file=cmake_file) + + # check whether the generated file are thge same with the existing file, ignoring the blank chars + # if the are same, skip the weiting process + with open(f"{current_work_dir}/CMakeLists.txt", "r") as old_cmake_file: + char_seq = old_cmake_file.read().split() + char_seq = "".join(char_seq) + + if char_seq != "".join(cmds.split()): + assert f"{current_work_dir}/CMakeLists.txt" not in self.modified_or_created_files, \ + f"the file {current_work_dir}/CMakeLists.txt are modified twice, which may cause some error" + self.modified_or_created_files.append( + f"{current_work_dir}/CMakeLists.txt") + with open(f"{current_work_dir}/CMakeLists.txt", "w") as cmake_file: + print(cmds, end="", file=cmake_file) if __name__ == "__main__": @@ -544,4 +560,8 @@ if __name__ == "__main__": cmake_generator = CMakeGenerator(current_work_dirs, args.ignore_cmake_dirs) cmake_generator.prepare_dist_ut_port() - cmake_generator.parse_csvs() + created = cmake_generator.parse_csvs() + + # summary the modified files + for f in created: + print("modified/new:", f)