From 41f844f8ac86fb4e1d37bb349ee0b97d917a682e Mon Sep 17 00:00:00 2001 From: quicksilver Date: Tue, 25 Aug 2020 09:39:25 +0800 Subject: [PATCH] Optimize clang-tidy workflow for code static analysis (#3432) * Optimize clang-tidy for code static analysis Signed-off-by: quicksilver * Optimize clang-tidy for code static analysis Signed-off-by: quicksilver --- core/CMakeLists.txt | 5 ++++ core/build-support/ignore_checks.txt | 1 + core/build-support/run_clang_tidy.py | 28 +++++++++++++++----- sdk/build-support/ignore_checks.txt | 1 + sdk/build-support/run_clang_tidy.py | 38 ++++++++++++++++++++++++---- 5 files changed, 61 insertions(+), 12 deletions(-) create mode 100644 core/build-support/ignore_checks.txt create mode 100644 sdk/build-support/ignore_checks.txt diff --git a/core/CMakeLists.txt b/core/CMakeLists.txt index 51cfa3a2..cda92ef2 100644 --- a/core/CMakeLists.txt +++ b/core/CMakeLists.txt @@ -130,6 +130,10 @@ if ( NOT LINT_EXCLUSIONS_FILE ) set( LINT_EXCLUSIONS_FILE ${BUILD_SUPPORT_DIR}/lint_exclusions.txt ) endif () +if ( NOT IGNORE_CHECKS_FILE ) + set( IGNORE_CHECKS_FILE ${BUILD_SUPPORT_DIR}/ignore_checks.txt ) +endif () + find_program( CPPLINT_BIN NAMES cpplint cpplint.py HINTS ${BUILD_SUPPORT_DIR} ) message( STATUS "Found cpplint executable at ${CPPLINT_BIN}" ) @@ -185,6 +189,7 @@ if ( ${CLANG_TIDY_FOUND} ) ${PYTHON_EXECUTABLE} ${BUILD_SUPPORT_DIR}/run_clang_tidy.py --clang_tidy_binary ${CLANG_TIDY_BIN} --exclude_globs ${LINT_EXCLUSIONS_FILE} + --ignore_checks ${IGNORE_CHECKS_FILE} --compile_commands ${CMAKE_BINARY_DIR}/compile_commands.json --source_dir ${CMAKE_CURRENT_SOURCE_DIR}/src ${MILVUS_LINT_QUIET} ) diff --git a/core/build-support/ignore_checks.txt b/core/build-support/ignore_checks.txt new file mode 100644 index 00000000..b2504ce5 --- /dev/null +++ b/core/build-support/ignore_checks.txt @@ -0,0 +1 @@ +clang-diagnostic-error \ No newline at end of file diff --git a/core/build-support/run_clang_tidy.py b/core/build-support/run_clang_tidy.py index 9db1ece7..a46db9cb 100755 --- a/core/build-support/run_clang_tidy.py +++ b/core/build-support/run_clang_tidy.py @@ -41,7 +41,7 @@ def _check_some_files(completed_processes, filenames): return lintutils.stdout_pathcolonline(result, filenames) -def _check_all(cmd, filenames): +def _check_all(cmd, filenames, ignore_checks): # each clang-tidy instance will process 16 files chunks = lintutils.chunk(filenames, 16) cmds = [cmd + some for some in chunks] @@ -65,15 +65,21 @@ def _check_all(cmd, filenames): if problem_files: msg = "clang-tidy suggested fixes for {}" print("\n".join(map(msg.format, problem_files))) - print(stdout.decode("utf-8")) # ignore thirdparty header file not found issue, such as: # error: 'fiu.h' file not found [clang-diagnostic-error] - cnt_error += _count_key(stdout, "error:") - cnt_warning += _count_key(stdout, "warning:") - cnt_ignore += _count_key(stdout, "clang-diagnostic-error") + cnt_info = "" + for line in stdout.splitlines(): + if any([len(re.findall(check, line)) > 0 for check in ignore_checks]): + cnt_info += line.replace(" error: ", " ignore: ").decode("utf-8") + "\n" + else: + cnt_info += line.decode("utf-8") + "\n" + cnt_error += _count_key(cnt_info, " error: ") + cnt_warning += _count_key(cnt_info, " warning: ") + cnt_ignore += _count_key(cnt_info, " ignore: ") + print(cnt_info) print("clang-tidy - error: {}, warning: {}, ignore {}". format(cnt_error, cnt_warning, cnt_ignore)) - error = error or (cnt_error > cnt_ignore or cnt_warning > 0) + error = error or (cnt_error > 0 or cnt_warning > 0) except Exception: error = True raise @@ -94,6 +100,9 @@ if __name__ == "__main__": parser.add_argument("--exclude_globs", help="Filename containing globs for files " "that should be excluded from the checks") + parser.add_argument("--ignore_checks", + help="Checkname containing checklist for files " + "that should be ignore from the checks") parser.add_argument("--compile_commands", required=True, help="compile_commands.json to pass clang-tidy") @@ -115,6 +124,11 @@ if __name__ == "__main__": for line in open(arguments.exclude_globs): exclude_globs.append(line.strip()) + ignore_checks = [] + if arguments.ignore_checks: + for line in open(arguments.ignore_checks): + ignore_checks.append(line.strip()) + linted_filenames = [] for path in lintutils.get_sources(arguments.source_dir, exclude_globs): linted_filenames.append(path) @@ -137,4 +151,4 @@ if __name__ == "__main__": sys.exit(returncode) else: - _check_all(cmd, linted_filenames) + _check_all(cmd, linted_filenames, ignore_checks) diff --git a/sdk/build-support/ignore_checks.txt b/sdk/build-support/ignore_checks.txt new file mode 100644 index 00000000..b2504ce5 --- /dev/null +++ b/sdk/build-support/ignore_checks.txt @@ -0,0 +1 @@ +clang-diagnostic-error \ No newline at end of file diff --git a/sdk/build-support/run_clang_tidy.py b/sdk/build-support/run_clang_tidy.py index 6df17d53..a46db9cb 100755 --- a/sdk/build-support/run_clang_tidy.py +++ b/sdk/build-support/run_clang_tidy.py @@ -23,12 +23,16 @@ import lintutils from subprocess import PIPE import sys from functools import partial +import re def _get_chunk_key(filenames): # lists are not hashable so key on the first filename in a chunk return filenames[0] +def _count_key(str, key): + m = re.findall(key, str) + return len(m) # clang-tidy outputs complaints in '/path:line_number: complaint' format, # so we can scan its output to get a list of files to fix @@ -37,7 +41,7 @@ def _check_some_files(completed_processes, filenames): return lintutils.stdout_pathcolonline(result, filenames) -def _check_all(cmd, filenames): +def _check_all(cmd, filenames, ignore_checks): # each clang-tidy instance will process 16 files chunks = lintutils.chunk(filenames, 16) cmds = [cmd + some for some in chunks] @@ -51,14 +55,31 @@ def _check_all(cmd, filenames): } checker = partial(_check_some_files, completed_processes) pool = mp.Pool() + error = False try: + cnt_error = 0 + cnt_warning = 0 + cnt_ignore = 0 # check output of completed clang-tidy invocations in parallel for problem_files, stdout in pool.imap(checker, chunks): if problem_files: msg = "clang-tidy suggested fixes for {}" print("\n".join(map(msg.format, problem_files))) - print(stdout) - error = True + # ignore thirdparty header file not found issue, such as: + # error: 'fiu.h' file not found [clang-diagnostic-error] + cnt_info = "" + for line in stdout.splitlines(): + if any([len(re.findall(check, line)) > 0 for check in ignore_checks]): + cnt_info += line.replace(" error: ", " ignore: ").decode("utf-8") + "\n" + else: + cnt_info += line.decode("utf-8") + "\n" + cnt_error += _count_key(cnt_info, " error: ") + cnt_warning += _count_key(cnt_info, " warning: ") + cnt_ignore += _count_key(cnt_info, " ignore: ") + print(cnt_info) + print("clang-tidy - error: {}, warning: {}, ignore {}". + format(cnt_error, cnt_warning, cnt_ignore)) + error = error or (cnt_error > 0 or cnt_warning > 0) except Exception: error = True raise @@ -79,6 +100,9 @@ if __name__ == "__main__": parser.add_argument("--exclude_globs", help="Filename containing globs for files " "that should be excluded from the checks") + parser.add_argument("--ignore_checks", + help="Checkname containing checklist for files " + "that should be ignore from the checks") parser.add_argument("--compile_commands", required=True, help="compile_commands.json to pass clang-tidy") @@ -100,6 +124,11 @@ if __name__ == "__main__": for line in open(arguments.exclude_globs): exclude_globs.append(line.strip()) + ignore_checks = [] + if arguments.ignore_checks: + for line in open(arguments.ignore_checks): + ignore_checks.append(line.strip()) + linted_filenames = [] for path in lintutils.get_sources(arguments.source_dir, exclude_globs): linted_filenames.append(path) @@ -122,5 +151,4 @@ if __name__ == "__main__": sys.exit(returncode) else: - _check_all(cmd, linted_filenames) - + _check_all(cmd, linted_filenames, ignore_checks) -- GitLab