Optimize clang-tidy workflow for code static analysis (#3432)

* Optimize clang-tidy for code static analysis
Signed-off-by: Nquicksilver <zhifeng.zhang@zilliz.com>

* Optimize clang-tidy for code static analysis
Signed-off-by: Nquicksilver <zhifeng.zhang@zilliz.com>

core/CMakeLists.txt

@@ -130,6 +130,10 @@ if ( NOT LINT_EXCLUSIONS_FILE )
     set( LINT_EXCLUSIONS_FILE ${BUILD_SUPPORT_DIR}/lint_exclusions.txt )
 endif ()
 
+if ( NOT IGNORE_CHECKS_FILE )
+    set( IGNORE_CHECKS_FILE ${BUILD_SUPPORT_DIR}/ignore_checks.txt )
+endif ()
+
 find_program( CPPLINT_BIN NAMES cpplint cpplint.py HINTS ${BUILD_SUPPORT_DIR} )
 message( STATUS "Found cpplint executable at ${CPPLINT_BIN}" )
 
@@ -185,6 +189,7 @@ if ( ${CLANG_TIDY_FOUND} )
         ${PYTHON_EXECUTABLE}    ${BUILD_SUPPORT_DIR}/run_clang_tidy.py
                                 --clang_tidy_binary ${CLANG_TIDY_BIN}
                                 --exclude_globs     ${LINT_EXCLUSIONS_FILE}
+                                --ignore_checks     ${IGNORE_CHECKS_FILE}
                                 --compile_commands  ${CMAKE_BINARY_DIR}/compile_commands.json
                                 --source_dir        ${CMAKE_CURRENT_SOURCE_DIR}/src
                                 ${MILVUS_LINT_QUIET} )

core/build-support/ignore_checks.txt

+clang-diagnostic-error
\ No newline at end of file

core/build-support/run_clang_tidy.py

@@ -41,7 +41,7 @@ def _check_some_files(completed_processes, filenames):
     return lintutils.stdout_pathcolonline(result, filenames)
 
 
-def _check_all(cmd, filenames):
+def _check_all(cmd, filenames, ignore_checks):
     # each clang-tidy instance will process 16 files
     chunks = lintutils.chunk(filenames, 16)
     cmds = [cmd + some for some in chunks]
@@ -65,15 +65,21 @@ def _check_all(cmd, filenames):
             if problem_files:
                 msg = "clang-tidy suggested fixes for {}"
                 print("\n".join(map(msg.format, problem_files)))
-                print(stdout.decode("utf-8"))
                 # ignore thirdparty header file not found issue, such as:
                 #   error: 'fiu.h' file not found [clang-diagnostic-error]
-                cnt_error += _count_key(stdout, "error:")
-                cnt_warning += _count_key(stdout, "warning:")
-                cnt_ignore += _count_key(stdout, "clang-diagnostic-error")
+                cnt_info = ""
+                for line in stdout.splitlines():
+                    if any([len(re.findall(check, line)) > 0 for check in ignore_checks]):
+                        cnt_info += line.replace(" error: ", " ignore: ").decode("utf-8") + "\n"
+                    else:
+                        cnt_info += line.decode("utf-8") + "\n"
+                cnt_error += _count_key(cnt_info, " error: ")
+                cnt_warning += _count_key(cnt_info, " warning: ")
+                cnt_ignore += _count_key(cnt_info, " ignore: ")
+                print(cnt_info)
                 print("clang-tidy - error: {}, warning: {}, ignore {}".
                       format(cnt_error, cnt_warning, cnt_ignore))
-                error = error or (cnt_error > cnt_ignore or cnt_warning > 0)
+                error = error or (cnt_error > 0 or cnt_warning > 0)
     except Exception:
         error = True
         raise
@@ -94,6 +100,9 @@ if __name__ == "__main__":
     parser.add_argument("--exclude_globs",
                         help="Filename containing globs for files "
                         "that should be excluded from the checks")
+    parser.add_argument("--ignore_checks",
+                        help="Checkname containing checklist for files "
+                        "that should be ignore from the checks")
     parser.add_argument("--compile_commands",
                         required=True,
                         help="compile_commands.json to pass clang-tidy")
@@ -115,6 +124,11 @@ if __name__ == "__main__":
         for line in open(arguments.exclude_globs):
             exclude_globs.append(line.strip())
 
+    ignore_checks = []
+    if arguments.ignore_checks:
+        for line in open(arguments.ignore_checks):
+            ignore_checks.append(line.strip())
+
     linted_filenames = []
     for path in lintutils.get_sources(arguments.source_dir, exclude_globs):
         linted_filenames.append(path)
@@ -137,4 +151,4 @@ if __name__ == "__main__":
                 sys.exit(returncode)
 
     else:
-        _check_all(cmd, linted_filenames)
+        _check_all(cmd, linted_filenames, ignore_checks)

sdk/build-support/ignore_checks.txt

+clang-diagnostic-error
\ No newline at end of file

sdk/build-support/run_clang_tidy.py

@@ -23,12 +23,16 @@ import lintutils
 from subprocess import PIPE
 import sys
 from functools import partial
+import re
 
 
 def _get_chunk_key(filenames):
     # lists are not hashable so key on the first filename in a chunk
     return filenames[0]
 
+def _count_key(str, key):
+    m = re.findall(key, str)
+    return len(m)
 
 # clang-tidy outputs complaints in '/path:line_number: complaint' format,
 # so we can scan its output to get a list of files to fix
@@ -37,7 +41,7 @@ def _check_some_files(completed_processes, filenames):
     return lintutils.stdout_pathcolonline(result, filenames)
 
 
-def _check_all(cmd, filenames):
+def _check_all(cmd, filenames, ignore_checks):
     # each clang-tidy instance will process 16 files
     chunks = lintutils.chunk(filenames, 16)
     cmds = [cmd + some for some in chunks]
@@ -51,14 +55,31 @@ def _check_all(cmd, filenames):
     }
     checker = partial(_check_some_files, completed_processes)
     pool = mp.Pool()
+    error = False
     try:
+        cnt_error = 0
+        cnt_warning = 0
+        cnt_ignore = 0
         # check output of completed clang-tidy invocations in parallel
         for problem_files, stdout in pool.imap(checker, chunks):
             if problem_files:
                 msg = "clang-tidy suggested fixes for {}"
                 print("\n".join(map(msg.format, problem_files)))
-                print(stdout)
-                error = True
+                # ignore thirdparty header file not found issue, such as:
+                #   error: 'fiu.h' file not found [clang-diagnostic-error]
+                cnt_info = ""
+                for line in stdout.splitlines():
+                    if any([len(re.findall(check, line)) > 0 for check in ignore_checks]):
+                        cnt_info += line.replace(" error: ", " ignore: ").decode("utf-8") + "\n"
+                    else:
+                        cnt_info += line.decode("utf-8") + "\n"
+                cnt_error += _count_key(cnt_info, " error: ")
+                cnt_warning += _count_key(cnt_info, " warning: ")
+                cnt_ignore += _count_key(cnt_info, " ignore: ")
+                print(cnt_info)
+                print("clang-tidy - error: {}, warning: {}, ignore {}".
+                      format(cnt_error, cnt_warning, cnt_ignore))
+                error = error or (cnt_error > 0 or cnt_warning > 0)
     except Exception:
         error = True
         raise
@@ -79,6 +100,9 @@ if __name__ == "__main__":
     parser.add_argument("--exclude_globs",
                         help="Filename containing globs for files "
                         "that should be excluded from the checks")
+    parser.add_argument("--ignore_checks",
+                        help="Checkname containing checklist for files "
+                        "that should be ignore from the checks")
     parser.add_argument("--compile_commands",
                         required=True,
                         help="compile_commands.json to pass clang-tidy")
@@ -100,6 +124,11 @@ if __name__ == "__main__":
         for line in open(arguments.exclude_globs):
             exclude_globs.append(line.strip())
 
+    ignore_checks = []
+    if arguments.ignore_checks:
+        for line in open(arguments.ignore_checks):
+            ignore_checks.append(line.strip())
+
     linted_filenames = []
     for path in lintutils.get_sources(arguments.source_dir, exclude_globs):
         linted_filenames.append(path)
@@ -122,5 +151,4 @@ if __name__ == "__main__":
                 sys.exit(returncode)
 
     else:
-        _check_all(cmd, linted_filenames)
-
+        _check_all(cmd, linted_filenames, ignore_checks)