diff --git a/cpp/CHANGELOG.md b/cpp/CHANGELOG.md index 820b16756b66b98d72542d40f46db9a9be76ea05..60374fa3100e35620c782088fec786b78a53fe49 100644 --- a/cpp/CHANGELOG.md +++ b/cpp/CHANGELOG.md @@ -25,6 +25,7 @@ Please mark all change in change log and use the ticket from JIRA. - MS-561 - Add contributing guidelines, code of conduct and README docs - MS-567 - Add NOTICE.md - MS-569 - Complete the NOTICE.md +- MS-575 - Add Clang-format & Clang-tidy & Cpplint # Milvus 0.4.0 (2019-09-12) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index bb15ddf075f780be799bb1dbc49e820c7c0d27aa..01007bf08538c24366f5725111dfeac9cd664326 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -21,6 +21,8 @@ cmake_minimum_required(VERSION 3.14) message(STATUS "Building using CMake version: ${CMAKE_VERSION}") +set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_CURRENT_SOURCE_DIR}/cmake") + MACRO (GET_CURRENT_TIME CURRENT_TIME) execute_process(COMMAND "date" +"%Y-%m-%d %H:%M.%S" OUTPUT_VARIABLE ${CURRENT_TIME}) ENDMACRO (GET_CURRENT_TIME) @@ -41,6 +43,10 @@ endif() set(MILVUS_VERSION "${GIT_BRANCH_NAME}") string(REGEX MATCH "[0-9]+\\.[0-9]+\\.[0-9]" MILVUS_VERSION "${MILVUS_VERSION}") +set(CLANG_FORMAT_VERSION "6.0") +find_package(ClangTools) +set(BUILD_SUPPORT_DIR "${CMAKE_SOURCE_DIR}/build-support") + if(CMAKE_BUILD_TYPE STREQUAL "Release") set(BUILD_TYPE "Release") else() @@ -111,8 +117,6 @@ else() include_directories(${MYSQL_INCLUDE_DIR}) endif() -set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_CURRENT_SOURCE_DIR}/cmake") - set(MILVUS_SOURCE_DIR ${PROJECT_SOURCE_DIR}) set(MILVUS_BINARY_DIR ${PROJECT_BINARY_DIR}) set(MILVUS_ENGINE_SRC ${PROJECT_SOURCE_DIR}/src) @@ -152,3 +156,9 @@ install(FILES conf/log_config.conf DESTINATION conf) + +add_custom_target(format ${BUILD_SUPPORT_DIR}/run_clang_format.py + ${CLANG_FORMAT_BIN} + ${BUILD_SUPPORT_DIR}/clang_format_exclusions.txt + ${CMAKE_CURRENT_SOURCE_DIR}/src) + diff --git a/cpp/build-support/clang_format_exclusions.txt b/cpp/build-support/clang_format_exclusions.txt new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/cpp/build-support/run_clang_format.py b/cpp/build-support/run_clang_format.py new file mode 100755 index 0000000000000000000000000000000000000000..4235b2d4af6281a64ab358c3265809004e3a56be --- /dev/null +++ b/cpp/build-support/run_clang_format.py @@ -0,0 +1,79 @@ +#!/usr/bin/env python +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import fnmatch +import os +import subprocess +import sys + +if len(sys.argv) < 4: + sys.stderr.write("Usage: %s $CLANG_FORMAT_VERSION exclude_globs.txt " + "$source_dir\n" % + sys.argv[0]) + sys.exit(1) + +CLANG_FORMAT = sys.argv[1] +EXCLUDE_GLOBS_FILENAME = sys.argv[2] +SOURCE_DIR = sys.argv[3] + +if len(sys.argv) > 4: + CHECK_FORMAT = int(sys.argv[4]) == 1 +else: + CHECK_FORMAT = False + + +exclude_globs = [line.strip() for line in open(EXCLUDE_GLOBS_FILENAME, "r")] + +files_to_format = [] +matches = [] +for directory, subdirs, files in os.walk(SOURCE_DIR): + for name in files: + name = os.path.join(directory, name) + if not (name.endswith('.h') or name.endswith('.cpp') or name.endswith('.cuh') or name.endswith('.cu')): + continue + + excluded = False + for g in exclude_globs: + if fnmatch.fnmatch(name, g): + excluded = True + break + if not excluded: + files_to_format.append(name) + +if CHECK_FORMAT: + output = subprocess.check_output([CLANG_FORMAT, '-output-replacements-xml'] + + files_to_format, + stderr=subprocess.STDOUT).decode('utf8') + + to_fix = [] + for line in output.split('\n'): + if 'offset' in line: + to_fix.append(line) + + if len(to_fix) > 0: + print("clang-format checks failed, run 'make format' to fix") + sys.exit(-1) +else: + try: + cmd = [CLANG_FORMAT, '-i'] + files_to_format + subprocess.check_output(cmd, stderr=subprocess.STDOUT) + except Exception as e: + print(e) + print(' '.join(cmd)) + raise + diff --git a/cpp/cmake/FindClangTools.cmake b/cpp/cmake/FindClangTools.cmake new file mode 100644 index 0000000000000000000000000000000000000000..075d9381c50d2c04e03f0aa0619a31555112fad5 --- /dev/null +++ b/cpp/cmake/FindClangTools.cmake @@ -0,0 +1,104 @@ +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Tries to find the clang-tidy and clang-format modules +# +# Usage of this module as follows: +# +# find_package(ClangTools) +# +# Variables used by this module, they can change the default behaviour and need +# to be set before calling find_package: +# +# ClangToolsBin_HOME - +# When set, this path is inspected instead of standard library binary locations +# to find clang-tidy and clang-format +# +# This module defines +# CLANG_TIDY_BIN, The path to the clang tidy binary +# CLANG_TIDY_FOUND, Whether clang tidy was found +# CLANG_FORMAT_BIN, The path to the clang format binary +# CLANG_TIDY_FOUND, Whether clang format was found + +find_program(CLANG_TIDY_BIN + NAMES + clang-tidy-6.0 + clang-tidy-5.0 + clang-tidy-4.0 + clang-tidy-3.9 + clang-tidy-3.8 + clang-tidy-3.7 + clang-tidy-3.6 + clang-tidy + PATHS ${ClangTools_PATH} $ENV{CLANG_TOOLS_PATH} /usr/local/bin /usr/bin + NO_DEFAULT_PATH +) + +if ( "${CLANG_TIDY_BIN}" STREQUAL "CLANG_TIDY_BIN-NOTFOUND" ) + set(CLANG_TIDY_FOUND 0) + message("clang-tidy not found") +else() + set(CLANG_TIDY_FOUND 1) + message("clang-tidy found at ${CLANG_TIDY_BIN}") +endif() + +if (CLANG_FORMAT_VERSION) + find_program(CLANG_FORMAT_BIN + NAMES clang-format-${CLANG_FORMAT_VERSION} + PATHS + ${ClangTools_PATH} + $ENV{CLANG_TOOLS_PATH} + /usr/local/bin /usr/bin + NO_DEFAULT_PATH + ) + + # If not found yet, search alternative locations + if (("${CLANG_FORMAT_BIN}" STREQUAL "CLANG_FORMAT_BIN-NOTFOUND") AND APPLE) + # Homebrew ships older LLVM versions in /usr/local/opt/llvm@version/ + STRING(REGEX REPLACE "^([0-9]+)\\.[0-9]+" "\\1" CLANG_FORMAT_MAJOR_VERSION "${CLANG_FORMAT_VERSION}") + STRING(REGEX REPLACE "^[0-9]+\\.([0-9]+)" "\\1" CLANG_FORMAT_MINOR_VERSION "${CLANG_FORMAT_VERSION}") + if ("${CLANG_FORMAT_MINOR_VERSION}" STREQUAL "0") + find_program(CLANG_FORMAT_BIN + NAMES clang-format + PATHS /usr/local/opt/llvm@${CLANG_FORMAT_MAJOR_VERSION}/bin + NO_DEFAULT_PATH + ) + else() + find_program(CLANG_FORMAT_BIN + NAMES clang-format + PATHS /usr/local/opt/llvm@${CLANG_FORMAT_VERSION}/bin + NO_DEFAULT_PATH + ) + endif() + endif() +else() + find_program(CLANG_FORMAT_BIN + NAMES clang-format-4.0 + clang-format-3.9 + clang-format-3.8 + clang-format-3.7 + clang-format-3.6 + clang-format + PATHS ${ClangTools_PATH} $ENV{CLANG_TOOLS_PATH} /usr/local/bin /usr/bin + NO_DEFAULT_PATH + ) +endif() + +if ( "${CLANG_FORMAT_BIN}" STREQUAL "CLANG_FORMAT_BIN-NOTFOUND" ) + set(CLANG_FORMAT_FOUND 0) + message("clang-format not found") +else() + set(CLANG_FORMAT_FOUND 1) + message("clang-format found at ${CLANG_FORMAT_BIN}") +endif() + diff --git a/cpp/src/core/knowhere/knowhere/common/Log.h b/cpp/src/core/knowhere/knowhere/common/Log.h new file mode 100644 index 0000000000000000000000000000000000000000..0d99c624abed4a67d9bbf7a0efd82c8cb809c238 --- /dev/null +++ b/cpp/src/core/knowhere/knowhere/common/Log.h @@ -0,0 +1,35 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + +#pragma once + +#include "utils/easylogging++.h" + +namespace zilliz { +namespace knowhere { +#define KNOWHERE_DOMAIN_NAME "[KNOWHERE] " +#define KNOWHERE_ERROR_TEXT "KNOWHERE Error:" + +#define KNOWHERE_LOG_TRACE LOG(TRACE) << KNOWHERE_DOMAIN_NAME +#define KNOWHERE_LOG_DEBUG LOG(DEBUG) << KNOWHERE_DOMAIN_NAME +#define KNOWHERE_LOG_INFO LOG(INFO) << KNOWHERE_DOMAIN_NAME +#define KNOWHERE_LOG_WARNING LOG(WARNING) << KNOWHERE_DOMAIN_NAME +#define KNOWHERE_LOG_ERROR LOG(ERROR) << KNOWHERE_DOMAIN_NAME +#define KNOWHERE_LOG_FATAL LOG(FATAL) << KNOWHERE_DOMAIN_NAME +} // namespace knowhere +} // namespace zilliz \ No newline at end of file diff --git a/cpp/src/core/knowhere/knowhere/common/exception.cpp b/cpp/src/core/knowhere/knowhere/common/exception.cpp index fcf3886d9b52e8f7b3c1d8bb86033763323e0066..eb19e21bb05cb91f4e3a4f9f7bcf100422db81e4 100644 --- a/cpp/src/core/knowhere/knowhere/common/exception.cpp +++ b/cpp/src/core/knowhere/knowhere/common/exception.cpp @@ -19,6 +19,7 @@ #include #include "exception.h" +#include "Log.h" namespace zilliz { namespace knowhere { diff --git a/cpp/src/core/test/CMakeLists.txt b/cpp/src/core/test/CMakeLists.txt index b62fd6cda3b2cc3fe4764b3c70d2839355de8191..cdd500ea7254caffb7c655bb951668d5ed20806f 100644 --- a/cpp/src/core/test/CMakeLists.txt +++ b/cpp/src/core/test/CMakeLists.txt @@ -22,6 +22,9 @@ set(basic_libs gomp gfortran pthread ) +set(util_srcs + ${MILVUS_ENGINE_SRC}/utils/easylogging++.cc + ) # set(ivf_srcs @@ -36,7 +39,7 @@ set(ivf_srcs utils.cpp ) if(NOT TARGET test_ivf) - add_executable(test_ivf test_ivf.cpp ${ivf_srcs}) + add_executable(test_ivf test_ivf.cpp ${ivf_srcs} ${util_srcs}) endif() target_link_libraries(test_ivf ${depend_libs} ${unittest_libs} ${basic_libs}) @@ -53,7 +56,7 @@ set(idmap_srcs utils.cpp ) if(NOT TARGET test_idmap) - add_executable(test_idmap test_idmap.cpp ${idmap_srcs}) + add_executable(test_idmap test_idmap.cpp ${idmap_srcs} ${util_srcs}) endif() target_link_libraries(test_idmap ${depend_libs} ${unittest_libs} ${basic_libs}) @@ -70,7 +73,7 @@ set(kdt_srcs utils.cpp ) if(NOT TARGET test_kdt) - add_executable(test_kdt test_kdt.cpp ${kdt_srcs}) + add_executable(test_kdt test_kdt.cpp ${kdt_srcs} ${util_srcs}) endif() target_link_libraries(test_kdt SPTAGLibStatic diff --git a/cpp/src/core/test/test_nsg/CMakeLists.txt b/cpp/src/core/test/test_nsg/CMakeLists.txt index 9ee8e765255105b445815dfd8e736ddf076e7d12..17b62cd40ddd4de8ea09d345be840e4d7c447a47 100644 --- a/cpp/src/core/test/test_nsg/CMakeLists.txt +++ b/cpp/src/core/test/test_nsg/CMakeLists.txt @@ -34,6 +34,7 @@ if(NOT TARGET test_nsg) test_nsg.cpp ${interface_src} ${nsg_src} + ${util_srcs} ) endif() diff --git a/cpp/src/core/test/utils.cpp b/cpp/src/core/test/utils.cpp index 97f4600ca74092d118967cde4f7839c855d3a599..4e240f5f74d4288843513ded37c894b82b657e9d 100644 --- a/cpp/src/core/test/utils.cpp +++ b/cpp/src/core/test/utils.cpp @@ -18,6 +18,15 @@ #include "utils.h" +INITIALIZE_EASYLOGGINGPP + +void InitLog() { + el::Configurations defaultConf; + defaultConf.setToDefault(); + defaultConf.set(el::Level::Debug, + el::ConfigurationType::Format, "[%thread-%datetime-%level]: %msg (%fbase:%line)"); + el::Loggers::reconfigureLogger("default", defaultConf); +} void DataGen::Init_with_default() { Generate(dim, nb, nq); diff --git a/cpp/src/core/test/utils.h b/cpp/src/core/test/utils.h index b9db01de51df1fb8c41ef4c1a88f7ab709befdce..df7f11d68952b500894056dfeb7a2ea29335a689 100644 --- a/cpp/src/core/test/utils.h +++ b/cpp/src/core/test/utils.h @@ -24,6 +24,7 @@ #include #include "knowhere/adapter/structure.h" +#include "knowhere/common/Log.h" class DataGen { protected: @@ -65,6 +66,8 @@ extern void GenBase(const int64_t &dim, float *xb, int64_t *ids); +extern void InitLog(); + zilliz::knowhere::DatasetPtr generate_dataset(int64_t nb, int64_t dim, float *xb, long *ids);