提交 8464b8a2 编写于 作者: Z zhiru

refactor cmake


Former-commit-id: 587cc22f501fd12cbdaf2d9950fce3f2483c7020
上级 80dbaebd
......@@ -13,6 +13,7 @@ Please mark all change in change log and use the ticket from JIRA.
- MS-556 - Add Job Definition in Scheduler
- MS-558 - Refine status code
- MS-562 - Add JobMgr and TaskCreator in Scheduler
- MS-566 - Refactor cmake
## New Feature
......
......@@ -51,13 +51,6 @@ message(STATUS "Build type = ${BUILD_TYPE}")
project(milvus VERSION "${MILVUS_VERSION}")
project(milvus_engine LANGUAGES CUDA CXX)
# Ensure that a default make is set
if("${MAKE}" STREQUAL "")
if(NOT MSVC)
find_program(MAKE make)
endif()
endif()
set(MILVUS_VERSION_MAJOR "${milvus_VERSION_MAJOR}")
set(MILVUS_VERSION_MINOR "${milvus_VERSION_MINOR}")
set(MILVUS_VERSION_PATCH "${milvus_VERSION_PATCH}")
......@@ -76,9 +69,6 @@ message(STATUS "Milvus version: "
"${MILVUS_VERSION_MAJOR}.${MILVUS_VERSION_MINOR}.${MILVUS_VERSION_PATCH} "
"(full: '${MILVUS_VERSION}')")
set(MILVUS_SOURCE_DIR ${PROJECT_SOURCE_DIR})
set(MILVUS_BINARY_DIR ${PROJECT_BINARY_DIR})
find_package(CUDA)
set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -Xcompiler -fPIC -std=c++11 -D_FORCE_INLINES -arch sm_60 --expt-extended-lambda")
......@@ -105,8 +95,28 @@ else()
set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -O0 -g")
endif()
# Ensure that a default make is set
if("${MAKE}" STREQUAL "")
if(NOT MSVC)
find_program(MAKE make)
endif()
endif()
find_path(MYSQL_INCLUDE_DIR
NAMES "mysql.h"
PATH_SUFFIXES "mysql")
if (${MYSQL_INCLUDE_DIR} STREQUAL "MYSQL_INCLUDE_DIR-NOTFOUND")
message(FATAL_ERROR "Could not found MySQL include directory")
else()
include_directories(${MYSQL_INCLUDE_DIR})
endif()
set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_CURRENT_SOURCE_DIR}/cmake")
set(MILVUS_SOURCE_DIR ${PROJECT_SOURCE_DIR})
set(MILVUS_BINARY_DIR ${PROJECT_BINARY_DIR})
set(MILVUS_ENGINE_SRC ${PROJECT_SOURCE_DIR}/src)
include(ExternalProject)
include(DefineOptions)
include(BuildUtils)
......@@ -114,7 +124,6 @@ include(ThirdPartyPackages)
config_summary()
set(MILVUS_ENGINE_SRC ${PROJECT_SOURCE_DIR}/src)
add_subdirectory(src)
if (BUILD_UNIT_TEST STREQUAL "ON")
......
......@@ -55,8 +55,6 @@ define_option_string(MILVUS_DEPENDENCY_SOURCE
define_option(MILVUS_VERBOSE_THIRDPARTY_BUILD
"Show output from ExternalProjects rather than just logging to files" ON)
define_option(MILVUS_WITH_ARROW "Build with ARROW" OFF)
define_option(MILVUS_BOOST_VENDORED "Use vendored Boost instead of existing Boost. \
Note that this requires linking Boost statically" ON)
......@@ -66,22 +64,10 @@ define_option(MILVUS_WITH_BZ2 "Build with BZ2 compression" ON)
define_option(MILVUS_WITH_EASYLOGGINGPP "Build with Easylogging++ library" ON)
define_option(MILVUS_WITH_FAISS "Build with FAISS library" OFF)
define_option(MILVUS_WITH_FAISS_GPU_VERSION "Build with FAISS GPU version" OFF)
define_option(MILVUS_WITH_LAPACK "Build with LAPACK library" OFF)
define_option(MILVUS_WITH_LZ4 "Build with lz4 compression" ON)
define_option(MILVUS_WITH_JSONCONS "Build with JSONCONS" OFF)
define_option(MILVUS_WITH_OPENBLAS "Build with OpenBLAS library" OFF)
define_option(MILVUS_WITH_PROMETHEUS "Build with PROMETHEUS library" ON)
define_option(MILVUS_WITH_ROCKSDB "Build with RocksDB library" OFF)
define_option(MILVUS_WITH_SNAPPY "Build with Snappy compression" ON)
define_option(MILVUS_WITH_SQLITE "Build with SQLite library" ON)
......@@ -94,10 +80,6 @@ define_option(MILVUS_WITH_YAMLCPP "Build with yaml-cpp library" ON)
define_option(MILVUS_WITH_ZLIB "Build with zlib compression" ON)
define_option(MILVUS_WITH_KNOWHERE "Build with Knowhere" OFF)
#define_option(MILVUS_ENABLE_PROFILING "Build with profiling" ON)
if(CMAKE_VERSION VERSION_LESS 3.7)
set(MILVUS_WITH_ZSTD_DEFAULT OFF)
else()
......@@ -106,13 +88,13 @@ else()
endif()
define_option(MILVUS_WITH_ZSTD "Build with zstd compression" ${MILVUS_WITH_ZSTD_DEFAULT})
define_option(MILVUS_WITH_AWS "Build with AWS SDK" ON)
if (MILVUS_ENABLE_PROFILING STREQUAL "ON")
define_option(MILVUS_WITH_LIBUNWIND "Build with libunwind" ON)
define_option(MILVUS_WITH_GPERFTOOLS "Build with gperftools" ON)
endif()
define_option(MILVUS_WITH_GRPC "Build with GRPC" ON)
#----------------------------------------------------------------------
if(MSVC)
set_option_category("MSVC")
......
此差异已折叠。
......@@ -17,25 +17,41 @@
# under the License.
#-------------------------------------------------------------------------------
include_directories(${MILVUS_SOURCE_DIR})
include_directories(${MILVUS_ENGINE_SRC})
add_subdirectory(core)
set(CORE_INCLUDE_DIRS ${CORE_INCLUDE_DIRS} PARENT_SCOPE)
foreach(dir ${CORE_INCLUDE_DIRS})
foreach (dir ${CORE_INCLUDE_DIRS})
include_directories(${dir})
endforeach()
endforeach ()
aux_source_directory(cache cache_files)
aux_source_directory(config config_files)
aux_source_directory(server server_files)
aux_source_directory(server/grpc_impl grpcserver_files)
aux_source_directory(utils utils_files)
aux_source_directory(db db_main_files)
aux_source_directory(db/engine db_engine_files)
aux_source_directory(db/insert db_insert_files)
aux_source_directory(db/meta db_meta_files)
aux_source_directory(db/scheduler scheduler_files)
aux_source_directory(db/scheduler/context scheduler_context_files)
aux_source_directory(db/scheduler/task scheduler_task_files)
set(db_scheduler_files
${scheduler_files}
${scheduler_context_files}
${scheduler_task_files}
)
set(grpc_service_files
grpc/gen-milvus/milvus.grpc.pb.cc
grpc/gen-milvus/milvus.pb.cc
grpc/gen-status/status.grpc.pb.cc
grpc/gen-status/status.pb.cc
)
aux_source_directory(metrics metrics_files)
aux_source_directory(wrapper/knowhere knowhere_files)
aux_source_directory(scheduler/action scheduler_action_files)
aux_source_directory(scheduler/event scheduler_event_files)
......@@ -43,7 +59,7 @@ aux_source_directory(scheduler/job scheduler_job_files)
aux_source_directory(scheduler/resource scheduler_resource_files)
aux_source_directory(scheduler/task scheduler_task_files)
aux_source_directory(scheduler scheduler_root_files)
set(scheduler_srcs
set(scheduler_files
${scheduler_action_files}
${scheduler_event_files}
${scheduler_job_files}
......@@ -52,32 +68,12 @@ set(scheduler_srcs
${scheduler_root_files}
)
aux_source_directory(db/scheduler scheduler_files)
aux_source_directory(db/scheduler/context scheduler_context_files)
aux_source_directory(db/scheduler/task scheduler_task_files)
set(db_scheduler_files
${scheduler_files}
${scheduler_context_files}
${scheduler_task_files}
)
set(license_check_files
license/LicenseLibrary.cpp
license/LicenseCheck.cpp
)
aux_source_directory(server server_files)
aux_source_directory(server/grpc_impl grpc_server_files)
set(license_generator_files
license/LicenseGenerator.cpp
license/LicenseLibrary.cpp
)
aux_source_directory(utils utils_files)
set(grpc_service_files
metrics/SystemInfo.cpp
metrics/SystemInfo.h
grpc/gen-milvus/milvus.grpc.pb.cc
grpc/gen-milvus/milvus.pb.cc
grpc/gen-status/status.grpc.pb.cc
grpc/gen-status/status.pb.cc
scheduler/Utils.h)
aux_source_directory(wrapper/knowhere knowhere_files)
set(db_files
${CMAKE_CURRENT_SOURCE_DIR}/main.cpp
......@@ -88,17 +84,12 @@ set(db_files
${db_meta_files}
${db_scheduler_files}
${metrics_files}
${knowhere_files}
${utils_files}
${knowhere_files}
)
set(s3_client_files
storage/s3/S3ClientWrapper.cpp
storage/s3/S3ClientWrapper.h)
include_directories(/usr/include)
include_directories("${CUDA_TOOLKIT_ROOT_DIR}/include")
include_directories(/usr/include/mysql)
include_directories(grpc/gen-status)
include_directories(grpc/gen-milvus)
......@@ -109,17 +100,22 @@ set(client_grpc_lib
grpc_protobuf
grpc_protoc)
set(third_party_libs
knowhere
sqlite
${client_grpc_lib}
yaml-cpp
set(prometheus_lib
prometheus-cpp-push
prometheus-cpp-pull
prometheus-cpp-core
prometheus-cpp-core)
set(boost_lib
boost_system_static
boost_filesystem_static
boost_serialization_static
boost_serialization_static)
set(third_party_libs
sqlite
${client_grpc_lib}
yaml-cpp
${prometheus_lib}
${boost_lib}
bzip2
lz4
snappy
......@@ -131,13 +127,12 @@ set(third_party_libs
${CUDA_TOOLKIT_ROOT_DIR}/lib64/stubs/libnvidia-ml.so
cudart
)
if (MILVUS_ENABLE_PROFILING STREQUAL "ON")
set(third_party_libs ${third_party_libs}
gperftools
libunwind)
endif()
libunwind
)
endif ()
link_directories("${CUDA_TOOLKIT_ROOT_DIR}/lib64")
set(engine_libs
......@@ -149,12 +144,11 @@ set(engine_libs
if (NOT ${CMAKE_SYSTEM_PROCESSOR} MATCHES "aarch64")
set(engine_libs
${engine_libs}
libquadmath.a
)
${engine_libs}
libquadmath.a
)
endif ()
cuda_add_library(milvus_engine STATIC ${db_files})
target_link_libraries(milvus_engine ${engine_libs} ${third_party_libs})
......@@ -162,9 +156,7 @@ add_library(metrics STATIC ${metrics_files})
set(metrics_lib
yaml-cpp
prometheus-cpp-push
prometheus-cpp-pull
prometheus-cpp-core
${prometheus_lib}
)
target_link_libraries(metrics ${metrics_lib})
......@@ -176,21 +168,17 @@ set(server_libs
metrics
)
set(knowhere_libs
knowhere
)
add_executable(milvus_server
${config_files}
${metrics_files}
${scheduler_files}
${server_files}
${grpcserver_files}
${utils_files}
${grpc_server_files}
${grpc_service_files}
${metrics_files}
${scheduler_srcs}
${utils_files}
)
target_link_libraries(milvus_server ${server_libs} ${knowhere_libs} ${third_party_libs})
target_link_libraries(milvus_server ${server_libs} knowhere ${third_party_libs})
install(TARGETS milvus_server DESTINATION bin)
......
......@@ -20,11 +20,6 @@
aux_source_directory(src src_files)
include_directories(src)
include_directories(../../include)
link_directories(${CMAKE_BINARY_DIR})
add_executable(sdk_simple
main.cpp
${src_files}
......
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#pragma once
#include "db/Status.h"
#include <string>
namespace zilliz {
namespace milvus {
namespace engine {
namespace storage {
class IStorage {
public:
virtual Status Create(const std::string &ip_address,
const std::string &port,
const std::string &access_key,
const std::string &secret_key) = 0;
virtual Status Close() = 0;
virtual Status CreateBucket(std::string& bucket_name) = 0;
virtual Status DeleteBucket(std::string& bucket_name) = 0;
virtual Status UploadFile(std::string &bucket_name, std::string &object_key, std::string &path_key) = 0;
virtual Status DownloadFile(std::string &bucket_name, std::string &object_key, std::string &path_key) = 0;
virtual Status DeleteFile(std::string &bucket_name, std::string &object_key) = 0;
};
}
}
}
}
\ No newline at end of file
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
//#include "S3ClientWrapper.h"
//
//#include <aws/s3/model/CreateBucketRequest.h>
//#include <aws/s3/model/DeleteBucketRequest.h>
//#include <aws/s3/model/PutObjectRequest.h>
//#include <aws/s3/model/GetObjectRequest.h>
//#include <aws/s3/model/DeleteObjectRequest.h>
//
//#include <iostream>
//#include <fstream>
//
//
//namespace zilliz {
//namespace milvus {
//namespace engine {
//namespace storage {
//
//Status
//S3ClientWrapper::Create(const std::string &ip_address,
// const std::string &port,
// const std::string &access_key,
// const std::string &secret_key) {
// Aws::InitAPI(options_);
// Aws::Client::ClientConfiguration cfg;
//
// // TODO: ip_address need to be validated.
//
// cfg.endpointOverride = ip_address + ":" + port; // S3 server ip address and port
// cfg.scheme = Aws::Http::Scheme::HTTP;
// cfg.verifySSL =
// false; //Aws::Auth::AWSCredentials cred("RPW421T9GSIO4A45Y9ZR", "2owKYy9emSS90Q0pXuyqpX1OxBCyEDYodsiBemcq"); // 认证的Key
// client_ =
// new S3Client(Aws::Auth::AWSCredentials(access_key, secret_key),
// cfg,
// Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Always,
// false);
// if (client_ == nullptr) {
// std::string error = "Can't connect server.";
// return Status::Error(error);
// } else {
// return Status::OK();
// }
//}
//
//
//Status
//S3ClientWrapper::Close() {
// if (client_ != nullptr) {
// delete client_;
// client_ = nullptr;
// }
// Aws::ShutdownAPI(options_);
// return Status::OK();
//}
//
//Status
//S3ClientWrapper::CreateBucket(std::string& bucket_name) {
// Aws::S3::Model::CreateBucketRequest request;
// request.SetBucket(bucket_name);
//
// auto outcome = client_->CreateBucket(request);
//
// if (outcome.IsSuccess())
// {
// return Status::OK();
// }
// else
// {
// std::cout << "CreateBucket error: "
// << outcome.GetError().GetExceptionName() << std::endl
// << outcome.GetError().GetMessage() << std::endl;
// switch(outcome.GetError().GetErrorType()) {
// case Aws::S3::S3Errors::BUCKET_ALREADY_EXISTS:
// case Aws::S3::S3Errors::BUCKET_ALREADY_OWNED_BY_YOU:
// return Status::AlreadyExist(outcome.GetError().GetMessage());
// default:
// return Status::Error(outcome.GetError().GetMessage());
// }
// }
//}
//
//Status
//S3ClientWrapper::DeleteBucket(std::string& bucket_name) {
// Aws::S3::Model::DeleteBucketRequest bucket_request;
// bucket_request.SetBucket(bucket_name);
//
// auto outcome = client_->DeleteBucket(bucket_request);
//
// if (outcome.IsSuccess())
// {
// return Status::OK();
// }
// else
// {
// std::cout << "DeleteBucket error: "
// << outcome.GetError().GetExceptionName() << " - "
// << outcome.GetError().GetMessage() << std::endl;
// return Status::Error(outcome.GetError().GetMessage());
// }
//}
//
//Status
//S3ClientWrapper::UploadFile(std::string &BucketName, std::string &objectKey, std::string &pathkey) {
//
// PutObjectRequest putObjectRequest;
// putObjectRequest.WithBucket(BucketName.c_str()).WithKey(objectKey.c_str());
//
// auto input_data = Aws::MakeShared<Aws::FStream>("PutObjectInputStream",
// pathkey.c_str(),
// std::ios_base::in | std::ios_base::binary);
// putObjectRequest.SetBody(input_data);
// auto put_object_result = client_->PutObject(putObjectRequest);
// if (put_object_result.IsSuccess()) {
// return Status::OK();
// } else {
// std::cout << "PutObject error: " << put_object_result.GetError().GetExceptionName() << " "
// << put_object_result.GetError().GetMessage() << std::endl;
// return Status::Error(put_object_result.GetError().GetMessage());
// }
//}
//
//Status
//S3ClientWrapper::DownloadFile(std::string &BucketName, std::string &objectKey, std::string &pathkey) {
// GetObjectRequest object_request;
// object_request.WithBucket(BucketName.c_str()).WithKey(objectKey.c_str());
// auto get_object_outcome = client_->GetObject(object_request);
// if (get_object_outcome.IsSuccess()) {
// Aws::OFStream local_file(pathkey.c_str(), std::ios::out | std::ios::binary);
// local_file << get_object_outcome.GetResult().GetBody().rdbuf();
// return Status::OK();
// } else {
// std::cout << "GetObject error: " << get_object_outcome.GetError().GetExceptionName() << " "
// << get_object_outcome.GetError().GetMessage() << std::endl;
// return Status::Error(get_object_outcome.GetError().GetMessage());
// }
//}
//
//Status
//S3ClientWrapper::DeleteFile(std::string &bucket_name, std::string &object_key) {
// Aws::S3::Model::DeleteObjectRequest object_request;
// object_request.WithBucket(bucket_name).WithKey(object_key);
//
// auto delete_object_outcome = client_->DeleteObject(object_request);
//
// if (delete_object_outcome.IsSuccess()) {
// return Status::OK();
// } else {
// std::cout << "DeleteObject error: " <<
// delete_object_outcome.GetError().GetExceptionName() << " " <<
// delete_object_outcome.GetError().GetMessage() << std::endl;
//
// return Status::Error(delete_object_outcome.GetError().GetMessage());
// }
//}
//
//}
//}
//}
//}
\ No newline at end of file
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
//#pragma once
//
//#include "storage/IStorage.h"
//
//
//#include <aws/s3/S3Client.h>
//#include <aws/core/Aws.h>
//#include <aws/core/auth/AWSCredentialsProvider.h>
//
//
//using namespace Aws::S3;
//using namespace Aws::S3::Model;
//
//namespace zilliz {
//namespace milvus {
//namespace engine {
//namespace storage {
//
//class S3ClientWrapper : public IStorage {
// public:
//
// S3ClientWrapper() = default;
// ~S3ClientWrapper() = default;
//
// Status Create(const std::string &ip_address,
// const std::string &port,
// const std::string &access_key,
// const std::string &secret_key) override;
// Status Close() override;
//
// Status CreateBucket(std::string& bucket_name) override;
// Status DeleteBucket(std::string& bucket_name) override;
// Status UploadFile(std::string &BucketName, std::string &objectKey, std::string &pathkey) override;
// Status DownloadFile(std::string &BucketName, std::string &objectKey, std::string &pathkey) override;
// Status DeleteFile(std::string &bucket_name, std::string &object_key) override;
//
// private:
// S3Client *client_ = nullptr;
// Aws::SDKOptions options_;
//};
//
//}
//}
//}
//}
\ No newline at end of file
ARROW_VERSION=zilliz
BOOST_VERSION=1.70.0
BZIP2_VERSION=1.0.6
EASYLOGGINGPP_VERSION=v9.96.7
FAISS_VERSION=branch-0.1.0
MKL_VERSION=2019.4.243
GTEST_VERSION=1.8.1
JSONCONS_VERSION=0.126.0
LAPACK_VERSION=v3.8.0
LZ4_VERSION=v1.9.1
MYSQLPP_VERSION=3.2.4
OPENBLAS_VERSION=v0.3.6
PROMETHEUS_VERSION=v0.7.0
ROCKSDB_VERSION=v6.0.2
SNAPPY_VERSION=1.1.7
SQLITE_VERSION=3280000
SQLITE_ORM_VERSION=master
YAMLCPP_VERSION=0.6.2
ZLIB_VERSION=v1.2.11
ZSTD_VERSION=v1.4.0
AWS_VERSION=1.7.125
LIBUNWIND_VERSION=1.3.1
GPERFTOOLS_VERSION=2.7
GRPC_VERSION=master
......
......@@ -17,20 +17,70 @@
# under the License.
#-------------------------------------------------------------------------------
link_directories(
"${CMAKE_BINARY_DIR}/lib"
)
include_directories(${MILVUS_SOURCE_DIR})
include_directories(${MILVUS_ENGINE_SRC})
aux_source_directory(${MILVUS_ENGINE_SRC}/cache cache_files)
aux_source_directory(${MILVUS_ENGINE_SRC}/config config_files)
aux_source_directory(${MILVUS_ENGINE_SRC}/db db_main_files)
aux_source_directory(${MILVUS_ENGINE_SRC}/db/engine db_engine_files)
aux_source_directory(${MILVUS_ENGINE_SRC}/db/insert db_insert_files)
aux_source_directory(${MILVUS_ENGINE_SRC}/db/meta db_meta_files)
aux_source_directory(${MILVUS_ENGINE_SRC}/db/scheduler scheduler_files)
aux_source_directory(${MILVUS_ENGINE_SRC}/db/scheduler/context scheduler_context_files)
aux_source_directory(${MILVUS_ENGINE_SRC}/db/scheduler/task scheduler_task_files)
set(db_scheduler_files
${scheduler_files}
${scheduler_context_files}
${scheduler_task_files}
)
set(grpc_service_files
grpc/gen-milvus/milvus.grpc.pb.cc
grpc/gen-milvus/milvus.pb.cc
grpc/gen-status/status.grpc.pb.cc
grpc/gen-status/status.pb.cc
)
aux_source_directory(${MILVUS_ENGINE_SRC}/metrics metrics_files)
aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler/action scheduler_action_files)
aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler/event scheduler_event_files)
aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler/job scheduler_job_files)
aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler/resource scheduler_resource_files)
aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler/task scheduler_task_files)
aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler scheduler_root_files)
set(scheduler_files
${scheduler_action_files}
${scheduler_event_files}
${scheduler_job_files}
${scheduler_resource_files}
${scheduler_task_files}
${scheduler_root_files}
)
aux_source_directory(${MILVUS_ENGINE_SRC}/server server_files)
aux_source_directory(${MILVUS_ENGINE_SRC}/server/grpc_impl grpc_server_files)
aux_source_directory(${MILVUS_ENGINE_SRC}/utils utils_files)
aux_source_directory(${MILVUS_ENGINE_SRC}/wrapper/knowhere knowhere_files)
aux_source_directory(${MILVUS_ENGINE_SRC}/db db_srcs)
aux_source_directory(${MILVUS_ENGINE_SRC}/config config_files)
set(unittest_srcs
set(unittest_files
${CMAKE_CURRENT_SOURCE_DIR}/main.cpp)
set(require_files
set(helper_files
${MILVUS_ENGINE_SRC}/server/ServerConfig.cpp
${MILVUS_ENGINE_SRC}/utils/CommonUtil.cpp
${MILVUS_ENGINE_SRC}/utils/TimeRecorder.cpp
${MILVUS_ENGINE_SRC}/utils/Status.cpp
${MILVUS_ENGINE_SRC}/utils/ValidationUtil.cpp
${MILVUS_ENGINE_SRC}/utils/easylogging++.cc
)
set(unittest_libs
......@@ -49,15 +99,16 @@ set(unittest_libs
z
${CUDA_TOOLKIT_ROOT_DIR}/lib64/stubs/libnvidia-ml.so
cudart
cublas
)
foreach(dir ${CORE_INCLUDE_DIRS})
include_directories(${dir})
endforeach()
add_subdirectory(server)
add_subdirectory(db)
add_subdirectory(knowhere)
add_subdirectory(metrics)
add_subdirectory(scheduler)
add_subdirectory(server)
#add_subdirectory(storage)
\ No newline at end of file
......@@ -17,61 +17,27 @@
# under the License.
#-------------------------------------------------------------------------------
aux_source_directory(${MILVUS_ENGINE_SRC}/db db_main_files)
aux_source_directory(${MILVUS_ENGINE_SRC}/db/engine db_engine_files)
aux_source_directory(${MILVUS_ENGINE_SRC}/db/insert db_insert_files)
aux_source_directory(${MILVUS_ENGINE_SRC}/db/meta db_meta_files)
aux_source_directory(${MILVUS_ENGINE_SRC}/config config_files)
aux_source_directory(${MILVUS_ENGINE_SRC}/cache cache_srcs)
aux_source_directory(${MILVUS_ENGINE_SRC}/wrapper/knowhere knowhere_src)
aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler/action scheduler_action_srcs)
aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler/event scheduler_event_srcs)
aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler/resource scheduler_resource_srcs)
aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler/task scheduler_task_srcs)
aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler scheduler_srcs)
aux_source_directory(./ test_srcs)
set(util_files
${MILVUS_ENGINE_SRC}/utils/Status.cpp
${MILVUS_ENGINE_SRC}/utils/ValidationUtil.cpp
${MILVUS_ENGINE_SRC}/utils/easylogging++.cc)
aux_source_directory(${MILVUS_ENGINE_SRC}/db/scheduler scheduler_files)
aux_source_directory(${MILVUS_ENGINE_SRC}/db/scheduler/context scheduler_context_files)
aux_source_directory(${MILVUS_ENGINE_SRC}/db/scheduler/task scheduler_task_files)
set(db_scheduler_srcs
aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR} test_files)
include_directories("${CUDA_TOOLKIT_ROOT_DIR}/include")
link_directories("${CUDA_TOOLKIT_ROOT_DIR}/lib64")
set(db_test_files
${cache_files}
${config_files}
${db_main_files}
${db_engine_files}
${db_insert_files}
${db_meta_files}
${db_scheduler_files}
${metrics_files}
${scheduler_files}
${scheduler_context_files}
${scheduler_task_files}
)
include_directories(/usr/local/cuda/include)
link_directories("/usr/local/cuda/lib64")
include_directories(/usr/include/mysql)
set(db_test_src
${config_files}
${cache_srcs}
${db_main_files}
${db_engine_files}
${db_insert_files}
${db_meta_files}
${db_scheduler_srcs}
${wrapper_src}
${scheduler_action_srcs}
${scheduler_event_srcs}
${scheduler_resource_srcs}
${scheduler_task_srcs}
${scheduler_srcs}
${knowhere_src}
${util_files}
${require_files}
${test_srcs}
${knowhere_files}
${helper_files}
${test_files}
)
cuda_add_executable(db_test ${db_test_src})
cuda_add_executable(db_test ${db_test_files})
set(db_libs
sqlite
......@@ -81,13 +47,7 @@ set(db_libs
mysqlpp
)
set(knowhere_libs
knowhere
cudart
cublas
)
target_link_libraries(db_test ${db_libs} ${knowhere_libs} ${unittest_libs})
target_link_libraries(db_test ${db_libs} knowhere ${unittest_libs})
install(TARGETS db_test DESTINATION unittest)
......@@ -20,15 +20,15 @@
include_directories("${CUDA_TOOLKIT_ROOT_DIR}/include")
link_directories("${CUDA_TOOLKIT_ROOT_DIR}/lib64")
set(knowhere_src
set(knowhere_files
${MILVUS_ENGINE_SRC}/wrapper/knowhere/data_transfer.cpp
${MILVUS_ENGINE_SRC}/wrapper/knowhere/vec_impl.cpp
${MILVUS_ENGINE_SRC}/wrapper/knowhere/vec_index.cpp)
set(helper
set(helper_files
utils.cpp
${MILVUS_ENGINE_SRC}/utils/easylogging++.cc
${MILVUS_ENGINE_SRC}/utils/easylogging++.h)
)
set(knowhere_libs
knowhere
......@@ -36,7 +36,7 @@ set(knowhere_libs
cublas
)
add_executable(knowhere_test knowhere_test.cpp ${knowhere_src} ${helper})
add_executable(knowhere_test knowhere_test.cpp ${knowhere_files} ${helper_files})
target_link_libraries(knowhere_test ${knowhere_libs} ${unittest_libs})
install(TARGETS knowhere_test DESTINATION unittest)
\ No newline at end of file
......@@ -17,86 +17,35 @@
# under the License.
#-------------------------------------------------------------------------------
include_directories(../../src)
aux_source_directory(${MILVUS_ENGINE_SRC}/db db_main_files)
aux_source_directory(${MILVUS_ENGINE_SRC}/db/engine db_engine_files)
aux_source_directory(${MILVUS_ENGINE_SRC}/db/insert db_insert_files)
aux_source_directory(${MILVUS_ENGINE_SRC}/db/meta db_meta_files)
aux_source_directory(${MILVUS_ENGINE_SRC}/config config_files)
aux_source_directory(${MILVUS_ENGINE_SRC}/cache cache_srcs)
aux_source_directory(${MILVUS_ENGINE_SRC}/wrapper wrapper_src)
aux_source_directory(${MILVUS_ENGINE_SRC}/wrapper/knowhere knowhere_src)
aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler/action scheduler_action_srcs)
aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler/event scheduler_event_srcs)
aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler/resource scheduler_resource_srcs)
aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler/task scheduler_task_srcs)
aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler scheduler_srcs)
aux_source_directory(${MILVUS_ENGINE_SRC}/src/metrics metrics_src)
aux_source_directory(./ test_srcs)
set(util_files
${MILVUS_ENGINE_SRC}/utils/Status.cpp
${MILVUS_ENGINE_SRC}/utils/ValidationUtil.cpp
${MILVUS_ENGINE_SRC}/utils/easylogging++.cc)
aux_source_directory(${MILVUS_ENGINE_SRC}/db/scheduler scheduler_files)
aux_source_directory(${MILVUS_ENGINE_SRC}/db/scheduler/context scheduler_context_files)
aux_source_directory(${MILVUS_ENGINE_SRC}/db/scheduler/task scheduler_task_files)
set(db_scheduler_srcs
${scheduler_files}
${scheduler_context_files}
${scheduler_task_files}
)
aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR} test_files)
include_directories(/usr/include)
include_directories(../../third_party/build/include)
link_directories(../../third_party/build/lib)
include_directories(/usr/local/cuda/include)
link_directories("/usr/local/cuda/lib64")
include_directories(../../src/metrics)
include_directories("${CUDA_TOOLKIT_ROOT_DIR}/include")
link_directories("${CUDA_TOOLKIT_ROOT_DIR}/lib64")
include_directories(/usr/include/mysql)
set(count_test_src
set(metrics_test_files
${cache_files}
${config_files}
${cache_srcs}
${db_main_files}
${db_engine_files}
${db_insert_files}
${db_meta_files}
${db_scheduler_srcs}
${wrapper_src}
${scheduler_action_srcs}
${scheduler_event_srcs}
${scheduler_resource_srcs}
${scheduler_task_srcs}
${scheduler_srcs}
${knowhere_src}
${metrics_src}
${test_srcs}
${util_files}
${db_scheduler_files}
${metrics_files}
${scheduler_files}
${knowhere_files}
${helper_files}
${test_files}
)
add_executable(metrics_test ${count_test_src} ${require_files} )
set(knowhere_libs
knowhere
)
add_executable(metrics_test ${metrics_test_files})
target_link_libraries(metrics_test
${knowhere_libs}
cudart
cublas
knowhere
sqlite
boost_system_static
boost_filesystem_static
lz4
metrics
gtest
pthread
z
mysqlpp
${unittest_libs}
)
......
......@@ -17,62 +17,27 @@
# under the License.
#-------------------------------------------------------------------------------
aux_source_directory(${MILVUS_ENGINE_SRC}/db db_main_files)
aux_source_directory(${MILVUS_ENGINE_SRC}/db/engine db_engine_files)
aux_source_directory(${MILVUS_ENGINE_SRC}/db/insert db_insert_files)
aux_source_directory(${MILVUS_ENGINE_SRC}/db/meta db_meta_files)
aux_source_directory(${MILVUS_ENGINE_SRC}/config config_files)
aux_source_directory(${MILVUS_ENGINE_SRC}/cache cache_srcs)
aux_source_directory(${MILVUS_ENGINE_SRC}/wrapper/knowhere knowhere_src)
aux_source_directory(${MILVUS_ENGINE_SRC}/db/scheduler scheduler_files)
aux_source_directory(${MILVUS_ENGINE_SRC}/db/scheduler/context scheduler_context_files)
aux_source_directory(${MILVUS_ENGINE_SRC}/db/scheduler/task scheduler_task_files)
aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR} test_files)
aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler/action scheduler_action_srcs)
aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler/event scheduler_event_srcs)
aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler/resource scheduler_resource_srcs)
aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler/task scheduler_task_srcs)
aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler scheduler_srcs)
aux_source_directory(./ test_srcs)
set(util_files
${MILVUS_ENGINE_SRC}/utils/Status.cpp
${MILVUS_ENGINE_SRC}/utils/ValidationUtil.cpp
${MILVUS_ENGINE_SRC}/utils/easylogging++.cc)
set(db_scheduler_srcs
${scheduler_files}
${scheduler_context_files}
${scheduler_task_files})
include_directories(/usr/local/cuda/include)
link_directories("/usr/local/cuda/lib64")
include_directories(/usr/include/mysql)
set(scheduler_test_src
${unittest_srcs}
${test_srcs}
${scheduler_action_srcs}
${scheduler_event_srcs}
${scheduler_resource_srcs}
${scheduler_task_srcs}
${scheduler_srcs}
include_directories("${CUDA_TOOLKIT_ROOT_DIR}/include")
link_directories("${CUDA_TOOLKIT_ROOT_DIR}/lib64")
set(scheduler_test_files
${cache_files}
${config_files}
${cache_srcs}
${db_main_files}
${db_engine_files}
${db_insert_files}
${db_meta_files}
${db_scheduler_srcs}
${wrapper_src}
${knowhere_src}
${util_files}
${require_files}
${db_scheduler_files}
${scheduler_files}
${knowhere_files}
${helper_files}
${unittest_files}
${test_files}
)
cuda_add_executable(scheduler_test ${scheduler_test_src})
cuda_add_executable(scheduler_test ${scheduler_test_files})
set(scheduler_libs
sqlite
......@@ -82,14 +47,7 @@ set(scheduler_libs
mysqlpp
)
set(knowhere_libs
knowhere
cudart
cublas
)
target_link_libraries(scheduler_test ${scheduler_libs} ${knowhere_libs} ${unittest_libs})
target_link_libraries(scheduler_test ${scheduler_libs} knowhere ${unittest_libs})
install(TARGETS scheduler_test DESTINATION unittest)
......@@ -17,75 +17,19 @@
# under the License.
#-------------------------------------------------------------------------------
include_directories(${MILVUS_ENGINE_SRC}/)
include_directories(/usr/include)
include_directories(/usr/include/mysql)
include_directories(/usr/local/cuda/include)
link_directories(/usr/local/cuda/lib64)
aux_source_directory(${MILVUS_ENGINE_SRC}/db db_main_files)
aux_source_directory(${MILVUS_ENGINE_SRC}/db/engine db_engine_files)
aux_source_directory(${MILVUS_ENGINE_SRC}/db/insert db_insert_files)
aux_source_directory(${MILVUS_ENGINE_SRC}/db/meta db_meta_files)
aux_source_directory(${MILVUS_ENGINE_SRC}/config config_files)
aux_source_directory(${MILVUS_ENGINE_SRC}/cache cache_srcs)
aux_source_directory(${MILVUS_ENGINE_SRC}/wrapper/knowhere knowhere_src)
aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler/action scheduler_action_srcs)
aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler/event scheduler_event_srcs)
aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler/resource scheduler_resource_srcs)
aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler/task scheduler_task_srcs)
aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler scheduler_srcs)
aux_source_directory(${MILVUS_ENGINE_SRC}/server server_src)
aux_source_directory(${MILVUS_ENGINE_SRC}/server/grpc_impl grpc_server_src)
aux_source_directory(${MILVUS_ENGINE_SRC}/db/scheduler scheduler_files)
aux_source_directory(${MILVUS_ENGINE_SRC}/db/scheduler/context scheduler_context_files)
aux_source_directory(${MILVUS_ENGINE_SRC}/db/scheduler/task scheduler_task_files)
aux_source_directory(./ test_srcs)
set(db_scheduler_srcs
${scheduler_files}
${scheduler_context_files}
${scheduler_task_files}
)
aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR} test_files)
set(util_files
${MILVUS_ENGINE_SRC}/utils/Status.cpp
${MILVUS_ENGINE_SRC}/utils/ValidationUtil.cpp
${MILVUS_ENGINE_SRC}/utils/easylogging++.cc)
include_directories(/usr/include)
include_directories("${CUDA_TOOLKIT_ROOT_DIR}/include")
link_directories("${CUDA_TOOLKIT_ROOT_DIR}/lib64")
set(db_src
${config_files}
${cache_srcs}
${db_main_files}
${db_engine_files}
${db_insert_files}
${db_meta_files}
${db_scheduler_srcs}
${wrapper_src}
${scheduler_action_srcs}
${scheduler_event_srcs}
${scheduler_resource_srcs}
${scheduler_task_srcs}
${scheduler_srcs}
${knowhere_src}
${util_files}
${require_files}
${test_srcs}
)
include_directories(${MILVUS_ENGINE_SRC}/grpc/gen-status)
include_directories(${MILVUS_ENGINE_SRC}/grpc/gen-milvus)
set(utils_srcs
set(helper_files ${helper_files}
${MILVUS_ENGINE_SRC}/utils/StringHelpFunctions.cpp
${MILVUS_ENGINE_SRC}/utils/TimeRecorder.cpp
${MILVUS_ENGINE_SRC}/utils/CommonUtil.cpp
${MILVUS_ENGINE_SRC}/utils/LogUtil.cpp
${MILVUS_ENGINE_SRC}/utils/ValidationUtil.cpp
${MILVUS_ENGINE_SRC}/utils/SignalUtil.cpp
)
${MILVUS_ENGINE_SRC}/utils/SignalUtil.cpp)
set(grpc_service_files
${MILVUS_ENGINE_SRC}/grpc/gen-milvus/milvus.grpc.pb.cc
......@@ -94,44 +38,49 @@ set(grpc_service_files
${MILVUS_ENGINE_SRC}/grpc/gen-status/status.pb.cc
)
include_directories(${MILVUS_ENGINE_SRC}/grpc/gen-status)
include_directories(${MILVUS_ENGINE_SRC}/grpc/gen-milvus)
cuda_add_executable(server_test
${db_src}
${unittest_srcs}
${grpc_server_src}
${server_src}
${utils_srcs}
set(server_test_files
${cache_files}
${config_files}
${db_main_files}
${db_engine_files}
${db_insert_files}
${db_meta_files}
${db_scheduler_files}
${scheduler_files}
${server_files}
${grpc_server_files}
${grpc_service_files}
${require_files}
${knowhere_files}
${helper_files}
${unittest_files}
${test_files}
)
cuda_add_executable(server_test ${server_test_files})
set(client_grpc_lib
grpcpp_channelz
grpc++
grpc
grpc_protobuf
grpc_protoc)
set(require_libs
knowhere
stdc++
cudart
cublas
sqlite
mysqlpp
boost_system_static
boost_filesystem_static
snappy
z
bz2
zstd
lz4
pthread
grpcpp_channelz
grpc++
grpc
grpc_protobuf
grpc_protoc
${client_grpc_lib}
)
target_link_libraries(server_test
${require_libs}
${cuda_library}
${unittest_libs}
)
......
#-------------------------------------------------------------------------------
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http:#www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
#-------------------------------------------------------------------------------
aux_source_directory(${MILVUS_ENGINE_SRC}/storage/s3 s3_client_src)
set(util_files
${MILVUS_ENGINE_SRC}/utils/ValidationUtil.cpp)
# Make sure that your call to link_directories takes place before your call to the relevant add_executable.
include_directories("${CUDA_TOOLKIT_ROOT_DIR}/include")
link_directories("${CUDA_TOOLKIT_ROOT_DIR}/lib64")
set(s3_client_test_src
${unittest_srcs}
${s3_client_src}
${require_files}
S3ClientTest.cpp
${MILVUS_ENGINE_SRC}/db/Status.cpp
)
add_executable(s3_test
${s3_client_test_src}
${config_files}
${util_files}
)
set(s3_client_libs
stdc++
aws-cpp-sdk-s3
aws-cpp-sdk-core
boost_filesystem_static
)
target_link_libraries(s3_test
${s3_client_libs}
${unittest_libs}
curl
crypto)
install(TARGETS s3_test DESTINATION unittest)
\ No newline at end of file
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
//
//#include "storage/IStorage.h"
//#include "storage/s3/S3ClientWrapper.h"
//#include <gtest/gtest.h>
//#include <memory.h>
//#include <fstream>
//
//
//using namespace zilliz::milvus::engine;
//
//TEST(s3_client_wrapper, CLIENT_WRAPPER_TEST) {
//
// std::shared_ptr<storage::IStorage> storage_ptr = std::make_shared<storage::S3ClientWrapper>();
//
// std::string ip_address = "127.0.0.1";
// std::string port = "9000";
// std::string access_key = "AKIAIOSFODNN7EXAMPLE";
// std::string secret_key = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY";
//
// Status status = storage_ptr->Create(ip_address, port, access_key, secret_key);
// ASSERT_TRUE(status.ok());
//
// std::string filename = "/tmp/s3_test_file";
// std::string bucket_name = "bucktname";
// std::string object_name = "test_file";
//
// status = storage_ptr->CreateBucket(bucket_name);
// std::cout << status.IsAlreadyExist() << std::endl;
// if (status.IsAlreadyExist()) {
// status = storage_ptr->DeleteBucket(bucket_name);
// status = storage_ptr->CreateBucket(bucket_name);
// }
//
// ASSERT_TRUE(status.ok());
//
// std::ofstream ofile(filename);
// std::stringstream ss;
// for (int i = 0; i < 1024; ++i) {
// ss << i;
// }
// ofile << ss.str() << std::endl;
// ofile.close();
// status = storage_ptr->UploadFile(bucket_name, object_name, filename);
// ASSERT_TRUE(status.ok());
//
// status = storage_ptr->DownloadFile(bucket_name, object_name, filename);
// std::ifstream infile(filename);
// std::string in_buffer;
// infile >> in_buffer;
// ASSERT_STREQ(in_buffer.c_str(), ss.str().c_str());
//
// status = storage_ptr->DeleteFile(bucket_name, object_name);
// ASSERT_TRUE(status.ok());
//
// status = storage_ptr->DeleteBucket(bucket_name);
// ASSERT_TRUE(status.ok());
//
// status = storage_ptr->Close();
// ASSERT_TRUE(status.ok());
//}
//
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册