diff --git a/cpp/CHANGELOG.md b/cpp/CHANGELOG.md index 91f73067c555eee50bcf1c480f7f340ab9a0eeb0..0b257f2a03b740e699d4af5b884adcb9cca61194 100644 --- a/cpp/CHANGELOG.md +++ b/cpp/CHANGELOG.md @@ -35,6 +35,7 @@ Please mark all change in change log and use the ticket from JIRA. - MS-84 - cmake: add arrow, jemalloc and jsoncons third party; default build option OFF - MS-85 - add NetIO metric - MS-96 - add new query interface for specified files +- MS-97 - Add S3 SDK for MinIO Storage ## Task - MS-74 - Change README.md in cpp diff --git a/cpp/cmake/DefineOptions.cmake b/cpp/cmake/DefineOptions.cmake index 09998c480cc75959ee5fc4b85b50971bd448bbb1..d95e7c7ed19081273c13513b2c81e1875b6fea4a 100644 --- a/cpp/cmake/DefineOptions.cmake +++ b/cpp/cmake/DefineOptions.cmake @@ -107,6 +107,8 @@ else() endif() define_option(MILVUS_WITH_ZSTD "Build with zstd compression" ${MILVUS_WITH_ZSTD_DEFAULT}) +define_option(MILVUS_WITH_AWS "Build with AWS SDK" ON) + #---------------------------------------------------------------------- if(MSVC) set_option_category("MSVC") diff --git a/cpp/cmake/ThirdPartyPackages.cmake b/cpp/cmake/ThirdPartyPackages.cmake index 147fed8e0fd6978eed8658c3bf61816a72503bf0..429d6c9d44dde477dc8bb630486ac849c70559e5 100644 --- a/cpp/cmake/ThirdPartyPackages.cmake +++ b/cpp/cmake/ThirdPartyPackages.cmake @@ -35,7 +35,8 @@ set(MILVUS_THIRDPARTY_DEPENDENCIES Thrift yaml-cpp ZLIB - ZSTD) + ZSTD + AWS) message(STATUS "Using ${MILVUS_DEPENDENCY_SOURCE} approach to find dependencies") @@ -83,6 +84,8 @@ macro(build_dependency DEPENDENCY_NAME) build_zlib() elseif("${DEPENDENCY_NAME}" STREQUAL "ZSTD") build_zstd() + elseif("${DEPENDENCY_NAME}" STREQUAL "AWS") + build_aws() else() message(FATAL_ERROR "Unknown thirdparty dependency to build: ${DEPENDENCY_NAME}") endif () @@ -330,6 +333,11 @@ else() set(ZSTD_SOURCE_URL "https://github.com/facebook/zstd/archive/${ZSTD_VERSION}.tar.gz") endif() +if(DEFINED ENV{MILVUS_AWS_URL}) + set(AWS_SOURCE_URL "$ENV{MILVUS_AWS_URL}") +else() + set(AWS_SOURCE_URL "https://github.com/aws/aws-sdk-cpp/archive/${AWS_VERSION}.tar.gz") +endif() # ---------------------------------------------------------------------- # ARROW @@ -1736,3 +1744,68 @@ if(MILVUS_WITH_ZSTD) link_directories(SYSTEM ${ZSTD_PREFIX}/lib) include_directories(SYSTEM ${ZSTD_INCLUDE_DIR}) endif() + +# ---------------------------------------------------------------------- +# aws +macro(build_aws) + message(STATUS "Building aws-${AWS_VERSION} from source") + set(AWS_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/aws_ep-prefix/src/aws_ep") + + set(AWS_CMAKE_ARGS + ${EP_COMMON_TOOLCHAIN} + "-DCMAKE_INSTALL_PREFIX=${AWS_PREFIX}" + -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} + -DCMAKE_INSTALL_LIBDIR=lib #${CMAKE_INSTALL_LIBDIR} + -DBUILD_ONLY=s3 + -DBUILD_SHARED_LIBS=off + -DENABLE_TESTING=off + -DENABLE_UNITY_BUILD=on + -DNO_ENCRYPTION=off) + + set(AWS_STATIC_LIB "${AWS_PREFIX}/lib/libs3.a") + # Only pass our C flags on Unix as on MSVC it leads to a + # "incompatible command-line options" error + set(AWS_CMAKE_ARGS + ${AWS_CMAKE_ARGS} + -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} + -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} + -DCMAKE_C_FLAGS=${EP_C_FLAGS} + -DCMAKE_CXX_FLAGS=${EP_CXX_FLAGS}) + + if(CMAKE_VERSION VERSION_LESS 3.7) + message(FATAL_ERROR "Building AWS using ExternalProject requires at least CMake 3.7") + endif() + + externalproject_add(aws_ep + ${EP_LOG_OPTIONS} + CMAKE_ARGS + ${AWS_CMAKE_ARGS} + BUILD_COMMAND + ${MAKE} + ${MAKE_BUILD_ARGS} + INSTALL_DIR + ${AWS_PREFIX} + URL + ${AWS_SOURCE_URL} + BUILD_BYPRODUCTS + "${AWS_STATIC_LIB}") + + + file(MAKE_DIRECTORY "${AWS_PREFIX}/include") + + add_library(aws STATIC IMPORTED) + set_target_properties(aws + PROPERTIES IMPORTED_LOCATION "${AWS_STATIC_LIB}" + INTERFACE_INCLUDE_DIRECTORIES "${AWS_PREFIX}/include") + + add_dependencies(aws aws_ep) +endmacro() + +if(MILVUS_WITH_AWS) + resolve_dependency(AWS) + + # TODO: Don't use global includes but rather target_include_directories + get_target_property(AWS_INCLUDE_DIR aws INTERFACE_INCLUDE_DIRECTORIES) + link_directories(SYSTEM ${AWS_PREFIX}/lib) + include_directories(SYSTEM ${AWS_INCLUDE_DIR}) +endif() diff --git a/cpp/src/CMakeLists.txt b/cpp/src/CMakeLists.txt index d81ccc5da2dd53643824548fbe01f2c0becb5344..4645ffcbe60828f8ff0695ee0eb57d98e0168021 100644 --- a/cpp/src/CMakeLists.txt +++ b/cpp/src/CMakeLists.txt @@ -14,12 +14,6 @@ aux_source_directory(db/scheduler db_scheduler_files) aux_source_directory(wrapper wrapper_files) aux_source_directory(metrics metrics_files) -#set(metrics_files -# metrics/Metrics.cpp -# metrics/MetricBase.h -#) - - set(license_check_files license/LicenseLibrary.cpp license/LicenseCheck.cpp @@ -53,6 +47,10 @@ set(engine_files set(get_sys_info_files license/GetSysInfo.cpp) +set(s3_client_files + storage/s3/S3ClientWrapper.cpp + storage/s3/S3ClientWrapper.h) + include_directories(/usr/include) include_directories("${CUDA_TOOLKIT_ROOT_DIR}/include") include_directories(thrift/gen-cpp) diff --git a/cpp/src/db/Status.h b/cpp/src/db/Status.h index ded32b339415c1c2707cf4f8315814ea92d9546c..52123eb8ce0aea593f6d42098adb74364f41391c 100644 --- a/cpp/src/db/Status.h +++ b/cpp/src/db/Status.h @@ -7,39 +7,44 @@ #include + namespace zilliz { namespace milvus { namespace engine { class Status { -public: + public: Status() noexcept : state_(nullptr) {} ~Status() { delete[] state_; } - Status(const Status& rhs); - Status& operator=(const Status& rhs); + Status(const Status &rhs); + Status &operator=(const Status &rhs); - Status(Status&& rhs) noexcept : state_(rhs.state_) { rhs.state_ = nullptr; } - Status& operator=(Status&& rhs_) noexcept; + Status(Status &&rhs) noexcept : state_(rhs.state_) { rhs.state_ = nullptr; } + Status &operator=(Status &&rhs_) noexcept; static Status OK() { return Status(); } - static Status NotFound(const std::string& msg, const std::string& msg2="") { + static Status NotFound(const std::string &msg, const std::string &msg2 = "") { return Status(kNotFound, msg, msg2); } - static Status Error(const std::string& msg, const std::string& msg2="") { + static Status Error(const std::string &msg, const std::string &msg2 = "") { return Status(kError, msg, msg2); } - static Status InvalidDBPath(const std::string& msg, const std::string& msg2="") { + static Status InvalidDBPath(const std::string &msg, const std::string &msg2 = "") { return Status(kInvalidDBPath, msg, msg2); } - static Status GroupError(const std::string& msg, const std::string& msg2="") { + static Status GroupError(const std::string &msg, const std::string &msg2 = "") { return Status(kGroupError, msg, msg2); } - static Status DBTransactionError(const std::string& msg, const std::string& msg2="") { + static Status DBTransactionError(const std::string &msg, const std::string &msg2 = "") { return Status(kDBTransactionError, msg, msg2); } + static Status AlreadyExist(const std::string &msg, const std::string &msg2 = "") { + return Status(kAlreadyExist, msg, msg2); + } + bool ok() const { return state_ == nullptr; } bool IsNotFound() const { return code() == kNotFound; } @@ -48,11 +53,12 @@ public: bool IsInvalidDBPath() const { return code() == kInvalidDBPath; } bool IsGroupError() const { return code() == kGroupError; } bool IsDBTransactionError() const { return code() == kDBTransactionError; } + bool IsAlreadyExist() const { return code() == kAlreadyExist; } std::string ToString() const; -private: - const char* state_; + private: + const char *state_ = nullptr; enum Code { kOK = 0, @@ -62,21 +68,23 @@ private: kInvalidDBPath, kGroupError, kDBTransactionError, + + kAlreadyExist, }; Code code() const { return (state_ == nullptr) ? kOK : static_cast(state_[4]); } - Status(Code code, const std::string& msg, const std::string& msg2); - static const char* CopyState(const char* s); + Status(Code code, const std::string &msg, const std::string &msg2); + static const char *CopyState(const char *s); }; // Status -inline Status::Status(const Status& rhs) { +inline Status::Status(const Status &rhs) { state_ = (rhs.state_ == nullptr) ? nullptr : CopyState(rhs.state_); } -inline Status& Status::operator=(const Status& rhs) { +inline Status &Status::operator=(const Status &rhs) { if (state_ != rhs.state_) { delete[] state_; state_ = (rhs.state_ == nullptr) ? nullptr : CopyState(rhs.state_); @@ -84,7 +92,7 @@ inline Status& Status::operator=(const Status& rhs) { return *this; } -inline Status& Status::operator=(Status&& rhs) noexcept { +inline Status &Status::operator=(Status &&rhs) noexcept { std::swap(state_, rhs.state_); return *this; } diff --git a/cpp/src/storage/IStorage.h b/cpp/src/storage/IStorage.h new file mode 100644 index 0000000000000000000000000000000000000000..918834b106b0f295b23d2553e998e6a308815960 --- /dev/null +++ b/cpp/src/storage/IStorage.h @@ -0,0 +1,32 @@ +#pragma once + +#include "db/Status.h" + +#include + + +namespace zilliz { +namespace milvus { +namespace engine { +namespace storage { + +class IStorage { + public: + virtual Status Create(const std::string &ip_address, + const std::string &port, + const std::string &access_key, + const std::string &secret_key) = 0; + + virtual Status Close() = 0; + + virtual Status CreateBucket(std::string& bucket_name) = 0; + virtual Status DeleteBucket(std::string& bucket_name) = 0; + virtual Status UploadFile(std::string &bucket_name, std::string &object_key, std::string &path_key) = 0; + virtual Status DownloadFile(std::string &bucket_name, std::string &object_key, std::string &path_key) = 0; + virtual Status DeleteFile(std::string &bucket_name, std::string &object_key) = 0; +}; + +} +} +} +} \ No newline at end of file diff --git a/cpp/src/storage/s3/S3ClientWrapper.cpp b/cpp/src/storage/s3/S3ClientWrapper.cpp new file mode 100644 index 0000000000000000000000000000000000000000..fcbab4f9bdee82941f3c197daaafcd8a9c069694 --- /dev/null +++ b/cpp/src/storage/s3/S3ClientWrapper.cpp @@ -0,0 +1,159 @@ +#include "S3ClientWrapper.h" + +#include +#include +#include +#include +#include + +#include +#include + + +namespace zilliz { +namespace milvus { +namespace engine { +namespace storage { + +Status +S3ClientWrapper::Create(const std::string &ip_address, + const std::string &port, + const std::string &access_key, + const std::string &secret_key) { + Aws::InitAPI(options_); + Aws::Client::ClientConfiguration cfg; + + // TODO: ip_address need to be validated. + + cfg.endpointOverride = ip_address + ":" + port; // S3 server ip address and port + cfg.scheme = Aws::Http::Scheme::HTTP; + cfg.verifySSL = + false; //Aws::Auth::AWSCredentials cred("RPW421T9GSIO4A45Y9ZR", "2owKYy9emSS90Q0pXuyqpX1OxBCyEDYodsiBemcq"); // 认证的Key + client_ = + new S3Client(Aws::Auth::AWSCredentials(access_key, secret_key), + cfg, + Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Always, + false); + if (client_ == nullptr) { + std::string error = "Can't connect server."; + return Status::Error(error); + } else { + return Status::OK(); + } +} + + +Status +S3ClientWrapper::Close() { + if (client_ != nullptr) { + delete client_; + client_ = nullptr; + } + Aws::ShutdownAPI(options_); + return Status::OK(); +} + +Status +S3ClientWrapper::CreateBucket(std::string& bucket_name) { + Aws::S3::Model::CreateBucketRequest request; + request.SetBucket(bucket_name); + + auto outcome = client_->CreateBucket(request); + + if (outcome.IsSuccess()) + { + return Status::OK(); + } + else + { + std::cout << "CreateBucket error: " + << outcome.GetError().GetExceptionName() << std::endl + << outcome.GetError().GetMessage() << std::endl; + switch(outcome.GetError().GetErrorType()) { + case Aws::S3::S3Errors::BUCKET_ALREADY_EXISTS: + case Aws::S3::S3Errors::BUCKET_ALREADY_OWNED_BY_YOU: + return Status::AlreadyExist(outcome.GetError().GetMessage()); + default: + return Status::Error(outcome.GetError().GetMessage()); + } + } +} + +Status +S3ClientWrapper::DeleteBucket(std::string& bucket_name) { + Aws::S3::Model::DeleteBucketRequest bucket_request; + bucket_request.SetBucket(bucket_name); + + auto outcome = client_->DeleteBucket(bucket_request); + + if (outcome.IsSuccess()) + { + return Status::OK(); + } + else + { + std::cout << "DeleteBucket error: " + << outcome.GetError().GetExceptionName() << " - " + << outcome.GetError().GetMessage() << std::endl; + return Status::Error(outcome.GetError().GetMessage()); + } +} + +Status +S3ClientWrapper::UploadFile(std::string &BucketName, std::string &objectKey, std::string &pathkey) { + + PutObjectRequest putObjectRequest; + putObjectRequest.WithBucket(BucketName.c_str()).WithKey(objectKey.c_str()); + + auto input_data = Aws::MakeShared("PutObjectInputStream", + pathkey.c_str(), + std::ios_base::in | std::ios_base::binary); + putObjectRequest.SetBody(input_data); + auto put_object_result = client_->PutObject(putObjectRequest); + if (put_object_result.IsSuccess()) { + return Status::OK(); + } else { + std::cout << "PutObject error: " << put_object_result.GetError().GetExceptionName() << " " + << put_object_result.GetError().GetMessage() << std::endl; + return Status::Error(put_object_result.GetError().GetMessage()); + } +} + +Status +S3ClientWrapper::DownloadFile(std::string &BucketName, std::string &objectKey, std::string &pathkey) { + GetObjectRequest object_request; + object_request.WithBucket(BucketName.c_str()).WithKey(objectKey.c_str()); + auto get_object_outcome = client_->GetObject(object_request); + if (get_object_outcome.IsSuccess()) { + Aws::OFStream local_file(pathkey.c_str(), std::ios::out | std::ios::binary); + local_file << get_object_outcome.GetResult().GetBody().rdbuf(); + return Status::OK(); + } else { + std::cout << "GetObject error: " << get_object_outcome.GetError().GetExceptionName() << " " + << get_object_outcome.GetError().GetMessage() << std::endl; + return Status::Error(get_object_outcome.GetError().GetMessage()); + } +} + +Status +S3ClientWrapper::DeleteFile(std::string &bucket_name, std::string &object_key) { + Aws::S3::Model::DeleteObjectRequest object_request; + object_request.WithBucket(bucket_name).WithKey(object_key); + + auto delete_object_outcome = client_->DeleteObject(object_request); + + if (delete_object_outcome.IsSuccess()) { + return Status::OK(); + } else { + std::cout << "DeleteObject error: " << + delete_object_outcome.GetError().GetExceptionName() << " " << + delete_object_outcome.GetError().GetMessage() << std::endl; + + return Status::Error(delete_object_outcome.GetError().GetMessage()); + } +} + +} +} +} +} \ No newline at end of file diff --git a/cpp/src/storage/s3/S3ClientWrapper.h b/cpp/src/storage/s3/S3ClientWrapper.h new file mode 100644 index 0000000000000000000000000000000000000000..9ef60174cfd372e1408ec51ed78bcfbf73fc42f7 --- /dev/null +++ b/cpp/src/storage/s3/S3ClientWrapper.h @@ -0,0 +1,45 @@ +#pragma once + +#include "storage/IStorage.h" + + +#include +#include +#include + + +using namespace Aws::S3; +using namespace Aws::S3::Model; + +namespace zilliz { +namespace milvus { +namespace engine { +namespace storage { + +class S3ClientWrapper : public IStorage { + public: + + S3ClientWrapper() = default; + ~S3ClientWrapper() = default; + + Status Create(const std::string &ip_address, + const std::string &port, + const std::string &access_key, + const std::string &secret_key) override; + Status Close() override; + + Status CreateBucket(std::string& bucket_name) override; + Status DeleteBucket(std::string& bucket_name) override; + Status UploadFile(std::string &BucketName, std::string &objectKey, std::string &pathkey) override; + Status DownloadFile(std::string &BucketName, std::string &objectKey, std::string &pathkey) override; + Status DeleteFile(std::string &bucket_name, std::string &object_key) override; + + private: + S3Client *client_ = nullptr; + Aws::SDKOptions options_; +}; + +} +} +} +} \ No newline at end of file diff --git a/cpp/thirdparty/versions.txt b/cpp/thirdparty/versions.txt index 484267b068361d429ced857e9096c211013990d2..269d81a498b17bf9f515b44c81cfd1caedf00ec6 100644 --- a/cpp/thirdparty/versions.txt +++ b/cpp/thirdparty/versions.txt @@ -17,5 +17,6 @@ THRIFT_VERSION=v0.12.0 YAMLCPP_VERSION=0.6.2 ZLIB_VERSION=v1.2.11 ZSTD_VERSION=v1.4.0 +AWS_VERSION=1.7.125 # vim: set filetype=sh: diff --git a/cpp/unittest/CMakeLists.txt b/cpp/unittest/CMakeLists.txt index f2fa5db0e0b5d2df244a27919e4914d248f08acc..2e2b1f91b6371f3b8bf026516fe525927c6305f0 100644 --- a/cpp/unittest/CMakeLists.txt +++ b/cpp/unittest/CMakeLists.txt @@ -8,12 +8,22 @@ link_directories( "${GTEST_PREFIX}/lib/" ) + +aux_source_directory(${MILVUS_ENGINE_SRC}/db db_srcs) +aux_source_directory(${MILVUS_ENGINE_SRC}/config config_files) + message(STATUS "GTEST LIB: ${GTEST_PREFIX}/lib") set(unittest_srcs ${CMAKE_CURRENT_SOURCE_DIR}/main.cpp) #${EASYLOGGINGPP_INCLUDE_DIR}/easylogging++.cc) +set(require_files + ${MILVUS_ENGINE_SRC}/server/ServerConfig.cpp + ${MILVUS_ENGINE_SRC}/utils/CommonUtil.cpp + ${MILVUS_ENGINE_SRC}/utils/TimeRecorder.cpp + ) + set(unittest_libs yaml-cpp gtest @@ -37,4 +47,5 @@ add_subdirectory(server) add_subdirectory(db) add_subdirectory(faiss_wrapper) add_subdirectory(license) -add_subdirectory(metrics) \ No newline at end of file +add_subdirectory(metrics) +add_subdirectory(storage) \ No newline at end of file diff --git a/cpp/unittest/db/CMakeLists.txt b/cpp/unittest/db/CMakeLists.txt index b8985067e9f23eea2848610e7f833a74ad6333fc..67ef5fccec0bc1cf6739d98a23db22e286e21c36 100644 --- a/cpp/unittest/db/CMakeLists.txt +++ b/cpp/unittest/db/CMakeLists.txt @@ -3,22 +3,16 @@ # Unauthorized copying of this file, via any medium is strictly prohibited. # Proprietary and confidential. #------------------------------------------------------------------------------- -include_directories(../../src) - -aux_source_directory(../../src/db db_srcs) -aux_source_directory(../../src/db/scheduler db_scheduler_srcs) -aux_source_directory(../../src/config config_files) -aux_source_directory(../../src/cache cache_srcs) -aux_source_directory(../../src/wrapper wrapper_src) +aux_source_directory(${MILVUS_ENGINE_SRC}/db db_srcs) +aux_source_directory(${MILVUS_ENGINE_SRC}/db/scheduler db_scheduler_srcs) +aux_source_directory(${MILVUS_ENGINE_SRC}/config config_files) +aux_source_directory(${MILVUS_ENGINE_SRC}/cache cache_srcs) +aux_source_directory(${MILVUS_ENGINE_SRC}/wrapper wrapper_src) include_directories(/usr/local/cuda/include) link_directories("/usr/local/cuda/lib64") -set(require_files - ../../src/server/ServerConfig.cpp - ../../src/utils/CommonUtil.cpp - ../../src/utils/TimeRecorder.cpp - ) + set(db_test_src ${unittest_srcs} diff --git a/cpp/unittest/faiss_wrapper/CMakeLists.txt b/cpp/unittest/faiss_wrapper/CMakeLists.txt index 76c8ca9777394a511b221f5c2cf11d05e3580b0c..b0830d5a87feee2ecc1a2f8049b5095b3e5d92e6 100644 --- a/cpp/unittest/faiss_wrapper/CMakeLists.txt +++ b/cpp/unittest/faiss_wrapper/CMakeLists.txt @@ -3,21 +3,13 @@ # Unauthorized copying of this file, via any medium is strictly prohibited. # Proprietary and confidential. #------------------------------------------------------------------------------- -include_directories(../../src) - -aux_source_directory(../../src/wrapper wrapper_src) -aux_source_directory(../../src/config config_files) +aux_source_directory(${MILVUS_ENGINE_SRC}/wrapper wrapper_src) +aux_source_directory(${MILVUS_ENGINE_SRC}/config config_files) # Make sure that your call to link_directories takes place before your call to the relevant add_executable. include_directories(/usr/local/cuda/include) link_directories("/usr/local/cuda/lib64") -set(require_files - ../../src/server/ServerConfig.cpp - ../../src/utils/CommonUtil.cpp - ../../src/utils/TimeRecorder.cpp - ) - set(wrapper_test_src ${unittest_srcs} ${wrapper_src} diff --git a/cpp/unittest/license/CMakeLists.txt b/cpp/unittest/license/CMakeLists.txt index 1182d256fcf92581bc7bfdae9bba5d3c8f6cf8f1..93af82ed56bbdbaccf5e16daa1e31ca1cf81c5f4 100644 --- a/cpp/unittest/license/CMakeLists.txt +++ b/cpp/unittest/license/CMakeLists.txt @@ -3,12 +3,10 @@ # Unauthorized copying of this file, via any medium is strictly prohibited. # Proprietary and confidential. #------------------------------------------------------------------------------- -include_directories(../../src) - -aux_source_directory(../../src/db db_srcs) -aux_source_directory(../../src/config config_files) -aux_source_directory(../../src/cache cache_srcs) -aux_source_directory(../../src/wrapper wrapper_src) +aux_source_directory(${MILVUS_ENGINE_SRC}/db db_srcs) +aux_source_directory(${MILVUS_ENGINE_SRC}/config config_files) +aux_source_directory(${MILVUS_ENGINE_SRC}/cache cache_srcs) +aux_source_directory(${MILVUS_ENGINE_SRC}/wrapper wrapper_src) include_directories(/usr/local/cuda/include) link_directories(/usr/local/cuda) @@ -18,19 +16,16 @@ link_directories(/usr/lib/nvidia-415) link_directories(/usr/local/cuda/targets/x86_64-linux/lib/stubs/) link_directories(/usr/local/cuda/lib64/stubs/) -set(require_files - ../../src/license/LicenseLibrary.cpp - ../../src/license/LicenseCheck.cpp +set(license_src_files + ${MILVUS_ENGINE_SRC}/license/LicenseLibrary.cpp + ${MILVUS_ENGINE_SRC}/license/LicenseCheck.cpp ) set(db_test_src - license_library_tests.cpp license_check_test.cpp - - - ${require_files}) + ${license_src_files}) cuda_add_executable(license_test ${db_test_src}) diff --git a/cpp/unittest/metrics/CMakeLists.txt b/cpp/unittest/metrics/CMakeLists.txt index 8a34d9c4d70762ddd0690cbe1f680a974d38e5cb..2f07cbed242a90e26c5f8e114c31d82f36de9cfb 100644 --- a/cpp/unittest/metrics/CMakeLists.txt +++ b/cpp/unittest/metrics/CMakeLists.txt @@ -17,6 +17,7 @@ aux_source_directory(../../src/db/scheduler db_scheduler_srcs) aux_source_directory(../../src/config config_files) aux_source_directory(../../src/cache cache_srcs) aux_source_directory(../../src/wrapper wrapper_src) +aux_source_directory(../../src/metrics metrics_src) include_directories(/usr/include) @@ -28,32 +29,26 @@ link_directories("/usr/local/cuda/lib64") include_directories(../../src/metrics) -set(require_files - ../../src/metrics/Metrics.cpp - -# ../../src/cache/CacheMgr.cpp - ../../src/metrics/PrometheusMetrics.cpp - ../../src/metrics/MetricBase.h - ../../src/server/ServerConfig.cpp - ../../src/utils/CommonUtil.cpp - ../../src/utils/TimeRecorder.cpp +#set(metrics_src_files +# ../../src/metrics/Metrics.cpp +# ../../src/metrics/Metrics.h # ../../src/metrics/PrometheusMetrics.cpp - ) +# ../../src/metrics/MetricBase.h +# ../../src/server/ServerConfig.cpp +# ../../src/utils/CommonUtil.cpp +# ../../src/utils/TimeRecorder.cpp +# ) set(count_test_src - # ${unittest_srcs} - # ${config_files} - # ${require_files} ${unittest_srcs} ${config_files} ${cache_srcs} ${db_srcs} ${db_scheduler_srcs} ${wrapper_src} - ${require_files} + ${metrics_src} metrics_test.cpp ../db/utils.cpp - ../../src/metrics/Metrics.h ) diff --git a/cpp/unittest/metrics/metrics_test.cpp b/cpp/unittest/metrics/metrics_test.cpp index 9a315f5557d0efe1f20b1c1ced4d2160feba40d4..82e1d99b93b7a9d311374e55e5841d6bcc227ae6 100644 --- a/cpp/unittest/metrics/metrics_test.cpp +++ b/cpp/unittest/metrics/metrics_test.cpp @@ -102,7 +102,7 @@ TEST_F(DBTest, Metric_Tes) { } }); - int loop = 100000; + int loop = 10; for (auto i=0; i +#include +#include +#include + + +using namespace zilliz::milvus::engine; + +TEST(s3_client_wrapper, CLIENT_WRAPPER_TEST) { + + std::shared_ptr storage_ptr = std::make_shared(); + + std::string ip_address = "127.0.0.1"; + std::string port = "9000"; + std::string access_key = "AKIAIOSFODNN7EXAMPLE"; + std::string secret_key = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"; + + Status status = storage_ptr->Create(ip_address, port, access_key, secret_key); + ASSERT_TRUE(status.ok()); + + std::string filename = "/tmp/s3_test_file"; + std::string bucket_name = "bucktname"; + std::string object_name = "test_file"; + + status = storage_ptr->CreateBucket(bucket_name); + std::cout << status.IsAlreadyExist() << std::endl; + if (status.IsAlreadyExist()) { + status = storage_ptr->DeleteBucket(bucket_name); + status = storage_ptr->CreateBucket(bucket_name); + } + + ASSERT_TRUE(status.ok()); + + std::ofstream ofile(filename); + std::stringstream ss; + for (int i = 0; i < 1024; ++i) { + ss << i; + } + ofile << ss.str() << std::endl; + ofile.close(); + status = storage_ptr->UploadFile(bucket_name, object_name, filename); + ASSERT_TRUE(status.ok()); + + status = storage_ptr->DownloadFile(bucket_name, object_name, filename); + std::ifstream infile(filename); + std::string in_buffer; + infile >> in_buffer; + ASSERT_STREQ(in_buffer.c_str(), ss.str().c_str()); + + status = storage_ptr->DeleteFile(bucket_name, object_name); + ASSERT_TRUE(status.ok()); + + status = storage_ptr->DeleteBucket(bucket_name); + ASSERT_TRUE(status.ok()); + + status = storage_ptr->Close(); + ASSERT_TRUE(status.ok()); +} +