From 1d18c0a938d277e379f28741d184f93f4f7ec146 Mon Sep 17 00:00:00 2001 From: "yudong.cai" Date: Mon, 9 Mar 2020 13:08:17 +0800 Subject: [PATCH] #1547 rename storage/file to storage/disk Signed-off-by: yudong.cai --- core/src/CMakeLists.txt | 4 +- core/src/storage/IStorage.h | 44 ------------------- .../DiskIOReader.cpp} | 12 ++--- .../FileIOReader.h => disk/DiskIOReader.h} | 6 +-- .../DiskIOWriter.cpp} | 10 ++--- .../FileIOWriter.h => disk/DiskIOWriter.h} | 6 +-- core/src/storage/s3/S3ClientWrapper.h | 23 +++++----- core/src/wrapper/VecIndex.cpp | 8 ++-- core/unittest/CMakeLists.txt | 4 +- core/unittest/storage/test_s3_client.cpp | 1 - core/unittest/wrapper/CMakeLists.txt | 4 +- 11 files changed, 39 insertions(+), 83 deletions(-) delete mode 100644 core/src/storage/IStorage.h rename core/src/storage/{file/FileIOReader.cpp => disk/DiskIOReader.cpp} (79%) rename core/src/storage/{file/FileIOReader.h => disk/DiskIOReader.h} (89%) rename core/src/storage/{file/FileIOWriter.cpp => disk/DiskIOWriter.cpp} (81%) rename core/src/storage/{file/FileIOWriter.h => disk/DiskIOWriter.h} (88%) diff --git a/core/src/CMakeLists.txt b/core/src/CMakeLists.txt index 3978a407..ef1af69e 100644 --- a/core/src/CMakeLists.txt +++ b/core/src/CMakeLists.txt @@ -106,11 +106,11 @@ set(web_server_files ) aux_source_directory(${MILVUS_ENGINE_SRC}/storage storage_main_files) -aux_source_directory(${MILVUS_ENGINE_SRC}/storage/file storage_file_files) +aux_source_directory(${MILVUS_ENGINE_SRC}/storage/disk storage_disk_files) aux_source_directory(${MILVUS_ENGINE_SRC}/storage/s3 storage_s3_files) set(storage_files ${storage_main_files} - ${storage_file_files} + ${storage_disk_files} ${storage_s3_files} ) diff --git a/core/src/storage/IStorage.h b/core/src/storage/IStorage.h deleted file mode 100644 index 6839f2bf..00000000 --- a/core/src/storage/IStorage.h +++ /dev/null @@ -1,44 +0,0 @@ -// Copyright (C) 2019-2020 Zilliz. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software distributed under the License -// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -// or implied. See the License for the specific language governing permissions and limitations under the License. - -#pragma once - -#include -#include -#include "utils/Status.h" - -namespace milvus { -namespace storage { - -class IStorage { - public: - virtual Status - CreateBucket() = 0; - virtual Status - DeleteBucket() = 0; - virtual Status - PutObjectFile(const std::string& object_name, const std::string& file_path) = 0; - virtual Status - PutObjectStr(const std::string& object_name, const std::string& content) = 0; - virtual Status - GetObjectFile(const std::string& object_name, const std::string& file_path) = 0; - virtual Status - GetObjectStr(const std::string& object_name, std::string& content) = 0; - virtual Status - ListObjects(std::vector& object_list, const std::string& marker = "") = 0; - virtual Status - DeleteObject(const std::string& object_name) = 0; - virtual Status - DeleteObjects(const std::string& marker) = 0; -}; - -} // namespace storage -} // namespace milvus diff --git a/core/src/storage/file/FileIOReader.cpp b/core/src/storage/disk/DiskIOReader.cpp similarity index 79% rename from core/src/storage/file/FileIOReader.cpp rename to core/src/storage/disk/DiskIOReader.cpp index 9aab29e3..2b3e649f 100644 --- a/core/src/storage/file/FileIOReader.cpp +++ b/core/src/storage/disk/DiskIOReader.cpp @@ -9,31 +9,31 @@ // is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express // or implied. See the License for the specific language governing permissions and limitations under the License. -#include "storage/file/FileIOReader.h" +#include "storage/disk/DiskIOReader.h" namespace milvus { namespace storage { -FileIOReader::FileIOReader(const std::string& name) : IOReader(name) { +DiskIOReader::DiskIOReader(const std::string& name) : IOReader(name) { fs_ = std::fstream(name_, std::ios::in | std::ios::binary); } -FileIOReader::~FileIOReader() { +DiskIOReader::~DiskIOReader() { fs_.close(); } void -FileIOReader::read(void* ptr, size_t size) { +DiskIOReader::read(void* ptr, size_t size) { fs_.read(reinterpret_cast(ptr), size); } void -FileIOReader::seekg(size_t pos) { +DiskIOReader::seekg(size_t pos) { fs_.seekg(pos); } size_t -FileIOReader::length() { +DiskIOReader::length() { fs_.seekg(0, fs_.end); return fs_.tellg(); } diff --git a/core/src/storage/file/FileIOReader.h b/core/src/storage/disk/DiskIOReader.h similarity index 89% rename from core/src/storage/file/FileIOReader.h rename to core/src/storage/disk/DiskIOReader.h index ed35c39d..08aa2fdd 100644 --- a/core/src/storage/file/FileIOReader.h +++ b/core/src/storage/disk/DiskIOReader.h @@ -18,10 +18,10 @@ namespace milvus { namespace storage { -class FileIOReader : public IOReader { +class DiskIOReader : public IOReader { public: - explicit FileIOReader(const std::string& name); - ~FileIOReader(); + explicit DiskIOReader(const std::string& name); + ~DiskIOReader(); void read(void* ptr, size_t size) override; diff --git a/core/src/storage/file/FileIOWriter.cpp b/core/src/storage/disk/DiskIOWriter.cpp similarity index 81% rename from core/src/storage/file/FileIOWriter.cpp rename to core/src/storage/disk/DiskIOWriter.cpp index 3992b583..08e77045 100644 --- a/core/src/storage/file/FileIOWriter.cpp +++ b/core/src/storage/disk/DiskIOWriter.cpp @@ -9,27 +9,27 @@ // is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express // or implied. See the License for the specific language governing permissions and limitations under the License. -#include "storage/file/FileIOWriter.h" +#include "storage/disk/DiskIOWriter.h" namespace milvus { namespace storage { -FileIOWriter::FileIOWriter(const std::string& name) : IOWriter(name) { +DiskIOWriter::DiskIOWriter(const std::string& name) : IOWriter(name) { fs_ = std::fstream(name_, std::ios::out | std::ios::binary); } -FileIOWriter::~FileIOWriter() { +DiskIOWriter::~DiskIOWriter() { fs_.close(); } void -FileIOWriter::write(void* ptr, size_t size) { +DiskIOWriter::write(void* ptr, size_t size) { fs_.write(reinterpret_cast(ptr), size); len_ += size; } size_t -FileIOWriter::length() { +DiskIOWriter::length() { return len_; } diff --git a/core/src/storage/file/FileIOWriter.h b/core/src/storage/disk/DiskIOWriter.h similarity index 88% rename from core/src/storage/file/FileIOWriter.h rename to core/src/storage/disk/DiskIOWriter.h index b06eb3fe..4bcbdaf2 100644 --- a/core/src/storage/file/FileIOWriter.h +++ b/core/src/storage/disk/DiskIOWriter.h @@ -18,10 +18,10 @@ namespace milvus { namespace storage { -class FileIOWriter : public IOWriter { +class DiskIOWriter : public IOWriter { public: - explicit FileIOWriter(const std::string& name); - ~FileIOWriter(); + explicit DiskIOWriter(const std::string& name); + ~DiskIOWriter(); void write(void* ptr, size_t size) override; diff --git a/core/src/storage/s3/S3ClientWrapper.h b/core/src/storage/s3/S3ClientWrapper.h index 452c5cc2..f0e2d8cf 100644 --- a/core/src/storage/s3/S3ClientWrapper.h +++ b/core/src/storage/s3/S3ClientWrapper.h @@ -16,12 +16,13 @@ #include #include #include -#include "storage/IStorage.h" + +#include "utils/Status.h" namespace milvus { namespace storage { -class S3ClientWrapper : public IStorage { +class S3ClientWrapper { public: static S3ClientWrapper& GetInstance() { @@ -35,23 +36,23 @@ class S3ClientWrapper : public IStorage { StopService(); Status - CreateBucket() override; + CreateBucket(); Status - DeleteBucket() override; + DeleteBucket(); Status - PutObjectFile(const std::string& object_key, const std::string& file_path) override; + PutObjectFile(const std::string& object_key, const std::string& file_path); Status - PutObjectStr(const std::string& object_key, const std::string& content) override; + PutObjectStr(const std::string& object_key, const std::string& content); Status - GetObjectFile(const std::string& object_key, const std::string& file_path) override; + GetObjectFile(const std::string& object_key, const std::string& file_path); Status - GetObjectStr(const std::string& object_key, std::string& content) override; + GetObjectStr(const std::string& object_key, std::string& content); Status - ListObjects(std::vector& object_list, const std::string& marker = "") override; + ListObjects(std::vector& object_list, const std::string& marker = ""); Status - DeleteObject(const std::string& object_key) override; + DeleteObject(const std::string& object_key); Status - DeleteObjects(const std::string& marker) override; + DeleteObjects(const std::string& marker); private: std::shared_ptr client_ptr_; diff --git a/core/src/wrapper/VecIndex.cpp b/core/src/wrapper/VecIndex.cpp index 9f5552c7..bc8f0626 100644 --- a/core/src/wrapper/VecIndex.cpp +++ b/core/src/wrapper/VecIndex.cpp @@ -23,8 +23,8 @@ #include "knowhere/index/vector_index/IndexNSG.h" #include "knowhere/index/vector_index/IndexSPTAG.h" #include "server/Config.h" -#include "storage/file/FileIOReader.h" -#include "storage/file/FileIOWriter.h" +#include "storage/disk/DiskIOReader.h" +#include "storage/disk/DiskIOWriter.h" #include "storage/s3/S3IOReader.h" #include "storage/s3/S3IOWriter.h" #include "utils/Exception.h" @@ -181,7 +181,7 @@ read_index(const std::string& location) { if (s3_enable) { reader_ptr = std::make_shared(location); } else { - reader_ptr = std::make_shared(location); + reader_ptr = std::make_shared(location); } recorder.RecordSection("Start"); @@ -254,7 +254,7 @@ write_index(VecIndexPtr index, const std::string& location) { if (s3_enable) { writer_ptr = std::make_shared(location); } else { - writer_ptr = std::make_shared(location); + writer_ptr = std::make_shared(location); } recorder.RecordSection("Start"); diff --git a/core/unittest/CMakeLists.txt b/core/unittest/CMakeLists.txt index 95b63523..aedbada0 100644 --- a/core/unittest/CMakeLists.txt +++ b/core/unittest/CMakeLists.txt @@ -98,11 +98,11 @@ aux_source_directory(${MILVUS_ENGINE_SRC}/wrapper wrapper_files) aux_source_directory(${MILVUS_ENGINE_SRC}/tracing tracing_files) aux_source_directory(${MILVUS_ENGINE_SRC}/storage storage_main_files) -aux_source_directory(${MILVUS_ENGINE_SRC}/storage/file storage_file_files) +aux_source_directory(${MILVUS_ENGINE_SRC}/storage/disk storage_disk_files) aux_source_directory(${MILVUS_ENGINE_SRC}/storage/s3 storage_s3_files) set(storage_files ${storage_main_files} - ${storage_file_files} + ${storage_disk_files} ${storage_s3_files} ) diff --git a/core/unittest/storage/test_s3_client.cpp b/core/unittest/storage/test_s3_client.cpp index fab6d5e9..96098a7b 100644 --- a/core/unittest/storage/test_s3_client.cpp +++ b/core/unittest/storage/test_s3_client.cpp @@ -21,7 +21,6 @@ #include "storage/s3/S3ClientWrapper.h" #include "storage/s3/S3IOReader.h" #include "storage/s3/S3IOWriter.h" -#include "storage/IStorage.h" #include "storage/utils.h" INITIALIZE_EASYLOGGINGPP diff --git a/core/unittest/wrapper/CMakeLists.txt b/core/unittest/wrapper/CMakeLists.txt index 7cf48466..487477f3 100644 --- a/core/unittest/wrapper/CMakeLists.txt +++ b/core/unittest/wrapper/CMakeLists.txt @@ -27,8 +27,8 @@ set(wrapper_files ) set(storage_files - ${MILVUS_ENGINE_SRC}/storage/file/FileIOReader.cpp - ${MILVUS_ENGINE_SRC}/storage/file/FileIOWriter.cpp + ${MILVUS_ENGINE_SRC}/storage/disk/DiskIOReader.cpp + ${MILVUS_ENGINE_SRC}/storage/disk/DiskIOWriter.cpp ) set(util_files -- GitLab