Add a feature of getting milvus all configs (#3347)

* change the header define to make it standardized
Signed-off-by: Ngodchen0212 <qingxiang.chen@zilliz.com>

* add a feature of getting milvus all configs
Signed-off-by: Ngodchen0212 <qingxiang.chen@zilliz.com>

* format code
Signed-off-by: Ngodchen0212 <qingxiang.chen@zilliz.com>

* format code
Signed-off-by: Ngodchen0212 <qingxiang.chen@zilliz.com>

* change the position of header json
Signed-off-by: Ngodchen0212 <qingxiang.chen@zilliz.com>

* remove useless line
Signed-off-by: Ngodchen0212 <qingxiang.chen@zilliz.com>

* format code
Signed-off-by: Ngodchen0212 <qingxiang.chen@zilliz.com>

* fix a magic bytes write bug and format code
Signed-off-by: Ngodchen0212 <qingxiang.chen@zilliz.com>

core/src/codecs/BlockFormat.cpp

@@ -26,7 +26,6 @@
 #include "storage/ExtraFileInfo.h"
 #include "utils/Exception.h"
 #include "utils/Log.h"
-#include "utils/TimeRecorder.h"
 
 namespace milvus {
 namespace codec {
@@ -124,7 +123,7 @@ BlockFormat::Write(const storage::FSHandlerPtr& fs_ptr, const std::string& file_
     }
     // TODO: add extra info
     std::unordered_map<std::string, std::string> maps;
-    WRITE_MAGIC(fs_ptr, file_path)
+    WRITE_MAGIC(fs_ptr, file_path);
     WRITE_HEADER(fs_ptr, file_path, maps);
 
     if (!fs_ptr->writer_ptr_->InOpen(file_path)) {
@@ -135,7 +134,7 @@ BlockFormat::Write(const storage::FSHandlerPtr& fs_ptr, const std::string& file_
 
     size_t num_bytes = raw->data_.size();
     fs_ptr->writer_ptr_->Write(&num_bytes, sizeof(size_t));
-    fs_ptr->writer_ptr_->Write((void*)(raw->data_.data()), num_bytes);
+    fs_ptr->writer_ptr_->Write(raw->data_.data(), num_bytes);
     fs_ptr->writer_ptr_->Close();
 
     WRITE_SUM(fs_ptr, file_path);

core/src/codecs/StructuredIndexFormat.cpp

@@ -148,7 +148,7 @@ StructuredIndexFormat::Write(const milvus::storage::FSHandlerPtr& fs_ptr, const
     std::string full_file_path = file_path + STRUCTURED_INDEX_POSTFIX;
     // TODO: add extra info
     std::unordered_map<std::string, std::string> maps;
-    WRITE_MAGIC(fs_ptr, full_file_path)
+    WRITE_MAGIC(fs_ptr, full_file_path);
     WRITE_HEADER(fs_ptr, full_file_path, maps);
 
     auto binaryset = index->Serialize(knowhere::Config());
@@ -163,12 +163,12 @@ StructuredIndexFormat::Write(const milvus::storage::FSHandlerPtr& fs_ptr, const
         auto meta = iter.first.c_str();
         size_t meta_length = iter.first.length();
         fs_ptr->writer_ptr_->Write(&meta_length, sizeof(meta_length));
-        fs_ptr->writer_ptr_->Write((void*)meta, meta_length);
+        fs_ptr->writer_ptr_->Write(const_cast<char*>(meta), meta_length);
 
         auto binary = iter.second;
         int64_t binary_length = binary->size;
         fs_ptr->writer_ptr_->Write(&binary_length, sizeof(binary_length));
-        fs_ptr->writer_ptr_->Write((void*)binary->data.get(), binary_length);
+        fs_ptr->writer_ptr_->Write(binary->data.get(), binary_length);
     }
 
     fs_ptr->writer_ptr_->Close();

core/src/codecs/VectorCompressFormat.cpp

@@ -74,7 +74,7 @@ VectorCompressFormat::Write(const storage::FSHandlerPtr& fs_ptr, const std::stri
     const std::string full_file_path = file_path + VECTOR_COMPRESS_POSTFIX;
     // TODO: add extra info
     std::unordered_map<std::string, std::string> maps;
-    WRITE_MAGIC(fs_ptr, full_file_path)
+    WRITE_MAGIC(fs_ptr, full_file_path);
     WRITE_HEADER(fs_ptr, full_file_path, maps);
     if (!fs_ptr->writer_ptr_->InOpen(full_file_path)) {
         THROW_ERROR(SERVER_CANNOT_OPEN_FILE, "Fail to open vector compress: " + full_file_path);

core/src/codecs/VectorIndexFormat.cpp

@@ -181,7 +181,7 @@ VectorIndexFormat::WriteIndex(const storage::FSHandlerPtr& fs_ptr, const std::st
     std::string full_file_path = file_path + VECTOR_INDEX_POSTFIX;
     // TODO: add extra info
     std::unordered_map<std::string, std::string> maps;
-    WRITE_MAGIC(fs_ptr, full_file_path)
+    WRITE_MAGIC(fs_ptr, full_file_path);
     WRITE_HEADER(fs_ptr, full_file_path, maps);
     auto binaryset = index->Serialize(knowhere::Config());
 
@@ -194,12 +194,12 @@ VectorIndexFormat::WriteIndex(const storage::FSHandlerPtr& fs_ptr, const std::st
         auto meta = iter.first.c_str();
         size_t meta_length = iter.first.length();
         fs_ptr->writer_ptr_->Write(&meta_length, sizeof(meta_length));
-        fs_ptr->writer_ptr_->Write((void*)meta, meta_length);
+        fs_ptr->writer_ptr_->Write(const_cast<char*>(meta), meta_length);
 
         auto binary = iter.second;
         int64_t binary_length = binary->size;
         fs_ptr->writer_ptr_->Write(&binary_length, sizeof(binary_length));
-        fs_ptr->writer_ptr_->Write((void*)binary->data.get(), binary_length);
+        fs_ptr->writer_ptr_->Write(binary->data.get(), binary_length);
     }
     fs_ptr->writer_ptr_->Close();
 

core/src/config/ConfigMgr.cpp

@@ -12,6 +12,7 @@
 #include <yaml-cpp/yaml.h>
 #include <cstring>
 #include <limits>
+#include <nlohmann/json.hpp>
 #include <unordered_map>
 
 #include "config/ConfigMgr.h"
@@ -246,6 +247,16 @@ ConfigMgr::Dump() const {
     return ss.str();
 }
 
+std::string
+ConfigMgr::JsonDump() const {
+    nlohmann::json j;
+    for (auto& kv : config_list_) {
+        auto& config = kv.second;
+        j[config->name_] = config->Get();
+    }
+    return j.dump();
+}
+
 void
 ConfigMgr::Attach(const std::string& name, ConfigObserver* observer) {
     std::lock_guard<std::mutex> lock(observer_mutex_);

core/src/config/ConfigMgr.h

@@ -68,6 +68,8 @@ class ConfigMgr {
 
     std::string
     Dump() const;
+    std::string
+    JsonDump() const;
 
  public:
     // Shared pointer should not be used here

core/src/server/delivery/request/CmdReq.cpp

@@ -57,6 +57,8 @@ CmdReq::OnExecute() {
         sys_info_inst.GetSysInfoJsonStr(result_);
     } else if (cmd_ == "build_commit_id") {
         result_ = LAST_COMMIT_ID;
+    } else if (cmd_ == "get_milvus_config") {
+        result_ = ConfigMgr::GetInstance().JsonDump();
     } else if (cmd_.substr(0, 3) == "get") {
         try {
             auto words = split(cmd_, ' ');

core/src/storage/ExtraFileInfo.cpp

@@ -17,6 +17,12 @@
 #include "crc32c/crc32c.h"
 #include "storage/ExtraFileInfo.h"
 
+const char* MAGIC = "Milvus";
+const int MAGIC_SIZE = 6;
+const int SINGLE_KV_DATA_SIZE = 64;
+const int HEADER_SIZE = 4096;
+const int SUM_SIZE = 16;
+
 namespace milvus {
 namespace storage {
 
@@ -43,7 +49,7 @@ WriteMagic(const storage::FSHandlerPtr& fs_ptr, const std::string& file_path) {
         LOG_ENGINE_ERROR_ << err_msg;
         throw Exception(SERVER_WRITE_ERROR, err_msg);
     }
-    fs_ptr->writer_ptr_->Write((void*)(MAGIC), MAGIC_SIZE);
+    fs_ptr->writer_ptr_->Write(const_cast<char*>(MAGIC), MAGIC_SIZE);
     fs_ptr->writer_ptr_->Close();
 }
 

core/src/storage/ExtraFileInfo.h

@@ -15,8 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-#ifndef EXTRA_FILE_INFO_H
-#define EXTRA_FILE_INFO_H
+#pragma once
 
 #include <cstdio>
 #include <cstring>
@@ -29,11 +28,11 @@
 
 #include "storage/FSHandler.h"
 
-#define MAGIC "Milvus"
-#define MAGIC_SIZE 6
-#define SINGLE_KV_DATA_SIZE 64
-#define HEADER_SIZE 4096
-#define SUM_SIZE 16
+extern const char* MAGIC;
+extern const int MAGIC_SIZE;
+extern const int SINGLE_KV_DATA_SIZE;
+extern const int HEADER_SIZE;
+extern const int SUM_SIZE;
 
 namespace milvus {
 namespace storage {
@@ -100,4 +99,3 @@ WriteHeaderValues(const storage::FSHandlerPtr& fs_ptr, const std::string& file_p
 
 }  // namespace storage
 }  // namespace milvus
-#endif  // end of EXTRA_FILE_INFO_H

core/src/storage/ExtraFileInfoTest.cpp

@@ -53,7 +53,7 @@ TEST_F(ExtraFileInfoTest, WriteFileTest) {
 
     size_t num_bytes = raw.size();
     fs_ptr->writer_ptr_->Write(&num_bytes, sizeof(size_t));
-    fs_ptr->writer_ptr_->Write((void*)(raw.data()), num_bytes);
+    fs_ptr->writer_ptr_->Write(raw.data(), num_bytes);
     fs_ptr->writer_ptr_->Close();
 
     int result_sum = CalculateSum(fs_ptr, file_path);