admin/milvus
1
0
Fork 0
mirror of https://gitee.com/milvus-io/milvus.git synced 2026-02-02 01:06:41 +08:00
Code Issues Packages Projects Releases Wiki Activity

Add a feature of getting milvus all configs (#3347)

Browse Source 
* change the header define to make it standardized

Signed-off-by: godchen0212 <qingxiang.chen@zilliz.com>

* add a feature of getting milvus all configs

Signed-off-by: godchen0212 <qingxiang.chen@zilliz.com>

* format code

Signed-off-by: godchen0212 <qingxiang.chen@zilliz.com>

* format code

Signed-off-by: godchen0212 <qingxiang.chen@zilliz.com>

* change the position of header json

Signed-off-by: godchen0212 <qingxiang.chen@zilliz.com>

* remove useless line

Signed-off-by: godchen0212 <qingxiang.chen@zilliz.com>

* format code

Signed-off-by: godchen0212 <qingxiang.chen@zilliz.com>

* fix a magic bytes write bug and format code

Signed-off-by: godchen0212 <qingxiang.chen@zilliz.com>
This commit is contained in:
chen qingxiang 2020-08-20 16:00:18 +08:00 committed by GitHub
parent 2a2cf0fe9f
commit f51959bb76
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 38 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
core/src/codecs/BlockFormat.cpp
View File

@ -26,7 +26,6 @@
#include "storage/ExtraFileInfo.h"
#include "utils/Exception.h"
#include "utils/Log.h"
#include "utils/TimeRecorder.h"
namespace milvus {
namespace codec {
@ -124,7 +123,7 @@ BlockFormat::Write(const storage::FSHandlerPtr& fs_ptr, const std::string& file_
}
// TODO: add extra info
std::unordered_map<std::string, std::string> maps;
WRITE_MAGIC(fs_ptr, file_path)
WRITE_MAGIC(fs_ptr, file_path);
WRITE_HEADER(fs_ptr, file_path, maps);
if (!fs_ptr->writer_ptr_->InOpen(file_path)) {
@ -135,7 +134,7 @@ BlockFormat::Write(const storage::FSHandlerPtr& fs_ptr, const std::string& file_
size_t num_bytes = raw->data_.size();
fs_ptr->writer_ptr_->Write(&num_bytes, sizeof(size_t));
fs_ptr->writer_ptr_->Write((void*)(raw->data_.data()), num_bytes);
fs_ptr->writer_ptr_->Write(raw->data_.data(), num_bytes);
fs_ptr->writer_ptr_->Close();
WRITE_SUM(fs_ptr, file_path);

6
core/src/codecs/StructuredIndexFormat.cpp
View File

@ -148,7 +148,7 @@ StructuredIndexFormat::Write(const milvus::storage::FSHandlerPtr& fs_ptr, const
std::string full_file_path = file_path + STRUCTURED_INDEX_POSTFIX;
// TODO: add extra info
std::unordered_map<std::string, std::string> maps;
WRITE_MAGIC(fs_ptr, full_file_path)
WRITE_MAGIC(fs_ptr, full_file_path);
WRITE_HEADER(fs_ptr, full_file_path, maps);
auto binaryset = index->Serialize(knowhere::Config());
@ -163,12 +163,12 @@ StructuredIndexFormat::Write(const milvus::storage::FSHandlerPtr& fs_ptr, const
auto meta = iter.first.c_str();
size_t meta_length = iter.first.length();
fs_ptr->writer_ptr_->Write(&meta_length, sizeof(meta_length));
fs_ptr->writer_ptr_->Write((void*)meta, meta_length);
fs_ptr->writer_ptr_->Write(const_cast<char*>(meta), meta_length);
auto binary = iter.second;
int64_t binary_length = binary->size;
fs_ptr->writer_ptr_->Write(&binary_length, sizeof(binary_length));
fs_ptr->writer_ptr_->Write((void*)binary->data.get(), binary_length);
fs_ptr->writer_ptr_->Write(binary->data.get(), binary_length);
}
fs_ptr->writer_ptr_->Close();

2
core/src/codecs/VectorCompressFormat.cpp
View File

@ -74,7 +74,7 @@ VectorCompressFormat::Write(const storage::FSHandlerPtr& fs_ptr, const std::stri
const std::string full_file_path = file_path + VECTOR_COMPRESS_POSTFIX;
// TODO: add extra info
std::unordered_map<std::string, std::string> maps;
WRITE_MAGIC(fs_ptr, full_file_path)
WRITE_MAGIC(fs_ptr, full_file_path);
WRITE_HEADER(fs_ptr, full_file_path, maps);
if (!fs_ptr->writer_ptr_->InOpen(full_file_path)) {
THROW_ERROR(SERVER_CANNOT_OPEN_FILE, "Fail to open vector compress: " + full_file_path);

6
core/src/codecs/VectorIndexFormat.cpp
View File

@ -181,7 +181,7 @@ VectorIndexFormat::WriteIndex(const storage::FSHandlerPtr& fs_ptr, const std::st
std::string full_file_path = file_path + VECTOR_INDEX_POSTFIX;
// TODO: add extra info
std::unordered_map<std::string, std::string> maps;
WRITE_MAGIC(fs_ptr, full_file_path)
WRITE_MAGIC(fs_ptr, full_file_path);
WRITE_HEADER(fs_ptr, full_file_path, maps);
auto binaryset = index->Serialize(knowhere::Config());
@ -194,12 +194,12 @@ VectorIndexFormat::WriteIndex(const storage::FSHandlerPtr& fs_ptr, const std::st
auto meta = iter.first.c_str();
size_t meta_length = iter.first.length();
fs_ptr->writer_ptr_->Write(&meta_length, sizeof(meta_length));
fs_ptr->writer_ptr_->Write((void*)meta, meta_length);
fs_ptr->writer_ptr_->Write(const_cast<char*>(meta), meta_length);
auto binary = iter.second;
int64_t binary_length = binary->size;
fs_ptr->writer_ptr_->Write(&binary_length, sizeof(binary_length));
fs_ptr->writer_ptr_->Write((void*)binary->data.get(), binary_length);
fs_ptr->writer_ptr_->Write(binary->data.get(), binary_length);
}
fs_ptr->writer_ptr_->Close();

11
core/src/config/ConfigMgr.cpp
View File

@ -12,6 +12,7 @@
#include <yaml-cpp/yaml.h>
#include <cstring>
#include <limits>
#include <nlohmann/json.hpp>
#include <unordered_map>
#include "config/ConfigMgr.h"
@ -246,6 +247,16 @@ ConfigMgr::Dump() const {
return ss.str();
}
std::string
ConfigMgr::JsonDump() const {
nlohmann::json j;
for (auto& kv : config_list_) {
auto& config = kv.second;
j[config->name_] = config->Get();
}
return j.dump();
}
void
ConfigMgr::Attach(const std::string& name, ConfigObserver* observer) {
std::lock_guard<std::mutex> lock(observer_mutex_);

2
core/src/config/ConfigMgr.h
View File

@ -68,6 +68,8 @@ class ConfigMgr {
std::string
Dump() const;
std::string
JsonDump() const;
public:
// Shared pointer should not be used here

2
core/src/server/delivery/request/CmdReq.cpp
View File

@ -57,6 +57,8 @@ CmdReq::OnExecute() {
sys_info_inst.GetSysInfoJsonStr(result_);
} else if (cmd_ == "build_commit_id") {
result_ = LAST_COMMIT_ID;
} else if (cmd_ == "get_milvus_config") {
result_ = ConfigMgr::GetInstance().JsonDump();
} else if (cmd_.substr(0, 3) == "get") {
try {
auto words = split(cmd_, ' ');

8
core/src/storage/ExtraFileInfo.cpp
View File

@ -17,6 +17,12 @@
#include "crc32c/crc32c.h"
#include "storage/ExtraFileInfo.h"
const char* MAGIC = "Milvus";
const int MAGIC_SIZE = 6;
const int SINGLE_KV_DATA_SIZE = 64;
const int HEADER_SIZE = 4096;
const int SUM_SIZE = 16;
namespace milvus {
namespace storage {
@ -43,7 +49,7 @@ WriteMagic(const storage::FSHandlerPtr& fs_ptr, const std::string& file_path) {
LOG_ENGINE_ERROR_ << err_msg;
throw Exception(SERVER_WRITE_ERROR, err_msg);
}
fs_ptr->writer_ptr_->Write((void*)(MAGIC), MAGIC_SIZE);
fs_ptr->writer_ptr_->Write(const_cast<char*>(MAGIC), MAGIC_SIZE);
fs_ptr->writer_ptr_->Close();
}

14
core/src/storage/ExtraFileInfo.h
View File

@ -15,8 +15,7 @@
// specific language governing permissions and limitations
// under the License.
#ifndef EXTRA_FILE_INFO_H
#define EXTRA_FILE_INFO_H
#pragma once
#include <cstdio>
#include <cstring>
@ -29,11 +28,11 @@
#include "storage/FSHandler.h"
#define MAGIC "Milvus"
#define MAGIC_SIZE 6
#define SINGLE_KV_DATA_SIZE 64
#define HEADER_SIZE 4096
#define SUM_SIZE 16
extern const char* MAGIC;
extern const int MAGIC_SIZE;
extern const int SINGLE_KV_DATA_SIZE;
extern const int HEADER_SIZE;
extern const int SUM_SIZE;
namespace milvus {
namespace storage {
@ -100,4 +99,3 @@ WriteHeaderValues(const storage::FSHandlerPtr& fs_ptr, const std::string& file_p
} // namespace storage
} // namespace milvus
#endif // end of EXTRA_FILE_INFO_H

2
core/src/storage/ExtraFileInfoTest.cpp
View File

@ -53,7 +53,7 @@ TEST_F(ExtraFileInfoTest, WriteFileTest) {
size_t num_bytes = raw.size();
fs_ptr->writer_ptr_->Write(&num_bytes, sizeof(size_t));
fs_ptr->writer_ptr_->Write((void*)(raw.data()), num_bytes);
fs_ptr->writer_ptr_->Write(raw.data(), num_bytes);
fs_ptr->writer_ptr_->Close();
int result_sum = CalculateSum(fs_ptr, file_path);