admin/milvus
1
0
Fork 0
mirror of https://gitee.com/milvus-io/milvus.git synced 2026-01-07 19:31:51 +08:00
Code Issues Packages Projects Releases Wiki Activity

rename 'minio' to 's3' in Storage Config section (#1088)

Browse Source 
* #1002 rename 'minio' to 's3' in Storage Config section

* #1002 clang format
This commit is contained in:
Cai Yudong 2020-01-18 10:07:46 +08:00 committed by Jin Hai
parent 015f0352a6
commit 364fc25766
13 changed files with 216 additions and 216 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
CHANGELOG.md
View File

@ -42,6 +42,7 @@ Please mark all change in change log and use the issue from GitHub
- \#908 - Move "primary_path" and "secondary_path" to storage config
- \#931 - Remove "collector" from config
- \#966 - Update NOTICE.md
- \#1002 - Rename minio to s3 in Storage Config section
## Task

24
core/conf/demo/server_config.yaml
View File

@ -70,27 +70,27 @@ db_config:
# secondary_path | A semicolon-separated list of secondary directories used | Path | |
# | to save vector data and index data. | | |
#----------------------+------------------------------------------------------------+------------+-----------------+
# minio_enable | Enable MinIO storage or not. | Boolean | false |
# s3_enable | Enable Simple Storage Service or not. | Boolean | false |
#----------------------+------------------------------------------------------------+------------+-----------------+
# minio_address | MinIO storage service IP address. | IP | 127.0.0.1 |
# s3_address | Simple Storage Service IP address. | IP | 127.0.0.1 |
#----------------------+------------------------------------------------------------+------------+-----------------+
# minio_port | MinIO storage service port. Port range (1024, 65535) | Integer | 9000 |
# s3_port | Simple Storage Service port. Port range (1024, 65535) | Integer | 9000 |
#----------------------+------------------------------------------------------------+------------+-----------------+
# minio_access_key | MinIO storage service access key. | String | minioadmin |
# s3_access_key | Simple Storage Service access key. | String | minioadmin |
#----------------------+------------------------------------------------------------+------------+-----------------+
# minio_secret_key | MinIO storage service secret key. | String | minioadmin |
# s3_secret_key | Simple Storage Service secret key. | String | minioadmin |
#----------------------+------------------------------------------------------------+------------+-----------------+
# minio_bucket | MinIO storage service bucket name. | String | milvus-bucket |
# s3_bucket | Simple Storage Service bucket name. | String | milvus-bucket |
#----------------------+------------------------------------------------------------+------------+-----------------+
storage_config:
primary_path: /var/lib/milvus
secondary_path:
minio_enable: false
minio_address: 127.0.0.1
minio_port: 9000
minio_access_key: minioadmin
minio_secret_key: minioadmin
minio_bucket: milvus-bucket
s3_enable: false
s3_address: 127.0.0.1
s3_port: 9000
s3_access_key: minioadmin
s3_secret_key: minioadmin
s3_bucket: milvus-bucket
#----------------------+------------------------------------------------------------+------------+-----------------+
# Metric Config | Description | Type | Default |

24
core/conf/server_cpu_config.template
View File

@ -70,27 +70,27 @@ db_config:
# secondary_path | A semicolon-separated list of secondary directories used | Path | |
# | to save vector data and index data. | | |
#----------------------+------------------------------------------------------------+------------+-----------------+
# minio_enable | Enable MinIO storage or not. | Boolean | false |
# s3_enable | Enable Simple Storage Service or not. | Boolean | false |
#----------------------+------------------------------------------------------------+------------+-----------------+
# minio_address | MinIO storage service IP address. | IP | 127.0.0.1 |
# s3_address | Simple Storage Service IP address. | IP | 127.0.0.1 |
#----------------------+------------------------------------------------------------+------------+-----------------+
# minio_port | MinIO storage service port. Port range (1024, 65535) | Integer | 9000 |
# s3_port | Simple Storage Service port. Port range (1024, 65535) | Integer | 9000 |
#----------------------+------------------------------------------------------------+------------+-----------------+
# minio_access_key | MinIO storage service access key. | String | minioadmin |
# s3_access_key | Simple Storage Service access key. | String | minioadmin |
#----------------------+------------------------------------------------------------+------------+-----------------+
# minio_secret_key | MinIO storage service secret key. | String | minioadmin |
# s3_secret_key | Simple Storage Service secret key. | String | minioadmin |
#----------------------+------------------------------------------------------------+------------+-----------------+
# minio_bucket | MinIO storage service bucket name. | String | milvus-bucket |
# s3_bucket | Simple Storage Service bucket name. | String | milvus-bucket |
#----------------------+------------------------------------------------------------+------------+-----------------+
storage_config:
primary_path: @MILVUS_DB_PATH@
secondary_path:
minio_enable: false
minio_address: 127.0.0.1
minio_port: 9000
minio_access_key: minioadmin
minio_secret_key: minioadmin
minio_bucket: milvus-bucket
s3_enable: false
s3_address: 127.0.0.1
s3_port: 9000
s3_access_key: minioadmin
s3_secret_key: minioadmin
s3_bucket: milvus-bucket
#----------------------+------------------------------------------------------------+------------+-----------------+
# Metric Config | Description | Type | Default |

24
core/conf/server_gpu_config.template
View File

@ -70,27 +70,27 @@ db_config:
# secondary_path | A semicolon-separated list of secondary directories used | Path | |
# | to save vector data and index data. | | |
#----------------------+------------------------------------------------------------+------------+-----------------+
# minio_enable | Enable MinIO storage or not. | Boolean | false |
# s3_enable | Enable Simple Storage Service or not. | Boolean | false |
#----------------------+------------------------------------------------------------+------------+-----------------+
# minio_address | MinIO storage service IP address. | IP | 127.0.0.1 |
# s3_address | Simple Storage Service IP address. | IP | 127.0.0.1 |
#----------------------+------------------------------------------------------------+------------+-----------------+
# minio_port | MinIO storage service port. Port range (1024, 65535) | Integer | 9000 |
# s3_port | Simple Storage Service port. Port range (1024, 65535) | Integer | 9000 |
#----------------------+------------------------------------------------------------+------------+-----------------+
# minio_access_key | MinIO storage service access key. | String | minioadmin |
# s3_access_key | Simple Storage Service access key. | String | minioadmin |
#----------------------+------------------------------------------------------------+------------+-----------------+
# minio_secret_key | MinIO storage service secret key. | String | minioadmin |
# s3_secret_key | Simple Storage Service secret key. | String | minioadmin |
#----------------------+------------------------------------------------------------+------------+-----------------+
# minio_bucket | MinIO storage service bucket name. | String | milvus-bucket |
# s3_bucket | Simple Storage Service bucket name. | String | milvus-bucket |
#----------------------+------------------------------------------------------------+------------+-----------------+
storage_config:
primary_path: @MILVUS_DB_PATH@
secondary_path:
minio_enable: false
minio_address: 127.0.0.1
minio_port: 9000
minio_access_key: minioadmin
minio_secret_key: minioadmin
minio_bucket: milvus-bucket
s3_enable: false
s3_address: 127.0.0.1
s3_port: 9000
s3_access_key: minioadmin
s3_secret_key: minioadmin
s3_bucket: milvus-bucket
#----------------------+------------------------------------------------------------+------------+-----------------+
# Metric Config | Description | Type | Default |

12
core/src/db/Utils.cpp
View File

@ -119,11 +119,11 @@ DeleteTablePath(const DBMetaOptions& options, const std::string& table_id, bool
}
}
bool minio_enable = false;
bool s3_enable = false;
server::Config& config = server::Config::GetInstance();
config.GetStorageConfigMinioEnable(minio_enable);
config.GetStorageConfigS3Enable(s3_enable);
if (minio_enable) {
if (s3_enable) {
std::string table_path = options.path_ + TABLES_FOLDER + table_id;
auto& storage_inst = milvus::storage::S3ClientWrapper::GetInstance();
@ -156,10 +156,10 @@ GetTableFilePath(const DBMetaOptions& options, meta::TableFileSchema& table_file
std::string parent_path = ConstructParentFolder(options.path_, table_file);
std::string file_path = parent_path + "/" + table_file.file_id_;
bool minio_enable = false;
bool s3_enable = false;
server::Config& config = server::Config::GetInstance();
config.GetStorageConfigMinioEnable(minio_enable);
if (minio_enable) {
config.GetStorageConfigS3Enable(s3_enable);
if (s3_enable) {
/* need not check file existence */
table_file.location_ = file_path;
return Status::OK();

133
core/src/server/Config.cpp
View File

@ -111,24 +111,24 @@ Config::ValidateConfig() {
std::string storage_secondary_path;
CONFIG_CHECK(GetStorageConfigSecondaryPath(storage_secondary_path));
bool storage_minio_enable;
CONFIG_CHECK(GetStorageConfigMinioEnable(storage_minio_enable));
std::cout << "MinIO " << (storage_minio_enable ? "ENABLED !" : "DISABLED !") << std::endl;
bool storage_s3_enable;
CONFIG_CHECK(GetStorageConfigS3Enable(storage_s3_enable));
std::cout << "S3 " << (storage_s3_enable ? "ENABLED !" : "DISABLED !") << std::endl;
std::string storage_minio_address;
CONFIG_CHECK(GetStorageConfigMinioAddress(storage_minio_address));
std::string storage_s3_address;
CONFIG_CHECK(GetStorageConfigS3Address(storage_s3_address));
std::string storage_minio_port;
CONFIG_CHECK(GetStorageConfigMinioPort(storage_minio_port));
std::string storage_s3_port;
CONFIG_CHECK(GetStorageConfigS3Port(storage_s3_port));
std::string storage_minio_access_key;
CONFIG_CHECK(GetStorageConfigMinioAccessKey(storage_minio_access_key));
std::string storage_s3_access_key;
CONFIG_CHECK(GetStorageConfigS3AccessKey(storage_s3_access_key));
std::string storage_minio_secret_key;
CONFIG_CHECK(GetStorageConfigMinioSecretKey(storage_minio_secret_key));
std::string storage_s3_secret_key;
CONFIG_CHECK(GetStorageConfigS3SecretKey(storage_s3_secret_key));
std::string storage_minio_bucket;
CONFIG_CHECK(GetStorageConfigMinioBucket(storage_minio_bucket));
std::string storage_s3_bucket;
CONFIG_CHECK(GetStorageConfigS3Bucket(storage_s3_bucket));
/* metric config */
bool metric_enable_monitor;
@ -208,12 +208,12 @@ Config::ResetDefaultConfig() {
/* storage config */
CONFIG_CHECK(SetStorageConfigPrimaryPath(CONFIG_STORAGE_PRIMARY_PATH_DEFAULT));
CONFIG_CHECK(SetStorageConfigSecondaryPath(CONFIG_STORAGE_SECONDARY_PATH_DEFAULT));
CONFIG_CHECK(SetStorageConfigMinioEnable(CONFIG_STORAGE_MINIO_ENABLE_DEFAULT));
CONFIG_CHECK(SetStorageConfigMinioAddress(CONFIG_STORAGE_MINIO_ADDRESS_DEFAULT));
CONFIG_CHECK(SetStorageConfigMinioPort(CONFIG_STORAGE_MINIO_PORT_DEFAULT));
CONFIG_CHECK(SetStorageConfigMinioAccessKey(CONFIG_STORAGE_MINIO_ACCESS_KEY_DEFAULT));
CONFIG_CHECK(SetStorageConfigMinioSecretKey(CONFIG_STORAGE_MINIO_SECRET_KEY_DEFAULT));
CONFIG_CHECK(SetStorageConfigMinioBucket(CONFIG_STORAGE_MINIO_BUCKET_DEFAULT));
CONFIG_CHECK(SetStorageConfigS3Enable(CONFIG_STORAGE_S3_ENABLE_DEFAULT));
CONFIG_CHECK(SetStorageConfigS3Address(CONFIG_STORAGE_S3_ADDRESS_DEFAULT));
CONFIG_CHECK(SetStorageConfigS3Port(CONFIG_STORAGE_S3_PORT_DEFAULT));
CONFIG_CHECK(SetStorageConfigS3AccessKey(CONFIG_STORAGE_S3_ACCESS_KEY_DEFAULT));
CONFIG_CHECK(SetStorageConfigS3SecretKey(CONFIG_STORAGE_S3_SECRET_KEY_DEFAULT));
CONFIG_CHECK(SetStorageConfigS3Bucket(CONFIG_STORAGE_S3_BUCKET_DEFAULT));
/* metric config */
CONFIG_CHECK(SetMetricConfigEnableMonitor(CONFIG_METRIC_ENABLE_MONITOR_DEFAULT));
@ -497,35 +497,34 @@ Config::CheckStorageConfigSecondaryPath(const std::string& value) {
}
Status
Config::CheckStorageConfigMinioEnable(const std::string& value) {
Config::CheckStorageConfigS3Enable(const std::string& value) {
if (!ValidationUtil::ValidateStringIsBool(value).ok()) {
std::string msg =
"Invalid storage config: " + value + ". Possible reason: storage_config.minio_enable is not a boolean.";
"Invalid storage config: " + value + ". Possible reason: storage_config.s3_enable is not a boolean.";
return Status(SERVER_INVALID_ARGUMENT, msg);
}
return Status::OK();
}
Status
Config::CheckStorageConfigMinioAddress(const std::string& value) {
Config::CheckStorageConfigS3Address(const std::string& value) {
if (!ValidationUtil::ValidateIpAddress(value).ok()) {
std::string msg =
"Invalid minio address: " + value + ". Possible reason: storage_config.minio_address is invalid.";
std::string msg = "Invalid s3 address: " + value + ". Possible reason: storage_config.s3_address is invalid.";
return Status(SERVER_INVALID_ARGUMENT, msg);
}
return Status::OK();
}
Status
Config::CheckStorageConfigMinioPort(const std::string& value) {
Config::CheckStorageConfigS3Port(const std::string& value) {
if (!ValidationUtil::ValidateStringIsNumber(value).ok()) {
std::string msg = "Invalid minio port: " + value + ". Possible reason: storage_config.port is not a number.";
std::string msg = "Invalid s3 port: " + value + ". Possible reason: storage_config.s3_port is not a number.";
return Status(SERVER_INVALID_ARGUMENT, msg);
} else {
int32_t port = std::stoi(value);
if (!(port > 1024 && port < 65535)) {
std::string msg = "Invalid minio port: " + value +
". Possible reason: storage_config.port is not in range (1024, 65535).";
std::string msg = "Invalid s3 port: " + value +
". Possible reason: storage_config.s3_port is not in range (1024, 65535).";
return Status(SERVER_INVALID_ARGUMENT, msg);
}
}
@ -533,25 +532,25 @@ Config::CheckStorageConfigMinioPort(const std::string& value) {
}
Status
Config::CheckStorageConfigMinioAccessKey(const std::string& value) {
Config::CheckStorageConfigS3AccessKey(const std::string& value) {
if (value.empty()) {
return Status(SERVER_INVALID_ARGUMENT, "storage_config.minio_access_key is empty.");
return Status(SERVER_INVALID_ARGUMENT, "storage_config.s3_access_key is empty.");
}
return Status::OK();
}
Status
Config::CheckStorageConfigMinioSecretKey(const std::string& value) {
Config::CheckStorageConfigS3SecretKey(const std::string& value) {
if (value.empty()) {
return Status(SERVER_INVALID_ARGUMENT, "storage_config.minio_secret_key is empty.");
return Status(SERVER_INVALID_ARGUMENT, "storage_config.s3_secret_key is empty.");
}
return Status::OK();
}
Status
Config::CheckStorageConfigMinioBucket(const std::string& value) {
Config::CheckStorageConfigS3Bucket(const std::string& value) {
if (value.empty()) {
return Status(SERVER_INVALID_ARGUMENT, "storage_config.minio_bucket is empty.");
return Status(SERVER_INVALID_ARGUMENT, "storage_config.s3_bucket is empty.");
}
return Status::OK();
}
@ -972,41 +971,41 @@ Config::GetStorageConfigSecondaryPath(std::string& value) {
}
Status
Config::GetStorageConfigMinioEnable(bool& value) {
std::string str = GetConfigStr(CONFIG_STORAGE, CONFIG_STORAGE_MINIO_ENABLE, CONFIG_STORAGE_MINIO_ENABLE_DEFAULT);
CONFIG_CHECK(CheckStorageConfigMinioEnable(str));
Config::GetStorageConfigS3Enable(bool& value) {
std::string str = GetConfigStr(CONFIG_STORAGE, CONFIG_STORAGE_S3_ENABLE, CONFIG_STORAGE_S3_ENABLE_DEFAULT);
CONFIG_CHECK(CheckStorageConfigS3Enable(str));
std::transform(str.begin(), str.end(), str.begin(), ::tolower);
value = (str == "true" || str == "on" || str == "yes" || str == "1");
return Status::OK();
}
Status
Config::GetStorageConfigMinioAddress(std::string& value) {
value = GetConfigStr(CONFIG_STORAGE, CONFIG_STORAGE_MINIO_ADDRESS, CONFIG_STORAGE_MINIO_ADDRESS_DEFAULT);
return CheckStorageConfigMinioAddress(value);
Config::GetStorageConfigS3Address(std::string& value) {
value = GetConfigStr(CONFIG_STORAGE, CONFIG_STORAGE_S3_ADDRESS, CONFIG_STORAGE_S3_ADDRESS_DEFAULT);
return CheckStorageConfigS3Address(value);
}
Status
Config::GetStorageConfigMinioPort(std::string& value) {
value = GetConfigStr(CONFIG_STORAGE, CONFIG_STORAGE_MINIO_PORT, CONFIG_STORAGE_MINIO_PORT_DEFAULT);
return CheckStorageConfigMinioPort(value);
Config::GetStorageConfigS3Port(std::string& value) {
value = GetConfigStr(CONFIG_STORAGE, CONFIG_STORAGE_S3_PORT, CONFIG_STORAGE_S3_PORT_DEFAULT);
return CheckStorageConfigS3Port(value);
}
Status
Config::GetStorageConfigMinioAccessKey(std::string& value) {
value = GetConfigStr(CONFIG_STORAGE, CONFIG_STORAGE_MINIO_ACCESS_KEY, CONFIG_STORAGE_MINIO_ACCESS_KEY_DEFAULT);
Config::GetStorageConfigS3AccessKey(std::string& value) {
value = GetConfigStr(CONFIG_STORAGE, CONFIG_STORAGE_S3_ACCESS_KEY, CONFIG_STORAGE_S3_ACCESS_KEY_DEFAULT);
return Status::OK();
}
Status
Config::GetStorageConfigMinioSecretKey(std::string& value) {
value = GetConfigStr(CONFIG_STORAGE, CONFIG_STORAGE_MINIO_SECRET_KEY, CONFIG_STORAGE_MINIO_SECRET_KEY_DEFAULT);
Config::GetStorageConfigS3SecretKey(std::string& value) {
value = GetConfigStr(CONFIG_STORAGE, CONFIG_STORAGE_S3_SECRET_KEY, CONFIG_STORAGE_S3_SECRET_KEY_DEFAULT);
return Status::OK();
}
Status
Config::GetStorageConfigMinioBucket(std::string& value) {
value = GetConfigStr(CONFIG_STORAGE, CONFIG_STORAGE_MINIO_BUCKET, CONFIG_STORAGE_MINIO_BUCKET_DEFAULT);
Config::GetStorageConfigS3Bucket(std::string& value) {
value = GetConfigStr(CONFIG_STORAGE, CONFIG_STORAGE_S3_BUCKET, CONFIG_STORAGE_S3_BUCKET_DEFAULT);
return Status::OK();
}
@ -1259,39 +1258,39 @@ Config::SetStorageConfigSecondaryPath(const std::string& value) {
}
Status
Config::SetStorageConfigMinioEnable(const std::string& value) {
CONFIG_CHECK(CheckStorageConfigMinioEnable(value));
return SetConfigValueInMem(CONFIG_STORAGE, CONFIG_STORAGE_MINIO_ENABLE, value);
Config::SetStorageConfigS3Enable(const std::string& value) {
CONFIG_CHECK(CheckStorageConfigS3Enable(value));
return SetConfigValueInMem(CONFIG_STORAGE, CONFIG_STORAGE_S3_ENABLE, value);
}
Status
Config::SetStorageConfigMinioAddress(const std::string& value) {
CONFIG_CHECK(CheckStorageConfigMinioAddress(value));
return SetConfigValueInMem(CONFIG_STORAGE, CONFIG_STORAGE_MINIO_ADDRESS, value);
Config::SetStorageConfigS3Address(const std::string& value) {
CONFIG_CHECK(CheckStorageConfigS3Address(value));
return SetConfigValueInMem(CONFIG_STORAGE, CONFIG_STORAGE_S3_ADDRESS, value);
}
Status
Config::SetStorageConfigMinioPort(const std::string& value) {
CONFIG_CHECK(CheckStorageConfigMinioPort(value));
return SetConfigValueInMem(CONFIG_STORAGE, CONFIG_STORAGE_MINIO_PORT, value);
Config::SetStorageConfigS3Port(const std::string& value) {
CONFIG_CHECK(CheckStorageConfigS3Port(value));
return SetConfigValueInMem(CONFIG_STORAGE, CONFIG_STORAGE_S3_PORT, value);
}
Status
Config::SetStorageConfigMinioAccessKey(const std::string& value) {
CONFIG_CHECK(CheckStorageConfigMinioAccessKey(value));
return SetConfigValueInMem(CONFIG_STORAGE, CONFIG_STORAGE_MINIO_ACCESS_KEY, value);
Config::SetStorageConfigS3AccessKey(const std::string& value) {
CONFIG_CHECK(CheckStorageConfigS3AccessKey(value));
return SetConfigValueInMem(CONFIG_STORAGE, CONFIG_STORAGE_S3_ACCESS_KEY, value);
}
Status
Config::SetStorageConfigMinioSecretKey(const std::string& value) {
CONFIG_CHECK(CheckStorageConfigMinioSecretKey(value));
return SetConfigValueInMem(CONFIG_STORAGE, CONFIG_STORAGE_MINIO_SECRET_KEY, value);
Config::SetStorageConfigS3SecretKey(const std::string& value) {
CONFIG_CHECK(CheckStorageConfigS3SecretKey(value));
return SetConfigValueInMem(CONFIG_STORAGE, CONFIG_STORAGE_S3_SECRET_KEY, value);
}
Status
Config::SetStorageConfigMinioBucket(const std::string& value) {
CONFIG_CHECK(CheckStorageConfigMinioBucket(value));
return SetConfigValueInMem(CONFIG_STORAGE, CONFIG_STORAGE_MINIO_BUCKET, value);
Config::SetStorageConfigS3Bucket(const std::string& value) {
CONFIG_CHECK(CheckStorageConfigS3Bucket(value));
return SetConfigValueInMem(CONFIG_STORAGE, CONFIG_STORAGE_S3_BUCKET, value);
}
/* metric config */

60
core/src/server/Config.h
View File

@ -71,18 +71,18 @@ static const char* CONFIG_STORAGE_PRIMARY_PATH = "primary_path";
static const char* CONFIG_STORAGE_PRIMARY_PATH_DEFAULT = "/tmp/milvus";
static const char* CONFIG_STORAGE_SECONDARY_PATH = "secondary_path";
static const char* CONFIG_STORAGE_SECONDARY_PATH_DEFAULT = "";
static const char* CONFIG_STORAGE_MINIO_ENABLE = "minio_enable";
static const char* CONFIG_STORAGE_MINIO_ENABLE_DEFAULT = "false";
static const char* CONFIG_STORAGE_MINIO_ADDRESS = "minio_address";
static const char* CONFIG_STORAGE_MINIO_ADDRESS_DEFAULT = "127.0.0.1";
static const char* CONFIG_STORAGE_MINIO_PORT = "minio_port";
static const char* CONFIG_STORAGE_MINIO_PORT_DEFAULT = "9000";
static const char* CONFIG_STORAGE_MINIO_ACCESS_KEY = "minio_access_key";
static const char* CONFIG_STORAGE_MINIO_ACCESS_KEY_DEFAULT = "minioadmin";
static const char* CONFIG_STORAGE_MINIO_SECRET_KEY = "minio_secret_key";
static const char* CONFIG_STORAGE_MINIO_SECRET_KEY_DEFAULT = "minioadmin";
static const char* CONFIG_STORAGE_MINIO_BUCKET = "minio_bucket";
static const char* CONFIG_STORAGE_MINIO_BUCKET_DEFAULT = "milvus-bucket";
static const char* CONFIG_STORAGE_S3_ENABLE = "s3_enable";
static const char* CONFIG_STORAGE_S3_ENABLE_DEFAULT = "false";
static const char* CONFIG_STORAGE_S3_ADDRESS = "s3_address";
static const char* CONFIG_STORAGE_S3_ADDRESS_DEFAULT = "127.0.0.1";
static const char* CONFIG_STORAGE_S3_PORT = "s3_port";
static const char* CONFIG_STORAGE_S3_PORT_DEFAULT = "9000";
static const char* CONFIG_STORAGE_S3_ACCESS_KEY = "s3_access_key";
static const char* CONFIG_STORAGE_S3_ACCESS_KEY_DEFAULT = "minioadmin";
static const char* CONFIG_STORAGE_S3_SECRET_KEY = "s3_secret_key";
static const char* CONFIG_STORAGE_S3_SECRET_KEY_DEFAULT = "minioadmin";
static const char* CONFIG_STORAGE_S3_BUCKET = "s3_bucket";
static const char* CONFIG_STORAGE_S3_BUCKET_DEFAULT = "milvus-bucket";
/* cache config */
static const char* CONFIG_CACHE = "cache_config";
@ -197,17 +197,17 @@ class Config {
Status
CheckStorageConfigSecondaryPath(const std::string& value);
Status
CheckStorageConfigMinioEnable(const std::string& value);
CheckStorageConfigS3Enable(const std::string& value);
Status
CheckStorageConfigMinioAddress(const std::string& value);
CheckStorageConfigS3Address(const std::string& value);
Status
CheckStorageConfigMinioPort(const std::string& value);
CheckStorageConfigS3Port(const std::string& value);
Status
CheckStorageConfigMinioAccessKey(const std::string& value);
CheckStorageConfigS3AccessKey(const std::string& value);
Status
CheckStorageConfigMinioSecretKey(const std::string& value);
CheckStorageConfigS3SecretKey(const std::string& value);
Status
CheckStorageConfigMinioBucket(const std::string& value);
CheckStorageConfigS3Bucket(const std::string& value);
/* metric config */
Status
@ -287,17 +287,17 @@ class Config {
Status
GetStorageConfigSecondaryPath(std::string& value);
Status
GetStorageConfigMinioEnable(bool& value);
GetStorageConfigS3Enable(bool& value);
Status
GetStorageConfigMinioAddress(std::string& value);
GetStorageConfigS3Address(std::string& value);
Status
GetStorageConfigMinioPort(std::string& value);
GetStorageConfigS3Port(std::string& value);
Status
GetStorageConfigMinioAccessKey(std::string& value);
GetStorageConfigS3AccessKey(std::string& value);
Status
GetStorageConfigMinioSecretKey(std::string& value);
GetStorageConfigS3SecretKey(std::string& value);
Status
GetStorageConfigMinioBucket(std::string& value);
GetStorageConfigS3Bucket(std::string& value);
/* metric config */
Status
@ -371,17 +371,17 @@ class Config {
Status
SetStorageConfigSecondaryPath(const std::string& value);
Status
SetStorageConfigMinioEnable(const std::string& value);
SetStorageConfigS3Enable(const std::string& value);
Status
SetStorageConfigMinioAddress(const std::string& value);
SetStorageConfigS3Address(const std::string& value);
Status
SetStorageConfigMinioPort(const std::string& value);
SetStorageConfigS3Port(const std::string& value);
Status
SetStorageConfigMinioAccessKey(const std::string& value);
SetStorageConfigS3AccessKey(const std::string& value);
Status
SetStorageConfigMinioSecretKey(const std::string& value);
SetStorageConfigS3SecretKey(const std::string& value);
Status
SetStorageConfigMinioBucket(const std::string& value);
SetStorageConfigS3Bucket(const std::string& value);
/* metric config */
Status

48
core/src/storage/s3/S3ClientWrapper.cpp
View File

@ -40,28 +40,28 @@ namespace storage {
Status
S3ClientWrapper::StartService() {
server::Config& config = server::Config::GetInstance();
bool minio_enable = false;
CONFIG_CHECK(config.GetStorageConfigMinioEnable(minio_enable));
fiu_do_on("S3ClientWrapper.StartService.minio_disable", minio_enable = false);
if (!minio_enable) {
STORAGE_LOG_INFO << "MinIO not enabled!";
bool s3_enable = false;
CONFIG_CHECK(config.GetStorageConfigS3Enable(s3_enable));
fiu_do_on("S3ClientWrapper.StartService.s3_disable", s3_enable = false);
if (!s3_enable) {
STORAGE_LOG_INFO << "S3 not enabled!";
return Status::OK();
}
CONFIG_CHECK(config.GetStorageConfigMinioAddress(minio_address_));
CONFIG_CHECK(config.GetStorageConfigMinioPort(minio_port_));
CONFIG_CHECK(config.GetStorageConfigMinioAccessKey(minio_access_key_));
CONFIG_CHECK(config.GetStorageConfigMinioSecretKey(minio_secret_key_));
CONFIG_CHECK(config.GetStorageConfigMinioBucket(minio_bucket_));
CONFIG_CHECK(config.GetStorageConfigS3Address(s3_address_));
CONFIG_CHECK(config.GetStorageConfigS3Port(s3_port_));
CONFIG_CHECK(config.GetStorageConfigS3AccessKey(s3_access_key_));
CONFIG_CHECK(config.GetStorageConfigS3SecretKey(s3_secret_key_));
CONFIG_CHECK(config.GetStorageConfigS3Bucket(s3_bucket_));
Aws::InitAPI(options_);
Aws::Client::ClientConfiguration cfg;
cfg.endpointOverride = minio_address_ + ":" + minio_port_;
cfg.endpointOverride = s3_address_ + ":" + s3_port_;
cfg.scheme = Aws::Http::Scheme::HTTP;
cfg.verifySSL = false;
client_ptr_ =
std::make_shared<Aws::S3::S3Client>(Aws::Auth::AWSCredentials(minio_access_key_, minio_secret_key_), cfg,
std::make_shared<Aws::S3::S3Client>(Aws::Auth::AWSCredentials(s3_access_key_, s3_secret_key_), cfg,
Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Always, false);
bool mock_enable = false;
@ -84,7 +84,7 @@ S3ClientWrapper::StopService() {
Status
S3ClientWrapper::CreateBucket() {
Aws::S3::Model::CreateBucketRequest request;
request.WithBucket(minio_bucket_);
request.WithBucket(s3_bucket_);
auto outcome = client_ptr_->CreateBucket(request);
@ -97,14 +97,14 @@ S3ClientWrapper::CreateBucket() {
}
}
STORAGE_LOG_DEBUG << "CreateBucket '" << minio_bucket_ << "' successfully!";
STORAGE_LOG_DEBUG << "CreateBucket '" << s3_bucket_ << "' successfully!";
return Status::OK();
}
Status
S3ClientWrapper::DeleteBucket() {
Aws::S3::Model::DeleteBucketRequest request;
request.WithBucket(minio_bucket_);
request.WithBucket(s3_bucket_);
auto outcome = client_ptr_->DeleteBucket(request);
@ -115,7 +115,7 @@ S3ClientWrapper::DeleteBucket() {
return Status(SERVER_UNEXPECTED_ERROR, err.GetMessage());
}
STORAGE_LOG_DEBUG << "DeleteBucket '" << minio_bucket_ << "' successfully!";
STORAGE_LOG_DEBUG << "DeleteBucket '" << s3_bucket_ << "' successfully!";
return Status::OK();
}
@ -129,7 +129,7 @@ S3ClientWrapper::PutObjectFile(const std::string& object_name, const std::string
}
Aws::S3::Model::PutObjectRequest request;
request.WithBucket(minio_bucket_).WithKey(object_name);
request.WithBucket(s3_bucket_).WithKey(object_name);
auto input_data =
Aws::MakeShared<Aws::FStream>("PutObjectFile", file_path.c_str(), std::ios_base::in | std::ios_base::binary);
@ -151,7 +151,7 @@ S3ClientWrapper::PutObjectFile(const std::string& object_name, const std::string
Status
S3ClientWrapper::PutObjectStr(const std::string& object_name, const std::string& content) {
Aws::S3::Model::PutObjectRequest request;
request.WithBucket(minio_bucket_).WithKey(object_name);
request.WithBucket(s3_bucket_).WithKey(object_name);
const std::shared_ptr<Aws::IOStream> input_data = Aws::MakeShared<Aws::StringStream>("");
input_data->write(content.data(), content.length());
@ -173,7 +173,7 @@ S3ClientWrapper::PutObjectStr(const std::string& object_name, const std::string&
Status
S3ClientWrapper::GetObjectFile(const std::string& object_name, const std::string& file_path) {
Aws::S3::Model::GetObjectRequest request;
request.WithBucket(minio_bucket_).WithKey(object_name);
request.WithBucket(s3_bucket_).WithKey(object_name);
auto outcome = client_ptr_->GetObject(request);
@ -196,7 +196,7 @@ S3ClientWrapper::GetObjectFile(const std::string& object_name, const std::string
Status
S3ClientWrapper::GetObjectStr(const std::string& object_name, std::string& content) {
Aws::S3::Model::GetObjectRequest request;
request.WithBucket(minio_bucket_).WithKey(object_name);
request.WithBucket(s3_bucket_).WithKey(object_name);
auto outcome = client_ptr_->GetObject(request);
@ -219,7 +219,7 @@ S3ClientWrapper::GetObjectStr(const std::string& object_name, std::string& conte
Status
S3ClientWrapper::ListObjects(std::vector<std::string>& object_list, const std::string& marker) {
Aws::S3::Model::ListObjectsRequest request;
request.WithBucket(minio_bucket_);
request.WithBucket(s3_bucket_);
if (!marker.empty()) {
request.WithMarker(marker);
@ -241,9 +241,9 @@ S3ClientWrapper::ListObjects(std::vector<std::string>& object_list, const std::s
}
if (marker.empty()) {
STORAGE_LOG_DEBUG << "ListObjects '" << minio_bucket_ << "' successfully!";
STORAGE_LOG_DEBUG << "ListObjects '" << s3_bucket_ << "' successfully!";
} else {
STORAGE_LOG_DEBUG << "ListObjects '" << minio_bucket_ << ":" << marker << "' successfully!";
STORAGE_LOG_DEBUG << "ListObjects '" << s3_bucket_ << ":" << marker << "' successfully!";
}
return Status::OK();
}
@ -251,7 +251,7 @@ S3ClientWrapper::ListObjects(std::vector<std::string>& object_list, const std::s
Status
S3ClientWrapper::DeleteObject(const std::string& object_name) {
Aws::S3::Model::DeleteObjectRequest request;
request.WithBucket(minio_bucket_).WithKey(object_name);
request.WithBucket(s3_bucket_).WithKey(object_name);
auto outcome = client_ptr_->DeleteObject(request);

10
core/src/storage/s3/S3ClientWrapper.h
View File

@ -63,11 +63,11 @@ class S3ClientWrapper : public IStorage {
std::shared_ptr<Aws::S3::S3Client> client_ptr_;
Aws::SDKOptions options_;
std::string minio_address_;
std::string minio_port_;
std::string minio_access_key_;
std::string minio_secret_key_;
std::string minio_bucket_;
std::string s3_address_;
std::string s3_port_;
std::string s3_access_key_;
std::string s3_secret_key_;
std::string s3_bucket_;
};
} // namespace storage

12
core/src/wrapper/VecIndex.cpp
View File

@ -167,12 +167,12 @@ read_index(const std::string& location) {
TimeRecorder recorder("read_index");
knowhere::BinarySet load_data_list;
bool minio_enable = false;
bool s3_enable = false;
server::Config& config = server::Config::GetInstance();
config.GetStorageConfigMinioEnable(minio_enable);
config.GetStorageConfigS3Enable(s3_enable);
std::shared_ptr<storage::IOReader> reader_ptr;
if (minio_enable) {
if (s3_enable) {
reader_ptr = std::make_shared<storage::S3IOReader>(location);
} else {
reader_ptr = std::make_shared<storage::FileIOReader>(location);
@ -235,12 +235,12 @@ write_index(VecIndexPtr index, const std::string& location) {
auto binaryset = index->Serialize();
auto index_type = index->GetType();
bool minio_enable = false;
bool s3_enable = false;
server::Config& config = server::Config::GetInstance();
config.GetStorageConfigMinioEnable(minio_enable);
config.GetStorageConfigS3Enable(s3_enable);
std::shared_ptr<storage::IOWriter> writer_ptr;
if (minio_enable) {
if (s3_enable) {
writer_ptr = std::make_shared<storage::S3IOWriter>(location);
} else {
writer_ptr = std::make_shared<storage::FileIOWriter>(location);

68
core/unittest/server/test_config.cpp
View File

@ -182,35 +182,35 @@ TEST_F(ConfigTest, SERVER_CONFIG_VALID_TEST) {
ASSERT_TRUE(config.GetStorageConfigSecondaryPath(str_val).ok());
ASSERT_TRUE(str_val == storage_secondary_path);
bool storage_minio_enable = true;
ASSERT_TRUE(config.SetStorageConfigMinioEnable(std::to_string(storage_minio_enable)).ok());
ASSERT_TRUE(config.GetStorageConfigMinioEnable(bool_val).ok());
ASSERT_TRUE(bool_val == storage_minio_enable);
bool storage_s3_enable = true;
ASSERT_TRUE(config.SetStorageConfigS3Enable(std::to_string(storage_s3_enable)).ok());
ASSERT_TRUE(config.GetStorageConfigS3Enable(bool_val).ok());
ASSERT_TRUE(bool_val == storage_s3_enable);
std::string storage_minio_addr = "192.168.1.100";
ASSERT_TRUE(config.SetStorageConfigMinioAddress(storage_minio_addr).ok());
ASSERT_TRUE(config.GetStorageConfigMinioAddress(str_val).ok());
ASSERT_TRUE(str_val == storage_minio_addr);
std::string storage_s3_addr = "192.168.1.100";
ASSERT_TRUE(config.SetStorageConfigS3Address(storage_s3_addr).ok());
ASSERT_TRUE(config.GetStorageConfigS3Address(str_val).ok());
ASSERT_TRUE(str_val == storage_s3_addr);
std::string storage_minio_port = "12345";
ASSERT_TRUE(config.SetStorageConfigMinioPort(storage_minio_port).ok());
ASSERT_TRUE(config.GetStorageConfigMinioPort(str_val).ok());
ASSERT_TRUE(str_val == storage_minio_port);
std::string storage_s3_port = "12345";
ASSERT_TRUE(config.SetStorageConfigS3Port(storage_s3_port).ok());
ASSERT_TRUE(config.GetStorageConfigS3Port(str_val).ok());
ASSERT_TRUE(str_val == storage_s3_port);
std::string storage_minio_access_key = "minioadmin";
ASSERT_TRUE(config.SetStorageConfigMinioAccessKey(storage_minio_access_key).ok());
ASSERT_TRUE(config.GetStorageConfigMinioAccessKey(str_val).ok());
ASSERT_TRUE(str_val == storage_minio_access_key);
std::string storage_s3_access_key = "minioadmin";
ASSERT_TRUE(config.SetStorageConfigS3AccessKey(storage_s3_access_key).ok());
ASSERT_TRUE(config.GetStorageConfigS3AccessKey(str_val).ok());
ASSERT_TRUE(str_val == storage_s3_access_key);
std::string storage_minio_secret_key = "minioadmin";
ASSERT_TRUE(config.SetStorageConfigMinioSecretKey(storage_minio_secret_key).ok());
ASSERT_TRUE(config.GetStorageConfigMinioSecretKey(str_val).ok());
ASSERT_TRUE(str_val == storage_minio_secret_key);
std::string storage_s3_secret_key = "minioadmin";
ASSERT_TRUE(config.SetStorageConfigS3SecretKey(storage_s3_secret_key).ok());
ASSERT_TRUE(config.GetStorageConfigS3SecretKey(str_val).ok());
ASSERT_TRUE(str_val == storage_s3_secret_key);
std::string storage_minio_bucket = "miniobucket";
ASSERT_TRUE(config.SetStorageConfigMinioBucket(storage_minio_bucket).ok());
ASSERT_TRUE(config.GetStorageConfigMinioBucket(str_val).ok());
ASSERT_TRUE(str_val == storage_minio_bucket);
std::string storage_s3_bucket = "s3bucket";
ASSERT_TRUE(config.SetStorageConfigS3Bucket(storage_s3_bucket).ok());
ASSERT_TRUE(config.GetStorageConfigS3Bucket(str_val).ok());
ASSERT_TRUE(str_val == storage_s3_bucket);
/* metric config */
bool metric_enable_monitor = false;
@ -354,9 +354,9 @@ TEST_F(ConfigTest, SERVER_CONFIG_CLI_TEST) {
ASSERT_TRUE(s.ok());
/* storage config */
std::string storage_minio_enable = "true";
get_cmd = gen_get_command(ms::CONFIG_STORAGE, ms::CONFIG_STORAGE_MINIO_ENABLE);
set_cmd = gen_set_command(ms::CONFIG_STORAGE, ms::CONFIG_STORAGE_MINIO_ENABLE, storage_minio_enable);
std::string storage_s3_enable = "true";
get_cmd = gen_get_command(ms::CONFIG_STORAGE, ms::CONFIG_STORAGE_S3_ENABLE);
set_cmd = gen_set_command(ms::CONFIG_STORAGE, ms::CONFIG_STORAGE_S3_ENABLE, storage_s3_enable);
s = config.ProcessConfigCli(dummy, set_cmd);
ASSERT_FALSE(s.ok());
s = config.ProcessConfigCli(result, get_cmd);
@ -512,18 +512,18 @@ TEST_F(ConfigTest, SERVER_CONFIG_INVALID_TEST) {
// ASSERT_FALSE(config.SetStorageConfigSecondaryPath("").ok());
ASSERT_FALSE(config.SetStorageConfigMinioEnable("10").ok());
ASSERT_FALSE(config.SetStorageConfigS3Enable("10").ok());
ASSERT_FALSE(config.SetStorageConfigMinioAddress("127.0.0").ok());
ASSERT_FALSE(config.SetStorageConfigS3Address("127.0.0").ok());
ASSERT_FALSE(config.SetStorageConfigMinioPort("100").ok());
ASSERT_FALSE(config.SetStorageConfigMinioPort("100000").ok());
ASSERT_FALSE(config.SetStorageConfigS3Port("100").ok());
ASSERT_FALSE(config.SetStorageConfigS3Port("100000").ok());
ASSERT_FALSE(config.SetStorageConfigMinioAccessKey("").ok());
ASSERT_FALSE(config.SetStorageConfigS3AccessKey("").ok());
ASSERT_FALSE(config.SetStorageConfigMinioSecretKey("").ok());
ASSERT_FALSE(config.SetStorageConfigS3SecretKey("").ok());
ASSERT_FALSE(config.SetStorageConfigMinioBucket("").ok());
ASSERT_FALSE(config.SetStorageConfigS3Bucket("").ok());
/* metric config */
ASSERT_FALSE(config.SetMetricConfigEnableMonitor("Y").ok());

4
core/unittest/storage/test_s3_client.cpp
View File

@ -144,9 +144,9 @@ TEST_F(StorageTest, S3_FAIL_TEST) {
auto& storage_inst = milvus::storage::S3ClientWrapper::GetInstance();
fiu_enable("S3ClientWrapper.StartService.minio_disable", 1, NULL, 0);
fiu_enable("S3ClientWrapper.StartService.s3_disable", 1, NULL, 0);
ASSERT_TRUE(storage_inst.StartService().ok());
fiu_disable("S3ClientWrapper.StartService.minio_disable");
fiu_disable("S3ClientWrapper.StartService.s3_disable");
fiu_enable("S3ClientWrapper.StartService.mock_enable", 1, NULL, 0);
ASSERT_TRUE(storage_inst.StartService().ok());

12
core/unittest/storage/utils.cpp
View File

@ -28,12 +28,12 @@ static const char* CONFIG_STR =
"storage_config:\n"
" primary_path: /tmp/milvus\n"
" secondary_path:\n"
" minio_enable: true\n"
" minio_address: 127.0.0.1\n"
" minio_port: 9000\n"
" minio_access_key: minioadmin\n"
" minio_secret_key: minioadmin\n"
" minio_bucket: milvus-bucket\n"
" s3_enable: true\n"
" s3_address: 127.0.0.1\n"
" s3_port: 9000\n"
" s3_access_key: minioadmin\n"
" s3_secret_key: minioadmin\n"
" s3_bucket: milvus-bucket\n"
"\n";
void