admin/milvus
1
0
Fork 0
mirror of https://gitee.com/milvus-io/milvus.git synced 2026-02-02 01:06:41 +08:00
Code Issues Packages Projects Releases Wiki Activity

Change to 0.8.0 and rename from preload_table to preload_collection (#1904)

Browse Source 
* Create new branch 0.8.0 and change preload_table to preload_collection

Signed-off-by: jinhai <hai.jin@zilliz.com>

* Fix format

Signed-off-by: JinHai-CN <hai.jin@zilliz.com>

* Update CHANGELOG

Signed-off-by: jinhai <hai.jin@zilliz.com>

* Update CHANGELOG

Signed-off-by: jinhai <hai.jin@zilliz.com>

* update helm version

Signed-off-by: zw <zw@zilliz.com>

Co-authored-by: zw <zw@zilliz.com>
This commit is contained in:
Jin Hai 2020-04-11 19:01:29 +08:00 committed by GitHub
parent d0a01c4e59
commit f8a28699b4
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
22 changed files with 64 additions and 65 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
CHANGELOG.md
View File

@ -659,7 +659,7 @@ Please mark all change in change log and use the issue from GitHub
- MS-65 Implement GetTableRowCount interface
- MS-45 Implement DeleteTable interface
- MS-75 cmake: change faiss version to 1.5.2; add CUDA gencode
- MS-81 fix faiss ptx issue; change cuda gencode
- MS-81 Fix faiss ptx issue; change cuda gencode
- MS-84 cmake: add arrow, jemalloc and jsoncons third party; default build option OFF
- MS-85 add NetIO metric
- MS-96 add new query interface for specified files
@ -698,7 +698,7 @@ Please mark all change in change log and use the issue from GitHub
- MS-26 CMake. Add thirdparty packages
- MS-31 cmake: add prometheus
- MS-33 cmake: add -j4 to make third party packages build faster
- MS-27 support gpu config and disable license build config in cmake
- MS-27 Support gpu config and disable license build config in cmake
- MS-47 Add query vps metrics
- MS-37 Add query, cache usage, disk write speed and file data size metrics
- MS-30 Use faiss v1.5.2

10
ci/jenkins/step/singleDevNightlyTest.groovy
View File

@ -1,8 +1,8 @@
dir ('milvus-helm') {
sh 'helm version'
sh 'helm repo add stable https://kubernetes.oss-cn-hangzhou.aliyuncs.com/charts'
sh 'helm repo update'
checkout([$class: 'GitSCM', branches: [[name: "master"]], userRemoteConfigs: [[url: "https://github.com/milvus-io/milvus-helm.git", name: 'origin', refspec: "+refs/heads/master:refs/remotes/origin/master"]]])
sh 'helm version'
sh 'helm repo add stable https://kubernetes.oss-cn-hangzhou.aliyuncs.com/charts'
sh 'helm repo update'
checkout([$class: 'GitSCM', branches: [[name: "0.8.0"]], userRemoteConfigs: [[url: "https://github.com/milvus-io/milvus-helm.git", name: 'origin', refspec: "+refs/heads/0.8.0:refs/remotes/origin/0.8.0"]]])
sh "helm install --wait --timeout 600s --set image.repository=registry.zilliz.com/milvus/engine --set image.tag=${DOCKER_VERSION} --set image.pullPolicy=Always --set service.type=ClusterIP -f ci/db_backend/mysql_${BINARY_VERSION}_values.yaml -f ci/filebeat/values.yaml --namespace milvus ${env.HELM_RELEASE_NAME} ."
}
@ -16,7 +16,7 @@ load "ci/jenkins/step/cleanupSingleDev.groovy"
if (!fileExists('milvus-helm')) {
dir ("milvus-helm") {
checkout([$class: 'GitSCM', branches: [[name: "master"]], userRemoteConfigs: [[url: "https://github.com/milvus-io/milvus-helm.git", name: 'origin', refspec: "+refs/heads/master:refs/remotes/origin/master"]]])
checkout([$class: 'GitSCM', branches: [[name: "0.8.0"]], userRemoteConfigs: [[url: "https://github.com/milvus-io/milvus-helm.git", name: 'origin', refspec: "+refs/heads/0.8.0:refs/remotes/origin/0.8.0"]]])
}
}
dir ("milvus-helm") {

8
ci/jenkins/step/singleDevTest.groovy
View File

@ -1,8 +1,8 @@
dir ('milvus-helm') {
sh 'helm version'
sh 'helm repo add stable https://kubernetes.oss-cn-hangzhou.aliyuncs.com/charts'
sh 'helm repo update'
checkout([$class: 'GitSCM', branches: [[name: "master"]], userRemoteConfigs: [[url: "https://github.com/milvus-io/milvus-helm.git", name: 'origin', refspec: "+refs/heads/master:refs/remotes/origin/master"]]])
sh 'helm version'
sh 'helm repo add stable https://kubernetes.oss-cn-hangzhou.aliyuncs.com/charts'
sh 'helm repo update'
checkout([$class: 'GitSCM', branches: [[name: "0.8.0"]], userRemoteConfigs: [[url: "https://github.com/milvus-io/milvus-helm.git", name: 'origin', refspec: "+refs/heads/0.8.0:refs/remotes/origin/0.8.0"]]])
sh "helm install --wait --timeout 600s --set image.repository=registry.zilliz.com/milvus/engine --set image.tag=${DOCKER_VERSION} --set image.pullPolicy=Always --set service.type=ClusterIP -f ci/db_backend/mysql_${BINARY_VERSION}_values.yaml -f ci/filebeat/values.yaml --namespace milvus ${env.HELM_RELEASE_NAME} ."
}

2
core/CMakeLists.txt
View File

@ -90,7 +90,7 @@ if (MILVUS_VERSION_MAJOR STREQUAL ""
OR MILVUS_VERSION_MINOR STREQUAL ""
OR MILVUS_VERSION_PATCH STREQUAL "")
message(WARNING "Failed to determine Milvus version from git branch name")
set(MILVUS_VERSION "0.7.1")
set(MILVUS_VERSION "0.8.0")
endif ()
message(STATUS "Build version = ${MILVUS_VERSION}")

8
core/conf/demo/server_config.yaml
View File

@ -9,7 +9,7 @@
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
# or implied. See the License for the specific language governing permissions and limitations under the License.
version: 0.2
version: 0.3
#----------------------+------------------------------------------------------------+------------+-----------------+
# Server Config | Description | Type | Default |
@ -42,8 +42,8 @@ server_config:
# | Keep 'dialect://:@:/', 'dialect' can be either 'sqlite' or | | |
# | 'mysql', replace other texts with real values. | | |
#----------------------+------------------------------------------------------------+------------+-----------------+
# preload_table | A comma-separated list of table names that need to be pre- | StringList | |
# | loaded when Milvus server starts up. | | |
# preload_collection | A comma-separated list of collection names that need to | StringList | |
# | be pre-loaded when Milvus server starts up. | | |
# | '*' means preload all existing tables (single-quote or | | |
# | double-quote required). | | |
#----------------------+------------------------------------------------------------+------------+-----------------+
@ -53,7 +53,7 @@ server_config:
#----------------------+------------------------------------------------------------+------------+-----------------+
db_config:
backend_url: sqlite://:@:/
preload_table:
preload_collection:
auto_flush_interval: 1
#----------------------+------------------------------------------------------------+------------+-----------------+

8
core/conf/server_cpu_config.template
View File

@ -9,7 +9,7 @@
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
# or implied. See the License for the specific language governing permissions and limitations under the License.
version: 0.2
version: 0.3
#----------------------+------------------------------------------------------------+------------+-----------------+
# Server Config | Description | Type | Default |
@ -42,8 +42,8 @@ server_config:
# | Keep 'dialect://:@:/', 'dialect' can be either 'sqlite' or | | |
# | 'mysql', replace other texts with real values. | | |
#----------------------+------------------------------------------------------------+------------+-----------------+
# preload_table | A comma-separated list of table names that need to be pre- | StringList | |
# | loaded when Milvus server starts up. | | |
# preload_collection | A comma-separated list of collection names that need to | StringList | |
# | be pre-loaded when Milvus server starts up. | | |
# | '*' means preload all existing tables (single-quote or | | |
# | double-quote required). | | |
#----------------------+------------------------------------------------------------+------------+-----------------+
@ -53,7 +53,7 @@ server_config:
#----------------------+------------------------------------------------------------+------------+-----------------+
db_config:
backend_url: sqlite://:@:/
preload_table:
preload_collection:
auto_flush_interval: 1
#----------------------+------------------------------------------------------------+------------+-----------------+

8
core/conf/server_gpu_config.template
View File

@ -9,7 +9,7 @@
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
# or implied. See the License for the specific language governing permissions and limitations under the License.
version: 0.2
version: 0.3
#----------------------+------------------------------------------------------------+------------+-----------------+
# Server Config | Description | Type | Default |
@ -42,8 +42,8 @@ server_config:
# | Keep 'dialect://:@:/', 'dialect' can be either 'sqlite' or | | |
# | 'mysql', replace other texts with real values. | | |
#----------------------+------------------------------------------------------------+------------+-----------------+
# preload_table | A comma-separated list of table names that need to be pre- | StringList | |
# | loaded when Milvus server starts up. | | |
# preload_collection | A comma-separated list of collection names that need to | StringList | |
# | be pre-loaded when Milvus server starts up. | | |
# | '*' means preload all existing tables (single-quote or | | |
# | double-quote required). | | |
#----------------------+------------------------------------------------------------+------------+-----------------+
@ -53,7 +53,7 @@ server_config:
#----------------------+------------------------------------------------------------+------------+-----------------+
db_config:
backend_url: sqlite://:@:/
preload_table:
preload_collection:
auto_flush_interval: 1
#----------------------+------------------------------------------------------------+------------+-----------------+

13
core/src/config/Config.cpp
View File

@ -39,9 +39,8 @@ namespace server {
constexpr int64_t GB = 1UL << 30;
static const std::unordered_map<std::string, std::string> milvus_config_version_map({{"0.6.0", "0.1"},
{"0.7.0", "0.2"},
{"0.7.1", "0.2"}});
static const std::unordered_map<std::string, std::string> milvus_config_version_map(
{{"0.6.0", "0.1"}, {"0.7.0", "0.2"}, {"0.7.1", "0.2"}, {"0.8.0", "0.3"}});
/////////////////////////////////////////////////////////////
Config::Config() {
@ -133,8 +132,8 @@ Config::ValidateConfig() {
std::string db_backend_url;
CONFIG_CHECK(GetDBConfigBackendUrl(db_backend_url));
std::string db_preload_table;
CONFIG_CHECK(GetDBConfigPreloadCollection(db_preload_table));
std::string db_preload_collection;
CONFIG_CHECK(GetDBConfigPreloadCollection(db_preload_collection));
int64_t db_archive_disk_threshold;
CONFIG_CHECK(GetDBConfigArchiveDiskThreshold(db_archive_disk_threshold));
@ -777,7 +776,7 @@ Config::CheckDBConfigBackendUrl(const std::string& value) {
Status
Config::CheckDBConfigPreloadCollection(const std::string& value) {
fiu_return_on("check_config_preload_table_fail", Status(SERVER_INVALID_ARGUMENT, ""));
fiu_return_on("check_config_preload_collection_fail", Status(SERVER_INVALID_ARGUMENT, ""));
if (value.empty() || value == "*") {
return Status::OK();
@ -803,7 +802,7 @@ Config::CheckDBConfigPreloadCollection(const std::string& value) {
if (table_set.size() != tables.size()) {
std::string msg =
"Invalid preload tables. "
"Possible reason: db_config.preload_table contains duplicate collection.";
"Possible reason: db_config.preload_collection contains duplicate collection.";
return Status(SERVER_INVALID_ARGUMENT, msg);
}

2
core/src/config/Config.h
View File

@ -57,7 +57,7 @@ static const char* CONFIG_DB_ARCHIVE_DISK_THRESHOLD = "archive_disk_threshold";
static const char* CONFIG_DB_ARCHIVE_DISK_THRESHOLD_DEFAULT = "0";
static const char* CONFIG_DB_ARCHIVE_DAYS_THRESHOLD = "archive_days_threshold";
static const char* CONFIG_DB_ARCHIVE_DAYS_THRESHOLD_DEFAULT = "0";
static const char* CONFIG_DB_PRELOAD_COLLECTION = "preload_table";
static const char* CONFIG_DB_PRELOAD_COLLECTION = "preload_collection";
static const char* CONFIG_DB_PRELOAD_COLLECTION_DEFAULT = "";
static const char* CONFIG_DB_AUTO_FLUSH_INTERVAL = "auto_flush_interval";
static const char* CONFIG_DB_AUTO_FLUSH_INTERVAL_DEFAULT = "1";

16
core/src/server/DBWrapper.cpp
View File

@ -205,16 +205,16 @@ DBWrapper::StartService() {
db_->Start();
// preload collection
std::string preload_tables;
s = config.GetDBConfigPreloadCollection(preload_tables);
std::string preload_collections;
s = config.GetDBConfigPreloadCollection(preload_collections);
if (!s.ok()) {
std::cerr << s.ToString() << std::endl;
return s;
}
s = PreloadCollections(preload_tables);
s = PreloadCollections(preload_collections);
if (!s.ok()) {
std::cerr << "ERROR! Failed to preload tables: " << preload_tables << std::endl;
std::cerr << "ERROR! Failed to preload tables: " << preload_collections << std::endl;
std::cerr << s.ToString() << std::endl;
kill(0, SIGUSR1);
}
@ -232,10 +232,10 @@ DBWrapper::StopService() {
}
Status
DBWrapper::PreloadCollections(const std::string& preload_tables) {
if (preload_tables.empty()) {
DBWrapper::PreloadCollections(const std::string& preload_collections) {
if (preload_collections.empty()) {
// do nothing
} else if (preload_tables == "*") {
} else if (preload_collections == "*") {
// load all tables
std::vector<engine::meta::CollectionSchema> table_schema_array;
db_->AllCollections(table_schema_array);
@ -248,7 +248,7 @@ DBWrapper::PreloadCollections(const std::string& preload_tables) {
}
} else {
std::vector<std::string> collection_names;
StringHelpFunctions::SplitStringByDelimeter(preload_tables, ",", collection_names);
StringHelpFunctions::SplitStringByDelimeter(preload_collections, ",", collection_names);
for (auto& name : collection_names) {
auto status = db_->PreloadCollection(name);
if (!status.ok()) {

2
core/src/server/DBWrapper.h
View File

@ -48,7 +48,7 @@ class DBWrapper {
private:
Status
PreloadCollections(const std::string& preload_tables);
PreloadCollections(const std::string& preload_collections);
private:
engine::DBPtr db_;

2
core/src/server/web_impl/README.md
View File

@ -1388,7 +1388,7 @@ $ curl -X GET "http://127.0.0.1:19121/system/version" -H "accept: application/js
##### Response
```json
{"code":0,"message":"OK","reply": "0.7.1" }
{"code":0,"message":"OK","reply": "0.8.0" }
```
### `system/{op}` (PUT)

4
core/unittest/server/test_config.cpp
View File

@ -964,10 +964,10 @@ TEST_F(ConfigTest, SERVER_CONFIG_RESET_DEFAULT_CONFIG_FAIL_TEST) {
ASSERT_FALSE(s.ok());
fiu_disable("check_config_backend_url_fail");
fiu_enable("check_config_preload_table_fail", 1, NULL, 0);
fiu_enable("check_config_preload_collection_fail", 1, NULL, 0);
s = config.ResetDefaultConfig();
ASSERT_FALSE(s.ok());
fiu_disable("check_config_preload_table_fail");
fiu_disable("check_config_preload_collection_fail");
fiu_enable("check_config_archive_disk_threshold_fail", 1, NULL, 0);
s = config.ResetDefaultConfig();

6
core/unittest/server/test_web.cpp
View File

@ -523,7 +523,7 @@ static const char* CONTROLLER_TEST_VALID_CONFIG_STR =
"-+\n"
"db_config:\n"
" backend_url: sqlite://:@:/\n"
" preload_table:\n"
" preload_collection:\n"
"\n"
"#----------------------+------------------------------------------------------------+------------+----------------"
"-+\n"
@ -1660,11 +1660,11 @@ TEST_F(WebControllerTest, CONFIG) {
OString collection_name_s = "milvus_test_webcontroller_test_preload_collection_s";
GenTable(client_ptr, conncetion_ptr, collection_name_s, 16, 10, "L2");
OString body_str = "{\"db_config\": {\"preload_table\": \"" + collection_name + "\"}}";
OString body_str = "{\"db_config\": {\"preload_collection\": \"" + collection_name + "\"}}";
response = client_ptr->op("config", body_str, conncetion_ptr);
ASSERT_EQ(OStatus::CODE_200.code, response->getStatusCode()) << response->readBodyToString()->c_str();
body_str = "{\"db_config\": {\"preload_table\": \"" + collection_name + "," + collection_name_s + "\"}}";
body_str = "{\"db_config\": {\"preload_collection\": \"" + collection_name + "," + collection_name_s + "\"}}";
response = client_ptr->op("config", body_str, conncetion_ptr);
ASSERT_EQ(OStatus::CODE_200.code, response->getStatusCode()) << response->readBodyToString()->c_str();
auto set_result_json = nlohmann::json::parse(response->readBodyToString()->c_str());

4
core/unittest/server/utils.cpp
View File

@ -22,7 +22,7 @@ namespace {
static const char* VALID_CONFIG_STR =
"# Default values are used when you make no changes to the following parameters.\n"
"\n"
"version: 0.2"
"version: 0.3"
"\n"
"server_config:\n"
" address: 0.0.0.0 # milvus server ip address (IPv4)\n"
@ -32,7 +32,7 @@ static const char* VALID_CONFIG_STR =
"\n"
"db_config:\n"
" backend_url: sqlite://:@:/ \n"
" preload_table: \n"
" preload_collection: \n"
"\n"
"storage_config:\n"
" primary_path: /tmp/milvus # path used to store data and meta\n"

2
sdk/grpc/ClientProxy.cpp
View File

@ -17,7 +17,7 @@
#include "grpc-gen/gen-milvus/milvus.grpc.pb.h"
#define MILVUS_SDK_VERSION "0.7.1";
#define MILVUS_SDK_VERSION "0.8.0";
namespace milvus {

2
tests/milvus_benchmark/conf/server_config.yaml
View File

@ -9,7 +9,7 @@ db_config:
secondary_path:
backend_url: sqlite://:@:/
insert_buffer_size: 4
preload_table:
preload_collection:
metric_config:
enable_monitor: true

2
tests/milvus_benchmark/conf/server_config.yaml.cpu
View File

@ -10,7 +10,7 @@ db_config:
backend_url: sqlite://:@:/
insert_buffer_size: 4
build_index_gpu: 0
preload_table:
preload_collection:
metric_config:
enable_monitor: false

2
tests/milvus_benchmark/conf/server_config.yaml.multi
View File

@ -10,7 +10,7 @@ db_config:
backend_url: sqlite://:@:/
insert_buffer_size: 4
build_index_gpu: 0
preload_table:
preload_collection:
metric_config:
enable_monitor: false

2
tests/milvus_benchmark/conf/server_config.yaml.single
View File

@ -10,7 +10,7 @@ db_config:
backend_url: sqlite://:@:/
insert_buffer_size: 4
build_index_gpu: 0
preload_table:
preload_collection:
metric_config:
enable_monitor: false

20
tests/milvus_python_test/test_config.py
View File

@ -1278,25 +1278,25 @@ class TestDBConfig:
assert status.OK()
@pytest.mark.level(2)
def test_get_preload_table_invalid_child_key(self, connect, collection):
def test_get_preload_collection_invalid_child_key(self, connect, collection):
'''
target: get invalid child key
method: call get_config without child_key: preload_table
method: call get_config without child_key: preload_collection
expected: status not ok
'''
invalid_configs = ["preloadtable", "preload_table "]
invalid_configs = ["preloadtable", "preload_collection "]
for config in invalid_configs:
status, config_value = connect.get_config("db_config", config)
assert not status.OK()
@pytest.mark.timeout(CONFIG_TIMEOUT)
def test_get_preload_table_valid(self, connect, collection):
def test_get_preload_collection_valid(self, connect, collection):
'''
target: get preload_table
target: get preload_collection
method: call get_config correctly
expected: status ok
'''
status, config_value = connect.get_config("db_config", "preload_table")
status, config_value = connect.get_config("db_config", "preload_collection")
assert status.OK()
@pytest.mark.level(2)
@ -1350,15 +1350,15 @@ class TestDBConfig:
assert status.OK()
assert config_value == 'sqlite://:@:/'
def test_set_preload_table_valid(self, connect, collection):
def test_set_preload_collection_valid(self, connect, collection):
'''
target: set preload_table
target: set preload_collection
method: call set_config correctly
expected: status ok, set successfully
'''
status, reply = connect.set_config("db_config", "preload_table", "")
status, reply = connect.set_config("db_config", "preload_collection", "")
assert status.OK()
status, config_value = connect.get_config("db_config", "preload_table")
status, config_value = connect.get_config("db_config", "preload_collection")
assert status.OK()
assert config_value == ""

2
tests/milvus_python_test/test_ping.py
View File

@ -1,7 +1,7 @@
import logging
import pytest
__version__ = '0.7.1'
__version__ = '0.8.0'
class TestPing: