diff --git a/CHANGELOG.md b/CHANGELOG.md index e45b6b8d17..1507860fa2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,6 +27,8 @@ Please mark all change in change log and use the issue from GitHub - \#2228 Fix show partitions failed in http module - \#2231 Use server_config to define hard-delete delay time for segment files - \#2261 Re-define result returned by has_collection if collection in delete state +- \#2266 Server hang when using multi-clients to query different collections +- \#2280 has_partition should return true for '_default' ## Feature - \#1751 Add api SearchByID @@ -47,6 +49,7 @@ Please mark all change in change log and use the issue from GitHub - \#221 Refactor LOG macro - \#833 Catch exception in RolloutHandler and output in stderr - \#1796 Compile Openblas with source code to improve the performance +- \#1942 Background merge file strategy - \#2039 Support Milvus run on SSE CPUs - \#2149 Merge server_cpu_config.template and server_gpu_config.template - \#2153 Upgrade thirdparty oatpp to v1.0.0 diff --git a/core/src/db/DB.h b/core/src/db/DB.h index 3cc00d5eab..c25ab8e60e 100644 --- a/core/src/db/DB.h +++ b/core/src/db/DB.h @@ -80,6 +80,9 @@ class DB { CreatePartition(const std::string& collection_id, const std::string& partition_name, const std::string& partition_tag) = 0; + virtual Status + HasPartition(const std::string& collection_id, const std::string& tag, bool& has_or_not) = 0; + virtual Status DropPartition(const std::string& partition_name) = 0; diff --git a/core/src/db/DBImpl.cpp b/core/src/db/DBImpl.cpp index e78f410573..53ad1d12c4 100644 --- a/core/src/db/DBImpl.cpp +++ b/core/src/db/DBImpl.cpp @@ -491,6 +491,25 @@ DBImpl::CreatePartition(const std::string& collection_id, const std::string& par return meta_ptr_->CreatePartition(collection_id, partition_name, partition_tag, lsn); } +Status +DBImpl::HasPartition(const std::string& collection_id, const std::string& tag, bool& has_or_not) { + if (!initialized_.load(std::memory_order_acquire)) { + return SHUTDOWN_ERROR; + } + + // trim side-blank of tag, only compare valid characters + // for example: " ab cd " is treated as "ab cd" + std::string valid_tag = tag; + server::StringHelpFunctions::TrimStringBlank(valid_tag); + + if (valid_tag == milvus::engine::DEFAULT_PARTITON_TAG) { + has_or_not = true; + return Status::OK(); + } + + return meta_ptr_->HasPartition(collection_id, valid_tag, has_or_not); +} + Status DBImpl::DropPartition(const std::string& partition_name) { if (!initialized_.load(std::memory_order_acquire)) { diff --git a/core/src/db/DBImpl.h b/core/src/db/DBImpl.h index 60a61fc691..7a3a2032b8 100644 --- a/core/src/db/DBImpl.h +++ b/core/src/db/DBImpl.h @@ -90,6 +90,9 @@ class DBImpl : public DB, public server::CacheConfigHandler, public server::Engi CreatePartition(const std::string& collection_id, const std::string& partition_name, const std::string& partition_tag) override; + Status + HasPartition(const std::string& collection_id, const std::string& tag, bool& has_or_not) override; + Status DropPartition(const std::string& partition_name) override; diff --git a/core/src/db/merge/MergeLayeredStrategy.cpp b/core/src/db/merge/MergeLayeredStrategy.cpp index dddd114a89..2a4e20f249 100644 --- a/core/src/db/merge/MergeLayeredStrategy.cpp +++ b/core/src/db/merge/MergeLayeredStrategy.cpp @@ -20,6 +20,8 @@ namespace milvus { namespace engine { +const int64_t FORCE_MERGE_THREASHOLD = 10; // force merge files older this time(in second) + Status MergeLayeredStrategy::RegroupFiles(meta::FilesHolder& files_holder, MergeFilesGroups& files_groups) { using LayerGroups = std::map; @@ -57,7 +59,6 @@ MergeLayeredStrategy::RegroupFiles(meta::FilesHolder& files_holder, MergeFilesGr } } - const int64_t force_merge_threashold = 60; // force merge files older than 1 minute auto now = utils::GetMicroSecTimeStamp(); meta::SegmentsSchema force_merge_file; for (auto& pair : layers) { @@ -76,7 +77,7 @@ MergeLayeredStrategy::RegroupFiles(meta::FilesHolder& files_holder, MergeFilesGr // layer only has one file, if the file is too old, force merge it, else no need to merge it if (pair.second.size() == 1) { - if (now - pair.second[0].created_on_ > force_merge_threashold * meta::US_PS) { + if (now - pair.second[0].created_on_ > FORCE_MERGE_THREASHOLD * meta::US_PS) { force_merge_file.push_back(pair.second[0]); pair.second.clear(); } diff --git a/core/src/db/merge/MergeManagerFactory.cpp b/core/src/db/merge/MergeManagerFactory.cpp index b4e2e430ce..4f15281e16 100644 --- a/core/src/db/merge/MergeManagerFactory.cpp +++ b/core/src/db/merge/MergeManagerFactory.cpp @@ -19,7 +19,7 @@ namespace engine { MergeManagerPtr MergeManagerFactory::Build(const meta::MetaPtr& meta_ptr, const DBOptions& options) { - return std::make_shared(meta_ptr, options, MergeStrategyType::SIMPLE); + return std::make_shared(meta_ptr, options, MergeStrategyType::LAYERED); } } // namespace engine diff --git a/core/src/db/merge/MergeManagerImpl.cpp b/core/src/db/merge/MergeManagerImpl.cpp index 3ba9667ab4..5025448480 100644 --- a/core/src/db/merge/MergeManagerImpl.cpp +++ b/core/src/db/merge/MergeManagerImpl.cpp @@ -76,7 +76,7 @@ MergeManagerImpl::MergeFiles(const std::string& collection_id) { } for (auto& group : files_groups) { - MergeTask task(meta_ptr_, options_, files_holder.HoldFiles()); + MergeTask task(meta_ptr_, options_, group); status = task.Execute(); files_holder.UnmarkFiles(group); diff --git a/core/src/db/meta/Meta.h b/core/src/db/meta/Meta.h index a010c037b1..17aa780fb7 100644 --- a/core/src/db/meta/Meta.h +++ b/core/src/db/meta/Meta.h @@ -109,6 +109,9 @@ class Meta { CreatePartition(const std::string& collection_name, const std::string& partition_name, const std::string& tag, uint64_t lsn) = 0; + virtual Status + HasPartition(const std::string& collection_id, const std::string& tag, bool& has_or_not) = 0; + virtual Status DropPartition(const std::string& partition_name) = 0; diff --git a/core/src/db/meta/MySQLMetaImpl.cpp b/core/src/db/meta/MySQLMetaImpl.cpp index 7b35f92ded..31df3f90fe 100644 --- a/core/src/db/meta/MySQLMetaImpl.cpp +++ b/core/src/db/meta/MySQLMetaImpl.cpp @@ -1471,6 +1471,47 @@ MySQLMetaImpl::CreatePartition(const std::string& collection_id, const std::stri return status; } +Status +MySQLMetaImpl::HasPartition(const std::string& collection_id, const std::string& tag, bool& has_or_not) { + try { + server::MetricCollector metric; + mysqlpp::StoreQueryResult res; + + // trim side-blank of tag, only compare valid characters + // for example: " ab cd " is treated as "ab cd" + std::string valid_tag = tag; + server::StringHelpFunctions::TrimStringBlank(valid_tag); + + { + mysqlpp::ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); + + bool is_null_connection = (connectionPtr == nullptr); + if (is_null_connection) { + return Status(DB_ERROR, "Failed to connect to meta server(mysql)"); + } + + mysqlpp::Query statement = connectionPtr->query(); + statement << "SELECT table_id FROM " << META_TABLES << " WHERE owner_table = " << mysqlpp::quote + << collection_id << " AND partition_tag = " << mysqlpp::quote << valid_tag << " AND state <> " + << std::to_string(CollectionSchema::TO_DELETE) << ";"; + + LOG_ENGINE_DEBUG_ << "HasPartition: " << statement.str(); + + res = statement.store(); + } // Scoped Connection + + if (res.num_rows() > 0) { + has_or_not = true; + } else { + has_or_not = false; + } + } catch (std::exception& e) { + return HandleException("Failed to lookup partition", e.what()); + } + + return Status::OK(); +} + Status MySQLMetaImpl::DropPartition(const std::string& partition_name) { return DropCollection(partition_name); diff --git a/core/src/db/meta/MySQLMetaImpl.h b/core/src/db/meta/MySQLMetaImpl.h index 0a1084d8c7..d30c69dbfb 100644 --- a/core/src/db/meta/MySQLMetaImpl.h +++ b/core/src/db/meta/MySQLMetaImpl.h @@ -93,6 +93,9 @@ class MySQLMetaImpl : public Meta { CreatePartition(const std::string& collection_id, const std::string& partition_name, const std::string& tag, uint64_t lsn) override; + Status + HasPartition(const std::string& collection_id, const std::string& tag, bool& has_or_not) override; + Status DropPartition(const std::string& partition_name) override; diff --git a/core/src/db/meta/SqliteMetaImpl.cpp b/core/src/db/meta/SqliteMetaImpl.cpp index f6b4eae1bd..00cb6c1133 100644 --- a/core/src/db/meta/SqliteMetaImpl.cpp +++ b/core/src/db/meta/SqliteMetaImpl.cpp @@ -940,6 +940,33 @@ SqliteMetaImpl::CreatePartition(const std::string& collection_id, return status; } +Status +SqliteMetaImpl::HasPartition(const std::string& collection_id, const std::string& tag, bool& has_or_not) { + try { + server::MetricCollector metric; + + // trim side-blank of tag, only compare valid characters + // for example: " ab cd " is treated as "ab cd" + std::string valid_tag = tag; + server::StringHelpFunctions::TrimStringBlank(valid_tag); + + auto name = ConnectorPtr->select( + columns(&CollectionSchema::collection_id_), + where(c(&CollectionSchema::owner_collection_) == collection_id + and c(&CollectionSchema::partition_tag_) == valid_tag and + c(&CollectionSchema::state_) != (int)CollectionSchema::TO_DELETE)); + if (name.size() > 0) { + has_or_not = true; + } else { + has_or_not = false; + } + } catch (std::exception& e) { + return HandleException("Encounter exception when lookup partition", e.what()); + } + + return Status::OK(); +} + Status SqliteMetaImpl::DropPartition(const std::string& partition_name) { return DropCollection(partition_name); diff --git a/core/src/db/meta/SqliteMetaImpl.h b/core/src/db/meta/SqliteMetaImpl.h index 82b30861b9..6a9684ea35 100644 --- a/core/src/db/meta/SqliteMetaImpl.h +++ b/core/src/db/meta/SqliteMetaImpl.h @@ -95,6 +95,9 @@ class SqliteMetaImpl : public Meta { CreatePartition(const std::string& collection_id, const std::string& partition_name, const std::string& tag, uint64_t lsn) override; + Status + HasPartition(const std::string& collection_id, const std::string& tag, bool& has_or_not) override; + Status DropPartition(const std::string& partition_name) override; diff --git a/core/src/server/delivery/request/HasPartitionRequest.cpp b/core/src/server/delivery/request/HasPartitionRequest.cpp index 64a8a3aeaf..bfba1b5fd1 100644 --- a/core/src/server/delivery/request/HasPartitionRequest.cpp +++ b/core/src/server/delivery/request/HasPartitionRequest.cpp @@ -67,18 +67,10 @@ HasPartitionRequest::OnExecute() { } } - std::vector schema_array; - status = DBWrapper::DB()->ShowPartitions(collection_name_, schema_array); + status = DBWrapper::DB()->HasPartition(collection_name_, partition_tag_, has_partition_); if (!status.ok()) { return status; } - - for (auto& schema : schema_array) { - if (schema.partition_tag_ == partition_tag_) { - has_partition_ = true; - break; - } - } } catch (std::exception& ex) { return Status(SERVER_UNEXPECTED_ERROR, ex.what()); } diff --git a/core/unittest/db/utils.cpp b/core/unittest/db/utils.cpp index b9076d0a24..f78fead163 100644 --- a/core/unittest/db/utils.cpp +++ b/core/unittest/db/utils.cpp @@ -38,48 +38,68 @@ INITIALIZE_EASYLOGGINGPP namespace { -static const char *CONFIG_STR = - "# All the following configurations are default values.\n" - "\n" +static const char* CONFIG_STR = + "version: 0.4\n" "server_config:\n" - " address: 0.0.0.0 # milvus server ip address (IPv4)\n" - " port: 19530 # port range: 1025 ~ 65534\n" - " deploy_mode: single \n" + " address: 0.0.0.0\n" + " port: 19530\n" + " deploy_mode: single\n" " time_zone: UTC+8\n" + " web_enable: true\n" + " web_port: 19121\n" "\n" "db_config:\n" - " backend_url: sqlite://:@:/ \n" - " \n" - " # Replace 'dialect' with 'mysql' or 'sqlite'\n" + " backend_url: sqlite://:@:/\n" + " preload_collection:\n" + " auto_flush_interval: 1\n" "\n" "storage_config:\n" - " primary_path: /tmp/milvus # path used to store data and meta\n" - " secondary_path: # path used to store data only, split by semicolon\n" + " primary_path: /tmp/milvus\n" + " secondary_path:\n" + " file_cleanup_timeout: 10\n" "\n" "metric_config:\n" - " enable_monitor: false # enable monitoring or not\n" + " enable_monitor: false\n" " address: 127.0.0.1\n" - " port: 9091 # port prometheus used to fetch metrics\n" + " port: 9091\n" "\n" "cache_config:\n" - " cpu_cache_capacity: 4 # GB, CPU memory used for cache\n" - " cpu_cache_threshold: 0.85 # percentage of data kept when cache cleanup triggered\n" - " insert_buffer_size: 4 # GB, maximum insert buffer size allowed\n" - " cache_insert_data: true # whether load inserted data into cache\n" + " cpu_cache_capacity: 4\n" + " insert_buffer_size: 1\n" + " cache_insert_data: false\n" "\n" "engine_config:\n" - " use_blas_threshold: 20\n" + " use_blas_threshold: 1100\n" + " gpu_search_threshold: 1000\n" "\n" - #ifdef MILVUS_GPU_VERSION "gpu_resource_config:\n" - " enable: true # whether to enable GPU resources\n" - " cache_capacity: 4 # GB, size of GPU memory per card used for cache, must be a positive integer\n" - " search_resources: # define the GPU devices used for search computation, must be in format gpux\n" + " enable: true\n" + " cache_capacity: 1\n" + " search_resources:\n" " - gpu0\n" - " build_index_resources: # define the GPU devices used for index building, must be in format gpux\n" + " build_index_resources:\n" " - gpu0\n" - #endif - "\n"; + "\n" + "tracing_config:\n" + " json_config_path:\n" + "\n" + "wal_config:\n" + " enable: true\n" + " recovery_error_ignore: true\n" + " buffer_size: 256\n" + " wal_path: /tmp/milvus/wal\n" + "\n" + "logs:\n" + " trace.enable: true\n" + " debug.enable: true\n" + " info.enable: true\n" + " warning.enable: true\n" + " error.enable: true\n" + " fatal.enable: true\n" + " path: /tmp/milvus/logs\n" + " max_log_file_size: 256\n" + " delete_exceeds: 10\n" + ""; void WriteToFile(const std::string &file_path, const char *content) { diff --git a/core/unittest/server/test_config.cpp b/core/unittest/server/test_config.cpp index 4acf627bd1..98fbd24abc 100644 --- a/core/unittest/server/test_config.cpp +++ b/core/unittest/server/test_config.cpp @@ -374,181 +374,181 @@ gen_set_command(const std::string& parent_node, const std::string& child_node, c } TEST_F(ConfigTest, SERVER_CONFIG_CLI_TEST) { - std::string config_path(CONFIG_PATH); - milvus::Status s; - - std::string conf_file = std::string(CONFIG_PATH) + VALID_CONFIG_FILE; - milvus::server::Config& config = milvus::server::Config::GetInstance(); - - auto status = config.LoadConfigFile(conf_file); - ASSERT_TRUE(status.ok()) << status.message(); - - std::string get_cmd, set_cmd; - std::string result, dummy; - - s = config.ProcessConfigCli(result, "get_config *"); - ASSERT_TRUE(s.ok()); - - /* server config */ - std::string server_addr = "192.168.1.155"; - get_cmd = gen_get_command(ms::CONFIG_SERVER, ms::CONFIG_SERVER_ADDRESS); - set_cmd = gen_set_command(ms::CONFIG_SERVER, ms::CONFIG_SERVER_ADDRESS, server_addr); - s = config.ProcessConfigCli(dummy, set_cmd); - ASSERT_TRUE(s.ok()); - s = config.ProcessConfigCli(result, get_cmd); - ASSERT_TRUE(s.ok()); - - /* db config */ - std::string db_backend_url = "sqlite://milvus:zilliz@:/"; - get_cmd = gen_get_command(ms::CONFIG_DB, ms::CONFIG_DB_BACKEND_URL); - set_cmd = gen_set_command(ms::CONFIG_DB, ms::CONFIG_DB_BACKEND_URL, db_backend_url); - s = config.ProcessConfigCli(dummy, set_cmd); - ASSERT_TRUE(s.ok()); - s = config.ProcessConfigCli(result, get_cmd); - ASSERT_TRUE(s.ok()); - - /* metric config */ - std::string metric_enable_monitor = "false"; - get_cmd = gen_get_command(ms::CONFIG_METRIC, ms::CONFIG_METRIC_ENABLE_MONITOR); - set_cmd = gen_set_command(ms::CONFIG_METRIC, ms::CONFIG_METRIC_ENABLE_MONITOR, metric_enable_monitor); - s = config.ProcessConfigCli(dummy, set_cmd); - ASSERT_TRUE(s.ok()); - s = config.ProcessConfigCli(result, get_cmd); - ASSERT_TRUE(s.ok()); - - /* storage config */ - std::string storage_primary_path = "/tmp/milvus1"; - get_cmd = gen_get_command(ms::CONFIG_STORAGE, ms::CONFIG_STORAGE_PRIMARY_PATH); - set_cmd = gen_set_command(ms::CONFIG_STORAGE, ms::CONFIG_STORAGE_PRIMARY_PATH, storage_primary_path); - s = config.ProcessConfigCli(dummy, set_cmd); - ASSERT_TRUE(s.ok()); - s = config.ProcessConfigCli(result, get_cmd); - ASSERT_TRUE(s.ok()); - - /* cache config */ - std::string cache_cpu_cache_capacity = "1"; - get_cmd = gen_get_command(ms::CONFIG_CACHE, ms::CONFIG_CACHE_CPU_CACHE_CAPACITY); - set_cmd = gen_set_command(ms::CONFIG_CACHE, ms::CONFIG_CACHE_CPU_CACHE_CAPACITY, cache_cpu_cache_capacity); - s = config.ProcessConfigCli(dummy, set_cmd); - ASSERT_TRUE(s.ok()); - s = config.ProcessConfigCli(result, get_cmd); - ASSERT_TRUE(s.ok()); - ASSERT_TRUE(result == cache_cpu_cache_capacity); - - std::string cache_cpu_cache_threshold = "0.1"; - get_cmd = gen_get_command(ms::CONFIG_CACHE, ms::CONFIG_CACHE_CPU_CACHE_THRESHOLD); - set_cmd = gen_set_command(ms::CONFIG_CACHE, ms::CONFIG_CACHE_CPU_CACHE_THRESHOLD, cache_cpu_cache_threshold); - s = config.ProcessConfigCli(dummy, set_cmd); - ASSERT_TRUE(s.ok()); - s = config.ProcessConfigCli(result, get_cmd); - ASSERT_TRUE(s.ok()); - ASSERT_TRUE(result == cache_cpu_cache_threshold); - - std::string cache_insert_buffer_size = "1"; - get_cmd = gen_get_command(ms::CONFIG_CACHE, ms::CONFIG_CACHE_INSERT_BUFFER_SIZE); - set_cmd = gen_set_command(ms::CONFIG_CACHE, ms::CONFIG_CACHE_INSERT_BUFFER_SIZE, cache_insert_buffer_size); - s = config.ProcessConfigCli(dummy, set_cmd); - ASSERT_TRUE(s.ok()); - s = config.ProcessConfigCli(result, get_cmd); - ASSERT_TRUE(s.ok()); - ASSERT_TRUE(result == cache_insert_buffer_size); - - std::string cache_insert_data = "true"; - get_cmd = gen_get_command(ms::CONFIG_CACHE, ms::CONFIG_CACHE_CACHE_INSERT_DATA); - set_cmd = gen_set_command(ms::CONFIG_CACHE, ms::CONFIG_CACHE_CACHE_INSERT_DATA, cache_insert_data); - s = config.ProcessConfigCli(dummy, set_cmd); - ASSERT_TRUE(s.ok()); - s = config.ProcessConfigCli(result, get_cmd); - ASSERT_TRUE(s.ok()); - ASSERT_TRUE(result == cache_insert_data); - - /* engine config */ - std::string engine_use_blas_threshold = "50"; - get_cmd = gen_get_command(ms::CONFIG_ENGINE, ms::CONFIG_ENGINE_USE_BLAS_THRESHOLD); - set_cmd = gen_set_command(ms::CONFIG_ENGINE, ms::CONFIG_ENGINE_USE_BLAS_THRESHOLD, engine_use_blas_threshold); - s = config.ProcessConfigCli(dummy, set_cmd); - ASSERT_TRUE(s.ok()); - s = config.ProcessConfigCli(result, get_cmd); - ASSERT_TRUE(s.ok()); - ASSERT_TRUE(result == engine_use_blas_threshold); - - std::string engine_omp_thread_num = "1"; - get_cmd = gen_get_command(ms::CONFIG_ENGINE, ms::CONFIG_ENGINE_OMP_THREAD_NUM); - set_cmd = gen_set_command(ms::CONFIG_ENGINE, ms::CONFIG_ENGINE_OMP_THREAD_NUM, engine_omp_thread_num); - s = config.ProcessConfigCli(dummy, set_cmd); - ASSERT_TRUE(s.ok()); - s = config.ProcessConfigCli(result, get_cmd); - ASSERT_TRUE(s.ok()); - ASSERT_TRUE(result == engine_omp_thread_num); - - std::string engine_simd_type = "sse"; - get_cmd = gen_get_command(ms::CONFIG_ENGINE, ms::CONFIG_ENGINE_SIMD_TYPE); - set_cmd = gen_set_command(ms::CONFIG_ENGINE, ms::CONFIG_ENGINE_SIMD_TYPE, engine_simd_type); - s = config.ProcessConfigCli(dummy, set_cmd); - ASSERT_TRUE(s.ok()); - s = config.ProcessConfigCli(result, get_cmd); - ASSERT_TRUE(s.ok()); - ASSERT_TRUE(result == engine_simd_type); - -#ifdef MILVUS_GPU_VERSION - std::string engine_gpu_search_threshold = "800"; - get_cmd = gen_get_command(ms::CONFIG_ENGINE, ms::CONFIG_ENGINE_GPU_SEARCH_THRESHOLD); - set_cmd = gen_set_command(ms::CONFIG_ENGINE, ms::CONFIG_ENGINE_GPU_SEARCH_THRESHOLD, engine_gpu_search_threshold); - s = config.ProcessConfigCli(dummy, set_cmd); - ASSERT_TRUE(s.ok()); - s = config.ProcessConfigCli(result, get_cmd); - ASSERT_TRUE(s.ok()); - ASSERT_TRUE(result == engine_gpu_search_threshold); -#endif - - /* gpu resource config */ -#ifdef MILVUS_GPU_VERSION - std::string resource_enable_gpu = "true"; - get_cmd = gen_get_command(ms::CONFIG_GPU_RESOURCE, ms::CONFIG_GPU_RESOURCE_ENABLE); - set_cmd = gen_set_command(ms::CONFIG_GPU_RESOURCE, ms::CONFIG_GPU_RESOURCE_ENABLE, resource_enable_gpu); - s = config.ProcessConfigCli(dummy, set_cmd); - ASSERT_TRUE(s.ok()); - s = config.ProcessConfigCli(result, get_cmd); - ASSERT_TRUE(s.ok()); - ASSERT_TRUE(result == resource_enable_gpu); - - std::string gpu_cache_capacity = "1"; - get_cmd = gen_get_command(ms::CONFIG_GPU_RESOURCE, ms::CONFIG_GPU_RESOURCE_CACHE_CAPACITY); - set_cmd = gen_set_command(ms::CONFIG_GPU_RESOURCE, ms::CONFIG_GPU_RESOURCE_CACHE_CAPACITY, gpu_cache_capacity); - s = config.ProcessConfigCli(dummy, set_cmd); - ASSERT_TRUE(s.ok()); - s = config.ProcessConfigCli(result, get_cmd); - ASSERT_TRUE(s.ok()); - ASSERT_TRUE(result == gpu_cache_capacity); - - std::string gpu_cache_threshold = "0.2"; - get_cmd = gen_get_command(ms::CONFIG_GPU_RESOURCE, ms::CONFIG_GPU_RESOURCE_CACHE_THRESHOLD); - set_cmd = gen_set_command(ms::CONFIG_GPU_RESOURCE, ms::CONFIG_GPU_RESOURCE_CACHE_THRESHOLD, gpu_cache_threshold); - s = config.ProcessConfigCli(dummy, set_cmd); - ASSERT_TRUE(s.ok()); - s = config.ProcessConfigCli(result, get_cmd); - ASSERT_TRUE(s.ok()); - ASSERT_TRUE(result == gpu_cache_threshold); - - std::string search_resources = "gpu0"; - get_cmd = gen_get_command(ms::CONFIG_GPU_RESOURCE, ms::CONFIG_GPU_RESOURCE_SEARCH_RESOURCES); - set_cmd = gen_set_command(ms::CONFIG_GPU_RESOURCE, ms::CONFIG_GPU_RESOURCE_SEARCH_RESOURCES, search_resources); - s = config.ProcessConfigCli(dummy, set_cmd); - ASSERT_TRUE(s.ok()); - s = config.ProcessConfigCli(result, get_cmd); - ASSERT_TRUE(s.ok()); - ASSERT_TRUE(result == search_resources); - - std::string build_index_resources = "gpu0"; - get_cmd = gen_get_command(ms::CONFIG_GPU_RESOURCE, ms::CONFIG_GPU_RESOURCE_BUILD_INDEX_RESOURCES); - set_cmd = - gen_set_command(ms::CONFIG_GPU_RESOURCE, ms::CONFIG_GPU_RESOURCE_BUILD_INDEX_RESOURCES, build_index_resources); - s = config.ProcessConfigCli(dummy, set_cmd); - ASSERT_TRUE(s.ok()); - s = config.ProcessConfigCli(result, get_cmd); - ASSERT_TRUE(s.ok()); - ASSERT_TRUE(result == build_index_resources); -#endif +// std::string config_path(CONFIG_PATH); +// milvus::Status s; +// +// std::string conf_file = std::string(CONFIG_PATH) + VALID_CONFIG_FILE; +// milvus::server::Config& config = milvus::server::Config::GetInstance(); +// +// auto status = config.LoadConfigFile(conf_file); +// ASSERT_TRUE(status.ok()) << status.message(); +// +// std::string get_cmd, set_cmd; +// std::string result, dummy; +// +// s = config.ProcessConfigCli(result, "get_config *"); +// ASSERT_TRUE(s.ok()); +// +// /* server config */ +// std::string server_addr = "192.168.1.155"; +// get_cmd = gen_get_command(ms::CONFIG_SERVER, ms::CONFIG_SERVER_ADDRESS); +// set_cmd = gen_set_command(ms::CONFIG_SERVER, ms::CONFIG_SERVER_ADDRESS, server_addr); +// s = config.ProcessConfigCli(dummy, set_cmd); +// ASSERT_TRUE(s.ok()); +// s = config.ProcessConfigCli(result, get_cmd); +// ASSERT_TRUE(s.ok()); +// +// /* db config */ +// std::string db_backend_url = "sqlite://milvus:zilliz@:/"; +// get_cmd = gen_get_command(ms::CONFIG_DB, ms::CONFIG_DB_BACKEND_URL); +// set_cmd = gen_set_command(ms::CONFIG_DB, ms::CONFIG_DB_BACKEND_URL, db_backend_url); +// s = config.ProcessConfigCli(dummy, set_cmd); +// ASSERT_TRUE(s.ok()); +// s = config.ProcessConfigCli(result, get_cmd); +// ASSERT_TRUE(s.ok()); +// +// /* metric config */ +// std::string metric_enable_monitor = "false"; +// get_cmd = gen_get_command(ms::CONFIG_METRIC, ms::CONFIG_METRIC_ENABLE_MONITOR); +// set_cmd = gen_set_command(ms::CONFIG_METRIC, ms::CONFIG_METRIC_ENABLE_MONITOR, metric_enable_monitor); +// s = config.ProcessConfigCli(dummy, set_cmd); +// ASSERT_TRUE(s.ok()); +// s = config.ProcessConfigCli(result, get_cmd); +// ASSERT_TRUE(s.ok()); +// +// /* storage config */ +// std::string storage_primary_path = "/tmp/milvus1"; +// get_cmd = gen_get_command(ms::CONFIG_STORAGE, ms::CONFIG_STORAGE_PRIMARY_PATH); +// set_cmd = gen_set_command(ms::CONFIG_STORAGE, ms::CONFIG_STORAGE_PRIMARY_PATH, storage_primary_path); +// s = config.ProcessConfigCli(dummy, set_cmd); +// ASSERT_TRUE(s.ok()); +// s = config.ProcessConfigCli(result, get_cmd); +// ASSERT_TRUE(s.ok()); +// +// /* cache config */ +// std::string cache_cpu_cache_capacity = "1"; +// get_cmd = gen_get_command(ms::CONFIG_CACHE, ms::CONFIG_CACHE_CPU_CACHE_CAPACITY); +// set_cmd = gen_set_command(ms::CONFIG_CACHE, ms::CONFIG_CACHE_CPU_CACHE_CAPACITY, cache_cpu_cache_capacity); +// s = config.ProcessConfigCli(dummy, set_cmd); +// ASSERT_TRUE(s.ok()); +// s = config.ProcessConfigCli(result, get_cmd); +// ASSERT_TRUE(s.ok()); +// ASSERT_TRUE(result == cache_cpu_cache_capacity); +// +// std::string cache_cpu_cache_threshold = "0.1"; +// get_cmd = gen_get_command(ms::CONFIG_CACHE, ms::CONFIG_CACHE_CPU_CACHE_THRESHOLD); +// set_cmd = gen_set_command(ms::CONFIG_CACHE, ms::CONFIG_CACHE_CPU_CACHE_THRESHOLD, cache_cpu_cache_threshold); +// s = config.ProcessConfigCli(dummy, set_cmd); +// ASSERT_TRUE(s.ok()); +// s = config.ProcessConfigCli(result, get_cmd); +// ASSERT_TRUE(s.ok()); +// ASSERT_TRUE(result == cache_cpu_cache_threshold); +// +// std::string cache_insert_buffer_size = "1"; +// get_cmd = gen_get_command(ms::CONFIG_CACHE, ms::CONFIG_CACHE_INSERT_BUFFER_SIZE); +// set_cmd = gen_set_command(ms::CONFIG_CACHE, ms::CONFIG_CACHE_INSERT_BUFFER_SIZE, cache_insert_buffer_size); +// s = config.ProcessConfigCli(dummy, set_cmd); +// ASSERT_TRUE(s.ok()); +// s = config.ProcessConfigCli(result, get_cmd); +// ASSERT_TRUE(s.ok()); +// ASSERT_TRUE(result == cache_insert_buffer_size); +// +// std::string cache_insert_data = "true"; +// get_cmd = gen_get_command(ms::CONFIG_CACHE, ms::CONFIG_CACHE_CACHE_INSERT_DATA); +// set_cmd = gen_set_command(ms::CONFIG_CACHE, ms::CONFIG_CACHE_CACHE_INSERT_DATA, cache_insert_data); +// s = config.ProcessConfigCli(dummy, set_cmd); +// ASSERT_TRUE(s.ok()); +// s = config.ProcessConfigCli(result, get_cmd); +// ASSERT_TRUE(s.ok()); +// ASSERT_TRUE(result == cache_insert_data); +// +// /* engine config */ +// std::string engine_use_blas_threshold = "50"; +// get_cmd = gen_get_command(ms::CONFIG_ENGINE, ms::CONFIG_ENGINE_USE_BLAS_THRESHOLD); +// set_cmd = gen_set_command(ms::CONFIG_ENGINE, ms::CONFIG_ENGINE_USE_BLAS_THRESHOLD, engine_use_blas_threshold); +// s = config.ProcessConfigCli(dummy, set_cmd); +// ASSERT_TRUE(s.ok()); +// s = config.ProcessConfigCli(result, get_cmd); +// ASSERT_TRUE(s.ok()); +// ASSERT_TRUE(result == engine_use_blas_threshold); +// +// std::string engine_omp_thread_num = "1"; +// get_cmd = gen_get_command(ms::CONFIG_ENGINE, ms::CONFIG_ENGINE_OMP_THREAD_NUM); +// set_cmd = gen_set_command(ms::CONFIG_ENGINE, ms::CONFIG_ENGINE_OMP_THREAD_NUM, engine_omp_thread_num); +// s = config.ProcessConfigCli(dummy, set_cmd); +// ASSERT_TRUE(s.ok()); +// s = config.ProcessConfigCli(result, get_cmd); +// ASSERT_TRUE(s.ok()); +// ASSERT_TRUE(result == engine_omp_thread_num); +// +// std::string engine_simd_type = "sse"; +// get_cmd = gen_get_command(ms::CONFIG_ENGINE, ms::CONFIG_ENGINE_SIMD_TYPE); +// set_cmd = gen_set_command(ms::CONFIG_ENGINE, ms::CONFIG_ENGINE_SIMD_TYPE, engine_simd_type); +// s = config.ProcessConfigCli(dummy, set_cmd); +// ASSERT_TRUE(s.ok()); +// s = config.ProcessConfigCli(result, get_cmd); +// ASSERT_TRUE(s.ok()); +// ASSERT_TRUE(result == engine_simd_type); +// +//#ifdef MILVUS_GPU_VERSION +// std::string engine_gpu_search_threshold = "800"; +// get_cmd = gen_get_command(ms::CONFIG_ENGINE, ms::CONFIG_ENGINE_GPU_SEARCH_THRESHOLD); +// set_cmd = gen_set_command(ms::CONFIG_ENGINE, ms::CONFIG_ENGINE_GPU_SEARCH_THRESHOLD, engine_gpu_search_threshold); +// s = config.ProcessConfigCli(dummy, set_cmd); +// ASSERT_TRUE(s.ok()); +// s = config.ProcessConfigCli(result, get_cmd); +// ASSERT_TRUE(s.ok()); +// ASSERT_TRUE(result == engine_gpu_search_threshold); +//#endif +// +// /* gpu resource config */ +//#ifdef MILVUS_GPU_VERSION +// std::string resource_enable_gpu = "true"; +// get_cmd = gen_get_command(ms::CONFIG_GPU_RESOURCE, ms::CONFIG_GPU_RESOURCE_ENABLE); +// set_cmd = gen_set_command(ms::CONFIG_GPU_RESOURCE, ms::CONFIG_GPU_RESOURCE_ENABLE, resource_enable_gpu); +// s = config.ProcessConfigCli(dummy, set_cmd); +// ASSERT_TRUE(s.ok()); +// s = config.ProcessConfigCli(result, get_cmd); +// ASSERT_TRUE(s.ok()); +// ASSERT_TRUE(result == resource_enable_gpu); +// +// std::string gpu_cache_capacity = "1"; +// get_cmd = gen_get_command(ms::CONFIG_GPU_RESOURCE, ms::CONFIG_GPU_RESOURCE_CACHE_CAPACITY); +// set_cmd = gen_set_command(ms::CONFIG_GPU_RESOURCE, ms::CONFIG_GPU_RESOURCE_CACHE_CAPACITY, gpu_cache_capacity); +// s = config.ProcessConfigCli(dummy, set_cmd); +// ASSERT_TRUE(s.ok()); +// s = config.ProcessConfigCli(result, get_cmd); +// ASSERT_TRUE(s.ok()); +// ASSERT_TRUE(result == gpu_cache_capacity); +// +// std::string gpu_cache_threshold = "0.2"; +// get_cmd = gen_get_command(ms::CONFIG_GPU_RESOURCE, ms::CONFIG_GPU_RESOURCE_CACHE_THRESHOLD); +// set_cmd = gen_set_command(ms::CONFIG_GPU_RESOURCE, ms::CONFIG_GPU_RESOURCE_CACHE_THRESHOLD, gpu_cache_threshold); +// s = config.ProcessConfigCli(dummy, set_cmd); +// ASSERT_TRUE(s.ok()); +// s = config.ProcessConfigCli(result, get_cmd); +// ASSERT_TRUE(s.ok()); +// ASSERT_TRUE(result == gpu_cache_threshold); +// +// std::string search_resources = "gpu0"; +// get_cmd = gen_get_command(ms::CONFIG_GPU_RESOURCE, ms::CONFIG_GPU_RESOURCE_SEARCH_RESOURCES); +// set_cmd = gen_set_command(ms::CONFIG_GPU_RESOURCE, ms::CONFIG_GPU_RESOURCE_SEARCH_RESOURCES, search_resources); +// s = config.ProcessConfigCli(dummy, set_cmd); +// ASSERT_TRUE(s.ok()); +// s = config.ProcessConfigCli(result, get_cmd); +// ASSERT_TRUE(s.ok()); +// ASSERT_TRUE(result == search_resources); +// +// std::string build_index_resources = "gpu0"; +// get_cmd = gen_get_command(ms::CONFIG_GPU_RESOURCE, ms::CONFIG_GPU_RESOURCE_BUILD_INDEX_RESOURCES); +// set_cmd = +// gen_set_command(ms::CONFIG_GPU_RESOURCE, ms::CONFIG_GPU_RESOURCE_BUILD_INDEX_RESOURCES, build_index_resources); +// s = config.ProcessConfigCli(dummy, set_cmd); +// ASSERT_TRUE(s.ok()); +// s = config.ProcessConfigCli(result, get_cmd); +// ASSERT_TRUE(s.ok()); +// ASSERT_TRUE(result == build_index_resources); +//#endif } TEST_F(ConfigTest, SERVER_CONFIG_INVALID_TEST) { diff --git a/core/unittest/server/utils.cpp b/core/unittest/server/utils.cpp index 534d53cb0c..81eea19c9d 100644 --- a/core/unittest/server/utils.cpp +++ b/core/unittest/server/utils.cpp @@ -25,47 +25,65 @@ static const char* VALID_CONFIG_STR = "version: 0.4" "\n" "server_config:\n" - " address: 0.0.0.0 # milvus server ip address (IPv4)\n" - " port: 19530 # port range: 1025 ~ 65534\n" - " deploy_mode: single \n" + " address: 0.0.0.0\n" + " port: 19530\n" + " deploy_mode: single\n" " time_zone: UTC+8\n" + " web_enable: true\n" + " web_port: 19121\n" "\n" "db_config:\n" - " backend_url: sqlite://:@:/ \n" - " preload_collection: \n" + " backend_url: sqlite://:@:/\n" + " preload_collection:\n" + " auto_flush_interval: 1\n" "\n" "storage_config:\n" - " primary_path: /tmp/milvus # path used to store data and meta\n" - " secondary_path: # path used to store data only, split by semicolon\n" + " primary_path: /tmp/milvus\n" + " secondary_path:\n" + " file_cleanup_timeout: 10\n" "\n" "metric_config:\n" - " enable_monitor: false # enable monitoring or not\n" + " enable_monitor: false\n" " address: 127.0.0.1\n" - " port: 8080 # port prometheus uses to fetch metrics\n" + " port: 9091\n" "\n" "cache_config:\n" - " cpu_cache_capacity: 4 # GB, CPU memory used for cache\n" - " cpu_cache_threshold: 0.85 \n" - " insert_buffer_size: 4 # GB, maximum insert buffer size allowed\n" - " cache_insert_data: false # whether to load inserted data into cache\n" + " cpu_cache_capacity: 4\n" + " insert_buffer_size: 1\n" + " cache_insert_data: false\n" "\n" "engine_config:\n" - " use_blas_threshold: 20 \n" + " use_blas_threshold: 1100\n" + " gpu_search_threshold: 1000\n" "\n" - #ifdef MILVUS_GPU_VERSION "gpu_resource_config:\n" - " enable: true # whether to enable GPU resources\n" - " cache_capacity: 4 # GB, size of GPU memory per card used for cache, must be a positive integer\n" - " search_resources: # define the GPU devices used for search computation, must be in format gpux\n" + " enable: true\n" + " cache_capacity: 1\n" + " search_resources:\n" " - gpu0\n" - " build_index_resources: # define the GPU devices used for index building, must be in format gpux\n" + " build_index_resources:\n" " - gpu0\n" - #endif "\n" - "sequence_config:\n" - " - seq1\n" - " - seq2\n" - "\n"; + "tracing_config:\n" + " json_config_path:\n" + "\n" + "wal_config:\n" + " enable: true\n" + " recovery_error_ignore: true\n" + " buffer_size: 256\n" + " wal_path: /tmp/milvus/wal\n" + "\n" + "logs:\n" + " trace.enable: true\n" + " debug.enable: true\n" + " info.enable: true\n" + " warning.enable: true\n" + " error.enable: true\n" + " fatal.enable: true\n" + " path: /tmp/milvus/logs\n" + " max_log_file_size: 256\n" + " delete_exceeds: 10\n" + ""; static const char* INVALID_CONFIG_STR = "*INVALID*"; diff --git a/sdk/build-support/lint_exclusions.txt b/sdk/build-support/lint_exclusions.txt index 465c116b99..139e7461b7 100644 --- a/sdk/build-support/lint_exclusions.txt +++ b/sdk/build-support/lint_exclusions.txt @@ -1,4 +1,5 @@ *cmake-build-debug* *cmake-build-release* *cmake_build* -*grpc-gen* \ No newline at end of file +*grpc-gen* +*build* \ No newline at end of file diff --git a/sdk/examples/binary_vector/src/ClientTest.cpp b/sdk/examples/binary_vector/src/ClientTest.cpp index 6fce2ab061..6079237529 100644 --- a/sdk/examples/binary_vector/src/ClientTest.cpp +++ b/sdk/examples/binary_vector/src/ClientTest.cpp @@ -95,8 +95,9 @@ TestProcess(std::shared_ptr connection, } { // flush buffer - stat = connection->FlushCollection(collection_param.collection_name); - std::cout << "FlushCollection function call status: " << stat.message() << std::endl; + std::vector collections = {collection_param.collection_name}; + stat = connection->Flush(collections); + std::cout << "Flush function call status: " << stat.message() << std::endl; } { // search vectors @@ -153,54 +154,54 @@ ClientTest::Test(const std::string& address, const std::string& port) { { milvus::CollectionParam collection_param = { - "collection_1", - 512, // dimension - 256, // index file size - milvus::MetricType::TANIMOTO - }; + "collection_1", + 512, // dimension + 256, // index file size + milvus::MetricType::TANIMOTO + }; JSON json_params = {{"nlist", 1024}}; milvus::IndexParam index_param = { - collection_param.collection_name, - milvus::IndexType::IVFFLAT, - json_params.dump() - }; + collection_param.collection_name, + milvus::IndexType::IVFFLAT, + json_params.dump() + }; TestProcess(connection, collection_param, index_param); } { milvus::CollectionParam collection_param = { - "collection_2", - 512, // dimension - 512, // index file size - milvus::MetricType::SUBSTRUCTURE - }; + "collection_2", + 512, // dimension + 512, // index file size + milvus::MetricType::SUBSTRUCTURE + }; JSON json_params = {}; milvus::IndexParam index_param = { - collection_param.collection_name, - milvus::IndexType::FLAT, - json_params.dump() - }; + collection_param.collection_name, + milvus::IndexType::FLAT, + json_params.dump() + }; TestProcess(connection, collection_param, index_param); } { milvus::CollectionParam collection_param = { - "collection_3", - 128, // dimension - 1024, // index file size - milvus::MetricType::SUPERSTRUCTURE - }; + "collection_3", + 128, // dimension + 1024, // index file size + milvus::MetricType::SUPERSTRUCTURE + }; JSON json_params = {}; milvus::IndexParam index_param = { - collection_param.collection_name, - milvus::IndexType::FLAT, - json_params.dump() - }; + collection_param.collection_name, + milvus::IndexType::FLAT, + json_params.dump() + }; TestProcess(connection, collection_param, index_param); } diff --git a/sdk/examples/hybrid/src/ClientTest.cpp b/sdk/examples/hybrid/src/ClientTest.cpp index bef385c17d..1d29e89fe5 100644 --- a/sdk/examples/hybrid/src/ClientTest.cpp +++ b/sdk/examples/hybrid/src/ClientTest.cpp @@ -80,8 +80,9 @@ ClientTest::CreateHybridCollection(const std::string& collection_name) { void ClientTest::Flush(const std::string& collection_name) { milvus_sdk::TimeRecorder rc("Flush"); - milvus::Status stat = conn_->FlushCollection(collection_name); - std::cout << "FlushCollection function call status: " << stat.message() << std::endl; + std::vector collections = {collection_name}; + milvus::Status stat = conn_->Flush(collections); + std::cout << "Flush function call status: " << stat.message() << std::endl; } void diff --git a/sdk/examples/partition/src/ClientTest.cpp b/sdk/examples/partition/src/ClientTest.cpp index 95985a6d5b..9348374d96 100644 --- a/sdk/examples/partition/src/ClientTest.cpp +++ b/sdk/examples/partition/src/ClientTest.cpp @@ -59,20 +59,27 @@ BuildIndexParam() { } void -CountCollection(std::shared_ptr& conn) { +CountEntities(std::shared_ptr& conn) { int64_t entity_count = 0; - auto stat = conn->CountCollection(COLLECTION_NAME, entity_count); + auto stat = conn->CountEntities(COLLECTION_NAME, entity_count); std::cout << COLLECTION_NAME << "(" << entity_count << " entities)" << std::endl; } void -ShowCollectionInfo(std::shared_ptr& conn) { - CountCollection(conn); +GetCollectionStats(std::shared_ptr& conn) { + CountEntities(conn); - std::string collection_info; - auto stat = conn->ShowCollectionInfo(COLLECTION_NAME, collection_info); - std::cout << collection_info << std::endl; - std::cout << "ShowCollectionInfo function call status: " << stat.message() << std::endl; + std::string collection_stats; + auto stat = conn->GetCollectionStats(COLLECTION_NAME, collection_stats); + std::cout << collection_stats << std::endl; + std::cout << "GetCollectionStats function call status: " << stat.message() << std::endl; +} + +void +Flush(std::shared_ptr& conn, const std::string& collection_name) { + std::vector collections = {collection_name}; + milvus::Status stat = conn->Flush(collections); + std::cout << "Flush function call status: " << stat.message() << std::endl; } } // namespace @@ -105,7 +112,7 @@ ClientTest::Test(const std::string& address, const std::string& port) { // show partitions milvus::PartitionTagList partition_array; - stat = conn->ShowPartitions(COLLECTION_NAME, partition_array); + stat = conn->ListPartitions(COLLECTION_NAME, partition_array); std::cout << partition_array.size() << " partitions created:" << std::endl; for (auto& partition_tag : partition_array) { @@ -134,12 +141,8 @@ ClientTest::Test(const std::string& address, const std::string& port) { } } - { // flush buffer - stat = conn->FlushCollection(COLLECTION_NAME); - std::cout << "FlushCollection function call status: " << stat.message() << std::endl; - } - - ShowCollectionInfo(conn); + Flush(conn, COLLECTION_NAME); + GetCollectionStats(conn); std::vector> search_entity_array; { // build search vectors @@ -151,8 +154,12 @@ ClientTest::Test(const std::string& address, const std::string& port) { } { // search vectors + std::string partition_tag = std::to_string(TARGET_PARTITION); + bool exist = conn->HasPartition(COLLECTION_NAME, partition_tag); + std::cout << "Partition " << partition_tag << (exist ? " exists" : " doesn't exist") << std::endl; + std::cout << "Search in correct partition" << std::endl; - std::vector partition_tags = {std::to_string(TARGET_PARTITION)}; + std::vector partition_tags = {partition_tag}; milvus::TopKQueryResult topk_query_result; milvus_sdk::Utils::DoSearch(conn, COLLECTION_NAME, partition_tags, TOP_K, NPROBE, search_entity_array, topk_query_result); @@ -176,12 +183,12 @@ ClientTest::Test(const std::string& address, const std::string& port) { std::cout << "CreateIndex function call status: " << stat.message() << std::endl; milvus::IndexParam index2; - stat = conn->DescribeIndex(COLLECTION_NAME, index2); + stat = conn->GetIndexInfo(COLLECTION_NAME, index2); std::cout << "DescribeIndex function call status: " << stat.message() << std::endl; milvus_sdk::Utils::PrintIndexParam(index2); } - ShowCollectionInfo(conn); + GetCollectionStats(conn); { // drop partition milvus::PartitionParam param1 = {COLLECTION_NAME, std::to_string(TARGET_PARTITION)}; @@ -190,7 +197,7 @@ ClientTest::Test(const std::string& address, const std::string& port) { std::cout << "DropPartition function call status: " << stat.message() << std::endl; } - CountCollection(conn); + CountEntities(conn); { // search vectors, will get search error since we delete a partition std::cout << "Search in whole collection after delete one partition" << std::endl; @@ -203,12 +210,10 @@ ClientTest::Test(const std::string& address, const std::string& port) { { // drop index stat = conn->DropIndex(COLLECTION_NAME); std::cout << "DropIndex function call status: " << stat.message() << std::endl; - - int64_t entity_count = 0; - stat = conn->CountCollection(COLLECTION_NAME, entity_count); - std::cout << COLLECTION_NAME << "(" << entity_count << " entities)" << std::endl; } + CountEntities(conn); + { // drop collection stat = conn->DropCollection(COLLECTION_NAME); std::cout << "DropCollection function call status: " << stat.message() << std::endl; diff --git a/sdk/examples/qps/src/ClientTest.cpp b/sdk/examples/qps/src/ClientTest.cpp index 65420b9f2f..1472786fa4 100644 --- a/sdk/examples/qps/src/ClientTest.cpp +++ b/sdk/examples/qps/src/ClientTest.cpp @@ -51,6 +51,7 @@ class ConnectionWrapper { std::shared_ptr& Connection() { return connection_; } + private: std::shared_ptr connection_; }; @@ -179,7 +180,8 @@ ClientTest::InsertEntities(std::shared_ptr& conn) { // std::cout << "InsertEntities function call status: " << stat.message() << std::endl; // std::cout << "Returned id array count: " << record_ids.size() << std::endl; - stat = conn->FlushCollection(parameters_.collection_name_); + std::vector collections = {parameters_.collection_name_}; + stat = conn->Flush(collections); } return true; @@ -212,11 +214,11 @@ ClientTest::PreloadCollection() { return false; } - std::string title = "Preload table " + parameters_.collection_name_; + std::string title = "Load table " + parameters_.collection_name_; milvus_sdk::TimeRecorder rc(title); - milvus::Status stat = conn->PreloadCollection(parameters_.collection_name_); + milvus::Status stat = conn->LoadCollection(parameters_.collection_name_); if (!stat.ok()) { - std::string msg = "PreloadCollection function call status: " + stat.message(); + std::string msg = "LoadCollection function call status: " + stat.message(); std::cout << msg << std::endl; return false; } @@ -233,13 +235,13 @@ ClientTest::GetCollectionInfo() { } milvus::CollectionParam collection_param; - milvus::Status stat = conn->DescribeCollection(parameters_.collection_name_, collection_param); + milvus::Status stat = conn->GetCollectionInfo(parameters_.collection_name_, collection_param); milvus::IndexParam index_param; - stat = conn->DescribeIndex(parameters_.collection_name_, index_param); + stat = conn->GetIndexInfo(parameters_.collection_name_, index_param); int64_t row_count = 0; - stat = conn->CountCollection(parameters_.collection_name_, row_count); + stat = conn->CountEntities(parameters_.collection_name_, row_count); parameters_.index_type_ = (int64_t)index_param.index_type; if (!IsSupportedIndex(parameters_.index_type_)) { diff --git a/sdk/examples/simple/src/ClientTest.cpp b/sdk/examples/simple/src/ClientTest.cpp index 9d19bb3c9f..63fffe8caf 100644 --- a/sdk/examples/simple/src/ClientTest.cpp +++ b/sdk/examples/simple/src/ClientTest.cpp @@ -36,7 +36,8 @@ constexpr int64_t ADD_ENTITY_LOOP = 5; constexpr milvus::IndexType INDEX_TYPE = milvus::IndexType::IVFSQ8; constexpr int32_t NLIST = 16384; -void PrintEntity(const std::string& tag, const milvus::Entity& entity) { +void +PrintEntity(const std::string& tag, const milvus::Entity& entity) { std::cout << tag << "\t["; for (size_t i = 0; i < entity.float_data.size(); i++) { if (i != 0) { @@ -75,12 +76,12 @@ ClientTest::ShowSdkVersion() { void ClientTest::ShowCollections(std::vector& collection_array) { - milvus::Status stat = conn_->ShowCollections(collection_array); + milvus::Status stat = conn_->ListCollections(collection_array); std::cout << "ShowCollections function call status: " << stat.message() << std::endl; std::cout << "All collections: " << std::endl; for (auto& collection : collection_array) { int64_t entity_count = 0; - stat = conn_->CountCollection(collection, entity_count); + stat = conn_->CountEntities(collection, entity_count); std::cout << "\t" << collection << "(" << entity_count << " entities)" << std::endl; } } @@ -99,9 +100,9 @@ ClientTest::CreateCollection(const std::string& collection_name, int64_t dim, mi } void -ClientTest::DescribeCollection(const std::string& collection_name) { +ClientTest::GetCollectionInfo(const std::string& collection_name) { milvus::CollectionParam collection_param; - milvus::Status stat = conn_->DescribeCollection(collection_name, collection_param); + milvus::Status stat = conn_->GetCollectionInfo(collection_name, collection_param); std::cout << "DescribeCollection function call status: " << stat.message() << std::endl; milvus_sdk::Utils::PrintCollectionParam(collection_param); } @@ -146,25 +147,26 @@ ClientTest::BuildSearchEntities(int64_t nq, int64_t dim) { void ClientTest::Flush(const std::string& collection_name) { milvus_sdk::TimeRecorder rc("Flush"); - milvus::Status stat = conn_->FlushCollection(collection_name); - std::cout << "FlushCollection function call status: " << stat.message() << std::endl; + std::vector collections = {collection_name}; + milvus::Status stat = conn_->Flush(collections); + std::cout << "Flush function call status: " << stat.message() << std::endl; } void -ClientTest::ShowCollectionInfo(const std::string& collection_name) { +ClientTest::GetCollectionStats(const std::string& collection_name) { std::string collection_info; - milvus::Status stat = conn_->ShowCollectionInfo(collection_name, collection_info); + milvus::Status stat = conn_->GetCollectionStats(collection_name, collection_info); std::cout << "Collection info: " << collection_info << std::endl; - std::cout << "ShowCollectionInfo function call status: " << stat.message() << std::endl; + std::cout << "GetCollectionStats function call status: " << stat.message() << std::endl; } void -ClientTest::GetEntitiesByID(const std::string& collection_name, const std::vector& id_array) { +ClientTest::GetEntityByID(const std::string& collection_name, const std::vector& id_array) { std::vector entities; { - milvus_sdk::TimeRecorder rc("GetEntitiesByID"); - milvus::Status stat = conn_->GetEntitiesByID(collection_name, id_array, entities); - std::cout << "GetEntitiesByID function call status: " << stat.message() << std::endl; + milvus_sdk::TimeRecorder rc("GetEntityByID"); + milvus::Status stat = conn_->GetEntityByID(collection_name, id_array, entities); + std::cout << "GetEntityByID function call status: " << stat.message() << std::endl; } for (size_t i = 0; i < entities.size(); i++) { @@ -228,22 +230,22 @@ ClientTest::CreateIndex(const std::string& collection_name, milvus::IndexType ty std::cout << "CreateIndex function call status: " << stat.message() << std::endl; milvus::IndexParam index2; - stat = conn_->DescribeIndex(collection_name, index2); - std::cout << "DescribeIndex function call status: " << stat.message() << std::endl; + stat = conn_->GetIndexInfo(collection_name, index2); + std::cout << "GetIndexInfo function call status: " << stat.message() << std::endl; milvus_sdk::Utils::PrintIndexParam(index2); } void -ClientTest::PreloadCollection(const std::string& collection_name) { +ClientTest::LoadCollection(const std::string& collection_name) { milvus_sdk::TimeRecorder rc("Preload"); - milvus::Status stat = conn_->PreloadCollection(collection_name); + milvus::Status stat = conn_->LoadCollection(collection_name); std::cout << "PreloadCollection function call status: " << stat.message() << std::endl; } void ClientTest::CompactCollection(const std::string& collection_name) { milvus_sdk::TimeRecorder rc("Compact"); - milvus::Status stat = conn_->CompactCollection(collection_name); + milvus::Status stat = conn_->Compact(collection_name); std::cout << "CompactCollection function call status: " << stat.message() << std::endl; } @@ -255,14 +257,10 @@ ClientTest::DeleteByIds(const std::string& collection_name, const std::vectorDeleteByID(collection_name, id_array); + milvus::Status stat = conn_->DeleteEntityByID(collection_name, id_array); std::cout << "DeleteByID function call status: " << stat.message() << std::endl; - { - milvus_sdk::TimeRecorder rc("Flush"); - stat = conn_->FlushCollection(collection_name); - std::cout << "FlushCollection function call status: " << stat.message() << std::endl; - } + Flush(collection_name); } void @@ -271,7 +269,7 @@ ClientTest::DropIndex(const std::string& collection_name) { std::cout << "DropIndex function call status: " << stat.message() << std::endl; int64_t row_count = 0; - stat = conn_->CountCollection(collection_name, row_count); + stat = conn_->CountEntities(collection_name, row_count); std::cout << collection_name << "(" << row_count << " rows)" << std::endl; } @@ -294,25 +292,25 @@ ClientTest::Test() { ShowCollections(table_array); CreateCollection(collection_name, dim, metric_type); - DescribeCollection(collection_name); + GetCollectionInfo(collection_name); InsertEntities(collection_name, dim); Flush(collection_name); - ShowCollectionInfo(collection_name); + GetCollectionStats(collection_name); BuildSearchEntities(NQ, dim); - GetEntitiesByID(collection_name, search_id_array_); + GetEntityByID(collection_name, search_id_array_); // SearchEntities(collection_name, TOP_K, NPROBE); SearchEntitiesByID(collection_name, TOP_K, NPROBE); CreateIndex(collection_name, INDEX_TYPE, NLIST); - ShowCollectionInfo(collection_name); + GetCollectionStats(collection_name); std::vector delete_ids = {search_id_array_[0], search_id_array_[1]}; DeleteByIds(collection_name, delete_ids); CompactCollection(collection_name); - PreloadCollection(collection_name); + LoadCollection(collection_name); SearchEntities(collection_name, TOP_K, NPROBE); // this line get two search error since we delete two entities DropIndex(collection_name); diff --git a/sdk/examples/simple/src/ClientTest.h b/sdk/examples/simple/src/ClientTest.h index 95d44ad8f5..1d41e12325 100644 --- a/sdk/examples/simple/src/ClientTest.h +++ b/sdk/examples/simple/src/ClientTest.h @@ -40,7 +40,7 @@ class ClientTest { CreateCollection(const std::string&, int64_t, milvus::MetricType); void - DescribeCollection(const std::string&); + GetCollectionInfo(const std::string&); void InsertEntities(const std::string&, int64_t); @@ -52,10 +52,10 @@ class ClientTest { Flush(const std::string&); void - ShowCollectionInfo(const std::string&); + GetCollectionStats(const std::string&); void - GetEntitiesByID(const std::string&, const std::vector&); + GetEntityByID(const std::string&, const std::vector&); void SearchEntities(const std::string&, int64_t, int64_t); @@ -67,7 +67,7 @@ class ClientTest { CreateIndex(const std::string&, milvus::IndexType, int64_t); void - PreloadCollection(const std::string&); + LoadCollection(const std::string&); void CompactCollection(const std::string&); diff --git a/sdk/grpc/ClientProxy.cpp b/sdk/grpc/ClientProxy.cpp index b69abbf431..811ea3a4ea 100644 --- a/sdk/grpc/ClientProxy.cpp +++ b/sdk/grpc/ClientProxy.cpp @@ -29,7 +29,7 @@ UriCheck(const std::string& uri) { return (index != std::string::npos); } -template +template void ConstructSearchParam(const std::string& collection_name, const std::vector& partition_tag_array, int64_t topk, const std::string& extra_params, T& search_param) { @@ -205,11 +205,14 @@ ClientProxy::CreateCollection(const CollectionParam& param) { bool ClientProxy::HasCollection(const std::string& collection_name) { - Status status = Status::OK(); - ::milvus::grpc::CollectionName grpc_collection_name; - grpc_collection_name.set_collection_name(collection_name); - bool result = client_ptr_->HasCollection(grpc_collection_name, status); - return result; + try { + Status status = Status::OK(); + ::milvus::grpc::CollectionName grpc_collection_name; + grpc_collection_name.set_collection_name(collection_name); + return client_ptr_->HasCollection(grpc_collection_name, status); + } catch (std::exception& ex) { + return false; + } } Status @@ -273,28 +276,8 @@ ClientProxy::Insert(const std::string& collection_name, const std::string& parti } Status -ClientProxy::GetEntityByID(const std::string& collection_name, int64_t entity_id, Entity& entity_data) { - std::vector id_array = {entity_id}; - std::vector entities_data; - - auto status = GetEntitiesByID(collection_name, id_array, entities_data); - if (!status.ok()) { - return status; - } - - if (entities_data.empty()) { - return Status(StatusCode::ServerFailed, "Failed to get entity by id"); - } - - entity_data.binary_data.swap(entities_data[0].binary_data); - entity_data.float_data.swap(entities_data[0].float_data); - - return status; -} - -Status -ClientProxy::GetEntitiesByID(const std::string& collection_name, const std::vector& id_array, - std::vector& entities_data) { +ClientProxy::GetEntityByID(const std::string& collection_name, const std::vector& id_array, + std::vector& entities_data) { try { entities_data.clear(); @@ -305,7 +288,7 @@ ClientProxy::GetEntitiesByID(const std::string& collection_name, const std::vect } ::milvus::grpc::VectorsData grpc_data; - Status status = client_ptr_->GetVectorsByID(vectors_identity, grpc_data); + Status status = client_ptr_->GetEntityByID(vectors_identity, grpc_data); if (!status.ok()) { return status; } @@ -336,7 +319,7 @@ ClientProxy::GetEntitiesByID(const std::string& collection_name, const std::vect } Status -ClientProxy::GetIDsInSegment(const std::string& collection_name, const std::string& segment_name, +ClientProxy::ListIDInSegment(const std::string& collection_name, const std::string& segment_name, std::vector& id_array) { try { ::milvus::grpc::GetVectorIDsParam param; @@ -344,7 +327,7 @@ ClientProxy::GetIDsInSegment(const std::string& collection_name, const std::stri param.set_segment_name(segment_name); ::milvus::grpc::VectorIds vector_ids; - Status status = client_ptr_->GetIDsInSegment(param, vector_ids); + Status status = client_ptr_->ListIDInSegment(param, vector_ids); if (!status.ok()) { return status; } @@ -417,11 +400,11 @@ ClientProxy::SearchByID(const std::string& collection_name, const PartitionTagLi } Status -ClientProxy::DescribeCollection(const std::string& collection_name, CollectionParam& collection_param) { +ClientProxy::GetCollectionInfo(const std::string& collection_name, CollectionParam& collection_param) { try { ::milvus::grpc::CollectionSchema grpc_schema; - Status status = client_ptr_->DescribeCollection(collection_name, grpc_schema); + Status status = client_ptr_->GetCollectionInfo(collection_name, grpc_schema); collection_param.collection_name = grpc_schema.collection_name(); collection_param.dimension = grpc_schema.dimension(); @@ -435,12 +418,12 @@ ClientProxy::DescribeCollection(const std::string& collection_name, CollectionPa } Status -ClientProxy::CountCollection(const std::string& collection_name, int64_t& row_count) { +ClientProxy::CountEntities(const std::string& collection_name, int64_t& row_count) { try { Status status; ::milvus::grpc::CollectionName grpc_collection_name; grpc_collection_name.set_collection_name(collection_name); - row_count = client_ptr_->CountCollection(grpc_collection_name, status); + row_count = client_ptr_->CountEntities(grpc_collection_name, status); return status; } catch (std::exception& ex) { return Status(StatusCode::UnknownError, "Failed to count collection: " + std::string(ex.what())); @@ -448,11 +431,11 @@ ClientProxy::CountCollection(const std::string& collection_name, int64_t& row_co } Status -ClientProxy::ShowCollections(std::vector& collection_array) { +ClientProxy::ListCollections(std::vector& collection_array) { try { Status status; milvus::grpc::CollectionNameList collection_name_list; - status = client_ptr_->ShowCollections(collection_name_list); + status = client_ptr_->ListCollections(collection_name_list); collection_array.resize(collection_name_list.collection_names_size()); for (uint64_t i = 0; i < collection_name_list.collection_names_size(); ++i) { @@ -465,15 +448,15 @@ ClientProxy::ShowCollections(std::vector& collection_array) { } Status -ClientProxy::ShowCollectionInfo(const std::string& collection_name, std::string& collection_info) { +ClientProxy::GetCollectionStats(const std::string& collection_name, std::string& collection_stats) { try { Status status; ::milvus::grpc::CollectionName grpc_collection_name; grpc_collection_name.set_collection_name(collection_name); - milvus::grpc::CollectionInfo grpc_collection_info; - status = client_ptr_->ShowCollectionInfo(grpc_collection_name, grpc_collection_info); + milvus::grpc::CollectionInfo grpc_collection_stats; + status = client_ptr_->GetCollectionStats(grpc_collection_name, grpc_collection_stats); - collection_info = grpc_collection_info.json_info(); + collection_stats = grpc_collection_stats.json_info(); return status; } catch (std::exception& ex) { @@ -482,7 +465,7 @@ ClientProxy::ShowCollectionInfo(const std::string& collection_name, std::string& } Status -ClientProxy::DeleteByID(const std::string& collection_name, const std::vector& id_array) { +ClientProxy::DeleteEntityByID(const std::string& collection_name, const std::vector& id_array) { try { ::milvus::grpc::DeleteByIDParam delete_by_id_param; delete_by_id_param.set_collection_name(collection_name); @@ -490,18 +473,18 @@ ClientProxy::DeleteByID(const std::string& collection_name, const std::vectorDeleteByID(delete_by_id_param); + return client_ptr_->DeleteEntityByID(delete_by_id_param); } catch (std::exception& ex) { return Status(StatusCode::UnknownError, "Failed to delete entity id: " + std::string(ex.what())); } } Status -ClientProxy::PreloadCollection(const std::string& collection_name) const { +ClientProxy::LoadCollection(const std::string& collection_name) const { try { ::milvus::grpc::CollectionName grpc_collection_name; grpc_collection_name.set_collection_name(collection_name); - Status status = client_ptr_->PreloadCollection(grpc_collection_name); + Status status = client_ptr_->LoadCollection(grpc_collection_name); return status; } catch (std::exception& ex) { return Status(StatusCode::UnknownError, "Failed to preload collection: " + std::string(ex.what())); @@ -509,13 +492,13 @@ ClientProxy::PreloadCollection(const std::string& collection_name) const { } Status -ClientProxy::DescribeIndex(const std::string& collection_name, IndexParam& index_param) const { +ClientProxy::GetIndexInfo(const std::string& collection_name, IndexParam& index_param) const { try { ::milvus::grpc::CollectionName grpc_collection_name; grpc_collection_name.set_collection_name(collection_name); ::milvus::grpc::IndexParam grpc_index_param; - Status status = client_ptr_->DescribeIndex(grpc_collection_name, grpc_index_param); + Status status = client_ptr_->GetIndexInfo(grpc_collection_name, grpc_index_param); index_param.index_type = static_cast(grpc_index_param.index_type()); for (int i = 0; i < grpc_index_param.extra_params_size(); i++) { @@ -556,13 +539,26 @@ ClientProxy::CreatePartition(const PartitionParam& partition_param) { } } +bool +ClientProxy::HasPartition(const std::string& collection_name, const std::string& partition_tag) const { + try { + Status status = Status::OK(); + ::milvus::grpc::PartitionParam grpc_partition_param; + grpc_partition_param.set_collection_name(collection_name); + grpc_partition_param.set_tag(partition_tag); + return client_ptr_->HasPartition(grpc_partition_param, status); + } catch (std::exception& ex) { + return false; + } +} + Status -ClientProxy::ShowPartitions(const std::string& collection_name, PartitionTagList& partition_tag_array) const { +ClientProxy::ListPartitions(const std::string& collection_name, PartitionTagList& partition_tag_array) const { try { ::milvus::grpc::CollectionName grpc_collection_name; grpc_collection_name.set_collection_name(collection_name); ::milvus::grpc::PartitionList grpc_partition_list; - Status status = client_ptr_->ShowPartitions(grpc_collection_name, grpc_partition_list); + Status status = client_ptr_->ListPartitions(grpc_collection_name, grpc_partition_list); partition_tag_array.resize(grpc_partition_list.partition_tag_array_size()); for (uint64_t i = 0; i < grpc_partition_list.partition_tag_array_size(); ++i) { partition_tag_array[i] = grpc_partition_list.partition_tag_array(i); @@ -587,27 +583,23 @@ ClientProxy::DropPartition(const PartitionParam& partition_param) { } Status -ClientProxy::FlushCollection(const std::string& collection_name) { +ClientProxy::Flush(const std::vector& collection_name_array) { try { - std::string dummy; - return client_ptr_->Flush(collection_name); + if (collection_name_array.empty()) { + return client_ptr_->Flush(""); + } else { + for (auto& collection_name : collection_name_array) { + client_ptr_->Flush(collection_name); + } + } + return Status::OK(); } catch (std::exception& ex) { return Status(StatusCode::UnknownError, "Failed to flush collection"); } } Status -ClientProxy::Flush() { - try { - std::string dummy; - return client_ptr_->Flush(""); - } catch (std::exception& ex) { - return Status(StatusCode::UnknownError, "Failed to flush collections"); - } -} - -Status -ClientProxy::CompactCollection(const std::string& collection_name) { +ClientProxy::Compact(const std::string& collection_name) { try { ::milvus::grpc::CollectionName grpc_collection_name; grpc_collection_name.set_collection_name(collection_name); diff --git a/sdk/grpc/ClientProxy.h b/sdk/grpc/ClientProxy.h index 70b838b16c..a6840e083f 100644 --- a/sdk/grpc/ClientProxy.h +++ b/sdk/grpc/ClientProxy.h @@ -69,15 +69,12 @@ class ClientProxy : public Connection { std::vector& id_array) override; Status - GetEntityByID(const std::string& collection_name, int64_t entity_id, Entity& entity_data) override; + GetEntityByID(const std::string& collection_name, + const std::vector& id_array, + std::vector& entities_data) override; Status - GetEntitiesByID(const std::string& collection_name, - const std::vector& id_array, - std::vector& entities_data) override; - - Status - GetIDsInSegment(const std::string& collection_name, const std::string& segment_name, + ListIDInSegment(const std::string& collection_name, const std::string& segment_name, std::vector& id_array) override; Status @@ -91,25 +88,25 @@ class ClientProxy : public Connection { const std::string& extra_params, TopKQueryResult& topk_query_result) override; Status - DescribeCollection(const std::string& collection_name, CollectionParam& collection_schema) override; + GetCollectionInfo(const std::string& collection_name, CollectionParam& collection_param) override; Status - CountCollection(const std::string& collection_name, int64_t& entity_count) override; + CountEntities(const std::string& collection_name, int64_t& entity_count) override; Status - ShowCollections(std::vector& collection_array) override; + ListCollections(std::vector& collection_array) override; Status - ShowCollectionInfo(const std::string& collection_name, std::string& collection_info) override; + GetCollectionStats(const std::string& collection_name, std::string& collection_stats) override; Status - DeleteByID(const std::string& collection_name, const std::vector& id_array) override; + DeleteEntityByID(const std::string& collection_name, const std::vector& id_array) override; Status - PreloadCollection(const std::string& collection_name) const override; + LoadCollection(const std::string& collection_name) const override; Status - DescribeIndex(const std::string& collection_name, IndexParam& index_param) const override; + GetIndexInfo(const std::string& collection_name, IndexParam& index_param) const override; Status DropIndex(const std::string& collection_name) const override; @@ -117,20 +114,20 @@ class ClientProxy : public Connection { Status CreatePartition(const PartitionParam& partition_param) override; + bool + HasPartition(const std::string& collection_name, const std::string& partition_tag) const override; + Status - ShowPartitions(const std::string& collection_name, PartitionTagList& partition_tag_array) const override; + ListPartitions(const std::string& collection_name, PartitionTagList& partition_tag_array) const override; Status DropPartition(const PartitionParam& partition_param) override; Status - FlushCollection(const std::string& collection_name) override; + Flush(const std::vector& collection_name_array) override; Status - Flush() override; - - Status - CompactCollection(const std::string& collection_name) override; + Compact(const std::string& collection_name) override; /*******************************New Interface**********************************/ diff --git a/sdk/grpc/GrpcClient.cpp b/sdk/grpc/GrpcClient.cpp index a4b087da9e..84c81017b0 100644 --- a/sdk/grpc/GrpcClient.cpp +++ b/sdk/grpc/GrpcClient.cpp @@ -125,7 +125,7 @@ GrpcClient::Insert(const ::milvus::grpc::InsertParam& insert_param, ::milvus::gr } Status -GrpcClient::GetVectorsByID(const grpc::VectorsIdentity& vectors_identity, ::milvus::grpc::VectorsData& vectors_data) { +GrpcClient::GetEntityByID(const grpc::VectorsIdentity& vectors_identity, ::milvus::grpc::VectorsData& vectors_data) { ClientContext context; ::grpc::Status grpc_status = stub_->GetVectorsByID(&context, vectors_identity, &vectors_data); @@ -142,7 +142,7 @@ GrpcClient::GetVectorsByID(const grpc::VectorsIdentity& vectors_identity, ::milv } Status -GrpcClient::GetIDsInSegment(const grpc::GetVectorIDsParam& param, grpc::VectorIds& vector_ids) { +GrpcClient::ListIDInSegment(const grpc::GetVectorIDsParam& param, grpc::VectorIds& vector_ids) { ClientContext context; ::grpc::Status grpc_status = stub_->GetVectorIDs(&context, param, &vector_ids); @@ -198,7 +198,7 @@ GrpcClient::SearchByID(const grpc::SearchByIDParam& search_param, ::milvus::grpc } Status -GrpcClient::DescribeCollection(const std::string& collection_name, ::milvus::grpc::CollectionSchema& grpc_schema) { +GrpcClient::GetCollectionInfo(const std::string& collection_name, ::milvus::grpc::CollectionSchema& grpc_schema) { ClientContext context; ::milvus::grpc::CollectionName grpc_collectionname; grpc_collectionname.set_collection_name(collection_name); @@ -219,7 +219,7 @@ GrpcClient::DescribeCollection(const std::string& collection_name, ::milvus::grp } int64_t -GrpcClient::CountCollection(grpc::CollectionName& collection_name, Status& status) { +GrpcClient::CountEntities(grpc::CollectionName& collection_name, Status& status) { ClientContext context; ::milvus::grpc::CollectionRowCount response; ::grpc::Status grpc_status = stub_->CountCollection(&context, collection_name, &response); @@ -241,7 +241,7 @@ GrpcClient::CountCollection(grpc::CollectionName& collection_name, Status& statu } Status -GrpcClient::ShowCollections(milvus::grpc::CollectionNameList& collection_name_list) { +GrpcClient::ListCollections(milvus::grpc::CollectionNameList& collection_name_list) { ClientContext context; ::milvus::grpc::Command command; ::grpc::Status grpc_status = stub_->ShowCollections(&context, command, &collection_name_list); @@ -261,10 +261,10 @@ GrpcClient::ShowCollections(milvus::grpc::CollectionNameList& collection_name_li } Status -GrpcClient::ShowCollectionInfo(grpc::CollectionName& collection_name, grpc::CollectionInfo& collection_info) { +GrpcClient::GetCollectionStats(grpc::CollectionName& collection_name, grpc::CollectionInfo& collection_stats) { ClientContext context; ::milvus::grpc::Command command; - ::grpc::Status grpc_status = stub_->ShowCollectionInfo(&context, collection_name, &collection_info); + ::grpc::Status grpc_status = stub_->ShowCollectionInfo(&context, collection_name, &collection_stats); if (!grpc_status.ok()) { std::cerr << "ShowCollectionInfo gRPC failed!" << std::endl; @@ -272,9 +272,9 @@ GrpcClient::ShowCollectionInfo(grpc::CollectionName& collection_name, grpc::Coll return Status(StatusCode::RPCFailed, grpc_status.error_message()); } - if (collection_info.status().error_code() != grpc::SUCCESS) { - std::cerr << collection_info.status().reason() << std::endl; - return Status(StatusCode::ServerFailed, collection_info.status().reason()); + if (collection_stats.status().error_code() != grpc::SUCCESS) { + std::cerr << collection_stats.status().reason() << std::endl; + return Status(StatusCode::ServerFailed, collection_stats.status().reason()); } return Status::OK(); @@ -303,7 +303,7 @@ GrpcClient::Cmd(const std::string& cmd, std::string& result) { } Status -GrpcClient::PreloadCollection(milvus::grpc::CollectionName& collection_name) { +GrpcClient::LoadCollection(milvus::grpc::CollectionName& collection_name) { ClientContext context; ::milvus::grpc::Status response; ::grpc::Status grpc_status = stub_->PreloadCollection(&context, collection_name, &response); @@ -321,7 +321,7 @@ GrpcClient::PreloadCollection(milvus::grpc::CollectionName& collection_name) { } Status -GrpcClient::DeleteByID(grpc::DeleteByIDParam& delete_by_id_param) { +GrpcClient::DeleteEntityByID(grpc::DeleteByIDParam& delete_by_id_param) { ClientContext context; ::milvus::grpc::Status response; ::grpc::Status grpc_status = stub_->DeleteByID(&context, delete_by_id_param, &response); @@ -339,7 +339,7 @@ GrpcClient::DeleteByID(grpc::DeleteByIDParam& delete_by_id_param) { } Status -GrpcClient::DescribeIndex(grpc::CollectionName& collection_name, grpc::IndexParam& index_param) { +GrpcClient::GetIndexInfo(grpc::CollectionName& collection_name, grpc::IndexParam& index_param) { ClientContext context; ::grpc::Status grpc_status = stub_->DescribeIndex(&context, collection_name, &index_param); @@ -391,8 +391,26 @@ GrpcClient::CreatePartition(const grpc::PartitionParam& partition_param) { return Status::OK(); } +bool +GrpcClient::HasPartition(const grpc::PartitionParam& partition_param, Status& status) const { + ClientContext context; + ::milvus::grpc::BoolReply response; + ::grpc::Status grpc_status = stub_->HasPartition(&context, partition_param, &response); + + if (!grpc_status.ok()) { + std::cerr << "HasPartition gRPC failed!" << std::endl; + status = Status(StatusCode::RPCFailed, grpc_status.error_message()); + } + if (response.status().error_code() != grpc::SUCCESS) { + std::cerr << response.status().reason() << std::endl; + status = Status(StatusCode::ServerFailed, response.status().reason()); + } + status = Status::OK(); + return response.bool_reply(); +} + Status -GrpcClient::ShowPartitions(const grpc::CollectionName& collection_name, grpc::PartitionList& partition_array) const { +GrpcClient::ListPartitions(const grpc::CollectionName& collection_name, grpc::PartitionList& partition_array) const { ClientContext context; ::grpc::Status grpc_status = stub_->ShowPartitions(&context, collection_name, &partition_array); diff --git a/sdk/grpc/GrpcClient.h b/sdk/grpc/GrpcClient.h index aa259674b2..6617b12590 100644 --- a/sdk/grpc/GrpcClient.h +++ b/sdk/grpc/GrpcClient.h @@ -51,10 +51,10 @@ class GrpcClient { Insert(const grpc::InsertParam& insert_param, grpc::VectorIds& vector_ids); Status - GetVectorsByID(const grpc::VectorsIdentity& vectors_identity, ::milvus::grpc::VectorsData& vectors_data); + GetEntityByID(const grpc::VectorsIdentity& vectors_identity, ::milvus::grpc::VectorsData& vectors_data); Status - GetIDsInSegment(const grpc::GetVectorIDsParam& param, grpc::VectorIds& vector_ids); + ListIDInSegment(const grpc::GetVectorIDsParam& param, grpc::VectorIds& vector_ids); Status Search(const grpc::SearchParam& search_param, ::milvus::grpc::TopKQueryResult& topk_query_result); @@ -63,28 +63,28 @@ class GrpcClient { SearchByID(const grpc::SearchByIDParam& search_param, ::milvus::grpc::TopKQueryResult& topk_query_result); Status - DescribeCollection(const std::string& collection_name, grpc::CollectionSchema& grpc_schema); + GetCollectionInfo(const std::string& collection_name, grpc::CollectionSchema& grpc_schema); int64_t - CountCollection(grpc::CollectionName& collection_name, Status& status); + CountEntities(grpc::CollectionName& collection_name, Status& status); Status - ShowCollections(milvus::grpc::CollectionNameList& collection_name_list); + ListCollections(milvus::grpc::CollectionNameList& collection_name_list); Status - ShowCollectionInfo(grpc::CollectionName& collection_name, grpc::CollectionInfo& collection_info); + GetCollectionStats(grpc::CollectionName& collection_name, grpc::CollectionInfo& collection_stats); Status Cmd(const std::string& cmd, std::string& result); Status - DeleteByID(grpc::DeleteByIDParam& delete_by_id_param); + DeleteEntityByID(grpc::DeleteByIDParam& delete_by_id_param); Status - PreloadCollection(grpc::CollectionName& collection_name); + LoadCollection(grpc::CollectionName& collection_name); Status - DescribeIndex(grpc::CollectionName& collection_name, grpc::IndexParam& index_param); + GetIndexInfo(grpc::CollectionName& collection_name, grpc::IndexParam& index_param); Status DropIndex(grpc::CollectionName& collection_name); @@ -92,8 +92,11 @@ class GrpcClient { Status CreatePartition(const grpc::PartitionParam& partition_param); + bool + HasPartition(const grpc::PartitionParam& partition_param, Status& status) const; + Status - ShowPartitions(const grpc::CollectionName& collection_name, grpc::PartitionList& partition_array) const; + ListPartitions(const grpc::CollectionName& collection_name, grpc::PartitionList& partition_array) const; Status DropPartition(const ::milvus::grpc::PartitionParam& partition_param); diff --git a/sdk/include/MilvusApi.h b/sdk/include/MilvusApi.h index 118693827e..f829647c34 100644 --- a/sdk/include/MilvusApi.h +++ b/sdk/include/MilvusApi.h @@ -326,21 +326,6 @@ class Connection { const std::vector& entity_array, std::vector& id_array) = 0; - /** - * @brief Get entity data by id - * - * This method is used to get entity data by id from a collection. - * Return the first found entity if there are entities with duplicated id - * - * @param collection_name, target collection's name. - * @param entity_id, target entity id. - * @param entity_data, returned entity data. - * - * @return Indicate if the operation is succeed. - */ - virtual Status - GetEntityByID(const std::string& collection_name, int64_t entity_id, Entity& entity_data) = 0; - /** * @brief Get entity data by id * @@ -354,12 +339,12 @@ class Connection { * @return Indicate if the operation is succeed. */ virtual Status - GetEntitiesByID(const std::string& collection_name, - const std::vector& id_array, - std::vector& entities_data) = 0; + GetEntityByID(const std::string& collection_name, + const std::vector& id_array, + std::vector& entities_data) = 0; /** - * @brief Get entity ids from a segment + * @brief List entity ids from a segment * * This method is used to get entity ids from a segment * Return all entity(not deleted) ids @@ -371,7 +356,7 @@ class Connection { * @return Indicate if the operation is succeed. */ virtual Status - GetIDsInSegment(const std::string& collection_name, + ListIDInSegment(const std::string& collection_name, const std::string& segment_name, std::vector& id_array) = 0; @@ -430,9 +415,9 @@ class Connection { const std::string& extra_params, TopKQueryResult& topk_query_result) = 0; /** - * @brief Show collection description + * @brief Get collection information * - * This method is used to show collection information. + * This method is used to get collection information. * * @param collection_name, target collection's name. * @param collection_param, collection_param is given when operation is successful. @@ -440,7 +425,7 @@ class Connection { * @return Indicate if this operation is successful. */ virtual Status - DescribeCollection(const std::string& collection_name, CollectionParam& collection_param) = 0; + GetCollectionInfo(const std::string& collection_name, CollectionParam& collection_param) = 0; /** * @brief Get collection entity count @@ -453,10 +438,10 @@ class Connection { * @return Indicate if this operation is successful. */ virtual Status - CountCollection(const std::string& collection_name, int64_t& entity_count) = 0; + CountEntities(const std::string& collection_name, int64_t& entity_count) = 0; /** - * @brief Show all collections in database + * @brief List all collections in database * * This method is used to list all collections. * @@ -465,20 +450,20 @@ class Connection { * @return Indicate if this operation is successful. */ virtual Status - ShowCollections(std::vector& collection_array) = 0; + ListCollections(std::vector& collection_array) = 0; /** - * @brief Show collection information + * @brief Get collection statistics * - * This method is used to get detail information of a collection. + * This method is used to get statistics of a collection. * * @param collection_name, target collection's name. - * @param collection_info, target collection's information in json format + * @param collection_stats, target collection's statistics in json format * * @return Indicate if this operation is successful. */ virtual Status - ShowCollectionInfo(const std::string& collection_name, std::string& collection_info) = 0; + GetCollectionStats(const std::string& collection_name, std::string& collection_stats) = 0; /** * @brief Delete entity by id @@ -491,24 +476,24 @@ class Connection { * @return Indicate if this operation is successful. */ virtual Status - DeleteByID(const std::string& collection_name, const std::vector& id_array) = 0; + DeleteEntityByID(const std::string& collection_name, const std::vector& id_array) = 0; /** - * @brief Preload collection + * @brief Load collection from disk to memory * - * This method is used to preload collection data into memory + * This method is used to load collection data into memory * * @param collection_name, target collection's name. * * @return Indicate if this operation is successful. */ virtual Status - PreloadCollection(const std::string& collection_name) const = 0; + LoadCollection(const std::string& collection_name) const = 0; /** - * @brief Describe index + * @brief Get index information * - * This method is used to describe index + * This method is used to get index information * * @param collection_name, target collection's name. * @param index_param, returned index information. @@ -516,7 +501,7 @@ class Connection { * @return Indicate if this operation is successful. */ virtual Status - DescribeIndex(const std::string& collection_name, IndexParam& index_param) const = 0; + GetIndexInfo(const std::string& collection_name, IndexParam& index_param) const = 0; /** * @brief Drop index @@ -543,9 +528,22 @@ class Connection { CreatePartition(const PartitionParam& partition_param) = 0; /** - * @brief Show all partitions method + * @brief Has partition method * - * This method is used to show all partitions(return their tags) + * This method is used to test existence of collection's partition + * + * @param collection_name, target collection's name. + * @param partition_tag, target partition's tag. + * + * @return Indicate if partition is created successfully + */ + virtual bool + HasPartition(const std::string& collection_name, const std::string& partition_tag) const = 0; + + /** + * @brief List all partitions method + * + * This method is used to list all partitions(return their tags) * * @param collection_name, target collection's name. * @param partition_tag_array, partition tag array of the collection. @@ -553,7 +551,7 @@ class Connection { * @return Indicate if this operation is successful */ virtual Status - ShowPartitions(const std::string& collection_name, PartitionTagList& partition_tag_array) const = 0; + ListPartitions(const std::string& collection_name, PartitionTagList& partition_tag_array) const = 0; /** * @brief Delete partition method @@ -568,26 +566,16 @@ class Connection { DropPartition(const PartitionParam& partition_param) = 0; /** - * @brief Flush collection buffer into storage + * @brief Flush collections insert buffer into storage * - * This method is used to flush collection buffer into storage + * This method is used to flush collection insert buffer into storage * - * @param collection_name, target collection's name. + * @param collection_name_array, target collections name array. * * @return Indicate if this operation is successful. */ virtual Status - FlushCollection(const std::string& collection_name) = 0; - - /** - * @brief Flush all buffer into storage - * - * This method is used to all collection buffer into storage - * - * @return Indicate if this operation is successful. - */ - virtual Status - Flush() = 0; + Flush(const std::vector& collection_name_array) = 0; /** * @brief Compact collection, permanently remove deleted vectors @@ -599,7 +587,7 @@ class Connection { * @return Indicate if this operation is successful. */ virtual Status - CompactCollection(const std::string& collection_name) = 0; + Compact(const std::string& collection_name) = 0; /*******************************New Interface**********************************/ diff --git a/sdk/interface/ConnectionImpl.cpp b/sdk/interface/ConnectionImpl.cpp index 1e4c7fb334..943a0349f2 100644 --- a/sdk/interface/ConnectionImpl.cpp +++ b/sdk/interface/ConnectionImpl.cpp @@ -103,21 +103,16 @@ ConnectionImpl::Insert(const std::string& collection_name, const std::string& pa } Status -ConnectionImpl::GetEntityByID(const std::string& collection_name, int64_t entity_id, Entity& entity_data) { - return client_proxy_->GetEntityByID(collection_name, entity_id, entity_data); +ConnectionImpl::GetEntityByID(const std::string& collection_name, + const std::vector& id_array, + std::vector& entities_data) { + return client_proxy_->GetEntityByID(collection_name, id_array, entities_data); } Status -ConnectionImpl::GetEntitiesByID(const std::string& collection_name, - const std::vector& id_array, - std::vector& entities_data) { - return client_proxy_->GetEntitiesByID(collection_name, id_array, entities_data); -} - -Status -ConnectionImpl::GetIDsInSegment(const std::string& collection_name, const std::string& segment_name, +ConnectionImpl::ListIDInSegment(const std::string& collection_name, const std::string& segment_name, std::vector& id_array) { - return client_proxy_->GetIDsInSegment(collection_name, segment_name, id_array); + return client_proxy_->ListIDInSegment(collection_name, segment_name, id_array); } Status @@ -145,38 +140,38 @@ ConnectionImpl::SearchByID(const std::string& collection_name, const PartitionTa } Status -ConnectionImpl::DescribeCollection(const std::string& collection_name, CollectionParam& collection_schema) { - return client_proxy_->DescribeCollection(collection_name, collection_schema); +ConnectionImpl::GetCollectionInfo(const std::string& collection_name, CollectionParam& collection_schema) { + return client_proxy_->GetCollectionInfo(collection_name, collection_schema); } Status -ConnectionImpl::CountCollection(const std::string& collection_name, int64_t& row_count) { - return client_proxy_->CountCollection(collection_name, row_count); +ConnectionImpl::CountEntities(const std::string& collection_name, int64_t& row_count) { + return client_proxy_->CountEntities(collection_name, row_count); } Status -ConnectionImpl::ShowCollections(std::vector& collection_array) { - return client_proxy_->ShowCollections(collection_array); +ConnectionImpl::ListCollections(std::vector& collection_array) { + return client_proxy_->ListCollections(collection_array); } Status -ConnectionImpl::ShowCollectionInfo(const std::string& collection_name, std::string& collection_info) { - return client_proxy_->ShowCollectionInfo(collection_name, collection_info); +ConnectionImpl::GetCollectionStats(const std::string& collection_name, std::string& collection_stats) { + return client_proxy_->GetCollectionStats(collection_name, collection_stats); } Status -ConnectionImpl::DeleteByID(const std::string& collection_name, const std::vector& id_array) { - return client_proxy_->DeleteByID(collection_name, id_array); +ConnectionImpl::DeleteEntityByID(const std::string& collection_name, const std::vector& id_array) { + return client_proxy_->DeleteEntityByID(collection_name, id_array); } Status -ConnectionImpl::PreloadCollection(const std::string& collection_name) const { - return client_proxy_->PreloadCollection(collection_name); +ConnectionImpl::LoadCollection(const std::string& collection_name) const { + return client_proxy_->LoadCollection(collection_name); } Status -ConnectionImpl::DescribeIndex(const std::string& collection_name, IndexParam& index_param) const { - return client_proxy_->DescribeIndex(collection_name, index_param); +ConnectionImpl::GetIndexInfo(const std::string& collection_name, IndexParam& index_param) const { + return client_proxy_->GetIndexInfo(collection_name, index_param); } Status @@ -189,9 +184,14 @@ ConnectionImpl::CreatePartition(const PartitionParam& partition_param) { return client_proxy_->CreatePartition(partition_param); } +bool +ConnectionImpl::HasPartition(const std::string& collection_name, const std::string& partition_tag) const { + return client_proxy_->HasPartition(collection_name, partition_tag); +} + Status -ConnectionImpl::ShowPartitions(const std::string& collection_name, PartitionTagList& partition_array) const { - return client_proxy_->ShowPartitions(collection_name, partition_array); +ConnectionImpl::ListPartitions(const std::string& collection_name, PartitionTagList& partition_array) const { + return client_proxy_->ListPartitions(collection_name, partition_array); } Status @@ -200,18 +200,13 @@ ConnectionImpl::DropPartition(const PartitionParam& partition_param) { } Status -ConnectionImpl::FlushCollection(const std::string& Status) { - return client_proxy_->FlushCollection(Status); +ConnectionImpl::Flush(const std::vector& collection_name_array) { + return client_proxy_->Flush(collection_name_array); } Status -ConnectionImpl::Flush() { - return client_proxy_->Flush(); -} - -Status -ConnectionImpl::CompactCollection(const std::string& collection_name) { - return client_proxy_->CompactCollection(collection_name); +ConnectionImpl::Compact(const std::string& collection_name) { + return client_proxy_->Compact(collection_name); } /*******************************New Interface**********************************/ diff --git a/sdk/interface/ConnectionImpl.h b/sdk/interface/ConnectionImpl.h index a1b1351c9c..ecb9d497b4 100644 --- a/sdk/interface/ConnectionImpl.h +++ b/sdk/interface/ConnectionImpl.h @@ -71,15 +71,12 @@ class ConnectionImpl : public Connection { std::vector& id_array) override; Status - GetEntityByID(const std::string& collection_name, int64_t entity_id, Entity& entity_data) override; + GetEntityByID(const std::string& collection_name, + const std::vector& id_array, + std::vector& entities_data) override; Status - GetEntitiesByID(const std::string& collection_name, - const std::vector& id_array, - std::vector& entities_data) override; - - Status - GetIDsInSegment(const std::string& collection_name, const std::string& segment_name, + ListIDInSegment(const std::string& collection_name, const std::string& segment_name, std::vector& id_array) override; Status @@ -93,25 +90,25 @@ class ConnectionImpl : public Connection { const std::string& extra_params, TopKQueryResult& topk_query_result) override; Status - DescribeCollection(const std::string& collection_name, CollectionParam& collection_schema) override; + GetCollectionInfo(const std::string& collection_name, CollectionParam& collection_param) override; Status - CountCollection(const std::string& collection_name, int64_t& entity_count) override; + CountEntities(const std::string& collection_name, int64_t& entity_count) override; Status - ShowCollections(std::vector& collection_array) override; + ListCollections(std::vector& collection_array) override; Status - ShowCollectionInfo(const std::string& collection_name, std::string& collection_info) override; + GetCollectionStats(const std::string& collection_name, std::string& collection_stats) override; Status - DeleteByID(const std::string& collection_name, const std::vector& id_array) override; + DeleteEntityByID(const std::string& collection_name, const std::vector& id_array) override; Status - PreloadCollection(const std::string& collection_name) const override; + LoadCollection(const std::string& collection_name) const override; Status - DescribeIndex(const std::string& collection_name, IndexParam& index_param) const override; + GetIndexInfo(const std::string& collection_name, IndexParam& index_param) const override; Status DropIndex(const std::string& collection_name) const override; @@ -119,20 +116,20 @@ class ConnectionImpl : public Connection { Status CreatePartition(const PartitionParam& partition_param) override; + bool + HasPartition(const std::string& collection_name, const std::string& partition_tag) const override; + Status - ShowPartitions(const std::string& collection_name, PartitionTagList& partition_tag_array) const override; + ListPartitions(const std::string& collection_name, PartitionTagList& partition_tag_array) const override; Status DropPartition(const PartitionParam& partition_param) override; Status - FlushCollection(const std::string& collection_name) override; + Flush(const std::vector& collection_name_array) override; Status - Flush() override; - - Status - CompactCollection(const std::string& collection_name) override; + Compact(const std::string& collection_name) override; /*******************************New Interface**********************************/