diff --git a/cpp/conf/server_config.template b/cpp/conf/server_config.template index 3441f1eaab..055a2053f6 100644 --- a/cpp/conf/server_config.template +++ b/cpp/conf/server_config.template @@ -1,5 +1,5 @@ server_config: - address: 0.0.0.0 + address: 0.0.0.0 # milvus server ip address port: 19530 # the port milvus listen to, default: 19530, range: 1025 ~ 65534 gpu_index: 0 # the gpu milvus use, default: 0, range: 0 ~ gpu number - 1 mode: single # milvus deployment type: single, cluster, read_only @@ -7,43 +7,35 @@ server_config: db_config: db_path: @MILVUS_DB_PATH@ # milvus data storage path db_slave_path: # secondry data storage path, split by semicolon - - parallel_reduce: false # use multi-threads to reduce topk result + parallel_reduce: false # use multi-threads to reduce topk result # URI format: dialect://username:password@host:port/database # All parts except dialect are optional, but you MUST include the delimiters # Currently dialect supports mysql or sqlite db_backend_url: sqlite://:@:/ - index_building_threshold: 1024 # index building trigger threshold, default: 1024, unit: MB archive_disk_threshold: 0 # triger archive action if storage size exceed this value, 0 means no limit, unit: GB archive_days_threshold: 0 # files older than x days will be archived, 0 means no limit, unit: day insert_buffer_size: 4 # maximum insert buffer size allowed, default: 4, unit: GB, should be at least 1 GB. # the sum of insert_buffer_size and cpu_cache_capacity should be less than total memory, unit: GB metric_config: - is_startup: off # if monitoring start: on, off - collector: prometheus # metrics collector: prometheus - prometheus_config: # following are prometheus configure - port: 8080 # the port prometheus use to fetch metrics - push_gateway_ip_address: 127.0.0.1 # push method configure: push gateway ip address - push_gateway_port: 9091 # push method configure: push gateway port + is_startup: off # if monitoring start: on, off + collector: prometheus # metrics collector: prometheus + prometheus_config: # following are prometheus configure + port: 8080 # the port prometheus use to fetch metrics + push_gateway_ip_address: 127.0.0.1 # push method configure: push gateway ip address + push_gateway_port: 9091 # push method configure: push gateway port - -license_config: # license configure - license_path: "@MILVUS_DB_PATH@/system.license" # license file path - -cache_config: # cache configure - cpu_cache_capacity: 16 # how many memory are used as cache, unit: GB, range: 0 ~ less than total memory - cpu_cache_free_percent: 0.85 # old data will be erased from cache when cache is full, this value specify how much memory should be kept, range: greater than zero ~ 1.0 - insert_cache_immediately: false # insert data will be load into cache immediately for hot query - gpu_cache_capacity: 5 # how many memory are used as cache in gpu, unit: GB, RANGE: 0 ~ less than total memory - gpu_cache_free_percent: 0.85 # old data will be erased from cache when cache is full, this value specify how much memory should be kept, range: greater than zero ~ 1.0 - gpu_ids: 0,1 # gpu id +cache_config: + cpu_cache_capacity: 16 # how many memory are used as cache, unit: GB, range: 0 ~ less than total memory + cpu_cache_free_percent: 0.85 # old data will be erased from cache when cache is full, this value specify how much memory should be kept, range: greater than zero ~ 1.0 + insert_cache_immediately: false # insert data will be load into cache immediately for hot query + gpu_cache_capacity: 5 # how many memory are used as cache in gpu, unit: GB, RANGE: 0 ~ less than total memory + gpu_cache_free_percent: 0.85 # old data will be erased from cache when cache is full, this value specify how much memory should be kept, range: greater than zero ~ 1.0 + gpu_ids: 0,1 # gpu id engine_config: - nprobe: 10 - nlist: 16384 use_blas_threshold: 20 - metric_type: L2 # compare vectors by euclidean distance(L2) or inner product(IP), optional: L2 or IP - omp_thread_num: 0 # how many compute threads be used by engine, 0 means use all cpu core to compute + metric_type: L2 # compare vectors by euclidean distance(L2) or inner product(IP), optional: L2 or IP + omp_thread_num: 0 # how many cpu cores be used by engine, 0 means use all cpu cores used diff --git a/cpp/src/db/DBImpl.cpp b/cpp/src/db/DBImpl.cpp index 4299801cb9..0910d04d7a 100644 --- a/cpp/src/db/DBImpl.cpp +++ b/cpp/src/db/DBImpl.cpp @@ -396,7 +396,7 @@ Status DBImpl::MergeFiles(const std::string& table_id, const meta::DateT& date, ENGINE_LOG_DEBUG << "Merging file " << file_schema.file_id_; index_size = index->Size(); - if (index_size >= options_.index_trigger_size) break; + if (index_size >= file_schema.index_file_size_) break; } //step 3: serialize to disk @@ -546,6 +546,11 @@ Status DBImpl::CreateIndex(const std::string& table_id, const TableIndex& index) //step 2: drop old index files DropIndex(table_id); + if(index.engine_type_ == (int)EngineType::FAISS_IDMAP) { + ENGINE_LOG_DEBUG << "index type = IDMAP, no need to build index"; + return Status::OK(); + } + //step 3: update index info status = meta_ptr_->UpdateTableIndexParam(table_id, index); @@ -607,7 +612,7 @@ Status DBImpl::BuildIndex(const meta::TableFileSchema& file) { try { server::CollectBuildIndexMetrics metrics; - index = to_index->BuildIndex(table_file.location_); + index = to_index->BuildIndex(table_file.location_, (EngineType)table_file.engine_type_); } catch (std::exception& ex) { //typical error: out of gpu memory std::string msg = "BuildIndex encounter exception" + std::string(ex.what()); diff --git a/cpp/src/db/Options.h b/cpp/src/db/Options.h index 8081531236..4cd262ccfa 100644 --- a/cpp/src/db/Options.h +++ b/cpp/src/db/Options.h @@ -55,9 +55,8 @@ struct Options { } MODE; Options(); - uint16_t memory_sync_interval = 1; //unit: second + uint16_t merge_trigger_number = 2; - size_t index_trigger_size = ONE_GB; //unit: byte DBMetaOptions meta; int mode = MODE::SINGLE; diff --git a/cpp/src/db/engine/ExecutionEngine.h b/cpp/src/db/engine/ExecutionEngine.h index f0ce5554ca..8f61a7c833 100644 --- a/cpp/src/db/engine/ExecutionEngine.h +++ b/cpp/src/db/engine/ExecutionEngine.h @@ -60,7 +60,7 @@ public: float *distances, long *labels) const = 0; - virtual std::shared_ptr BuildIndex(const std::string&) = 0; + virtual std::shared_ptr BuildIndex(const std::string& location, EngineType engine_type) = 0; virtual Status Cache() = 0; diff --git a/cpp/src/db/engine/ExecutionEngineImpl.cpp b/cpp/src/db/engine/ExecutionEngineImpl.cpp index 2952d70ef4..35cb88c453 100644 --- a/cpp/src/db/engine/ExecutionEngineImpl.cpp +++ b/cpp/src/db/engine/ExecutionEngineImpl.cpp @@ -214,11 +214,11 @@ Status ExecutionEngineImpl::Merge(const std::string &location) { } ExecutionEnginePtr -ExecutionEngineImpl::BuildIndex(const std::string &location) { +ExecutionEngineImpl::BuildIndex(const std::string &location, EngineType engine_type) { ENGINE_LOG_DEBUG << "Build index file: " << location << " from: " << location_; auto from_index = std::dynamic_pointer_cast(index_); - auto to_index = CreatetVecIndex(index_type_); + auto to_index = CreatetVecIndex(engine_type); if (!to_index) { throw Exception("Create Empty VecIndex"); } @@ -236,7 +236,7 @@ ExecutionEngineImpl::BuildIndex(const std::string &location) { build_cfg); if (ec != server::KNOWHERE_SUCCESS) { throw Exception("Build index error"); } - return std::make_shared(to_index, location, index_type_, metric_type_, nlist_); + return std::make_shared(to_index, location, engine_type, metric_type_, nlist_); } Status ExecutionEngineImpl::Search(long n, diff --git a/cpp/src/db/engine/ExecutionEngineImpl.h b/cpp/src/db/engine/ExecutionEngineImpl.h index 90be5bddf2..74d49d1926 100644 --- a/cpp/src/db/engine/ExecutionEngineImpl.h +++ b/cpp/src/db/engine/ExecutionEngineImpl.h @@ -59,7 +59,7 @@ public: float *distances, long *labels) const override; - ExecutionEnginePtr BuildIndex(const std::string &) override; + ExecutionEnginePtr BuildIndex(const std::string &location, EngineType engine_type) override; Status Cache() override; diff --git a/cpp/src/db/insert/MemTableFile.cpp b/cpp/src/db/insert/MemTableFile.cpp index 99f2e19b35..48e52d0d9b 100644 --- a/cpp/src/db/insert/MemTableFile.cpp +++ b/cpp/src/db/insert/MemTableFile.cpp @@ -87,7 +87,7 @@ Status MemTableFile::Serialize() { table_file_schema_.file_size_ = execution_engine_->PhysicalSize(); table_file_schema_.row_count_ = execution_engine_->Count(); - table_file_schema_.file_type_ = (size >= options_.index_trigger_size) ? + table_file_schema_.file_type_ = (size >= table_file_schema_.index_file_size_) ? meta::TableFileSchema::TO_INDEX : meta::TableFileSchema::RAW; auto status = meta_->UpdateTableFile(table_file_schema_); diff --git a/cpp/src/db/meta/MetaTypes.h b/cpp/src/db/meta/MetaTypes.h index 5fa4c0bcfc..61af792746 100644 --- a/cpp/src/db/meta/MetaTypes.h +++ b/cpp/src/db/meta/MetaTypes.h @@ -19,7 +19,7 @@ namespace meta { constexpr int32_t DEFAULT_ENGINE_TYPE = (int)EngineType::FAISS_IDMAP; constexpr int32_t DEFAULT_NLIST = 16384; -constexpr int32_t DEFAULT_INDEX_FILE_SIZE = 1024*ONE_MB; +constexpr int32_t DEFAULT_INDEX_FILE_SIZE = ONE_GB; constexpr int32_t DEFAULT_METRIC_TYPE = (int)MetricType::L2; constexpr int64_t FLAG_MASK_USERID = 1; @@ -71,6 +71,7 @@ struct TableFileSchema { int64_t created_on_ = 0; int32_t engine_type_ = DEFAULT_ENGINE_TYPE; int32_t nlist_ = DEFAULT_NLIST; //not persist to meta + int32_t index_file_size_ = DEFAULT_INDEX_FILE_SIZE; //not persist to meta int32_t metric_type_ = DEFAULT_METRIC_TYPE; //not persist to meta }; // TableFileSchema diff --git a/cpp/src/db/meta/MySQLMetaImpl.cpp b/cpp/src/db/meta/MySQLMetaImpl.cpp index 6e952642bd..459da69deb 100644 --- a/cpp/src/db/meta/MySQLMetaImpl.cpp +++ b/cpp/src/db/meta/MySQLMetaImpl.cpp @@ -835,6 +835,7 @@ Status MySQLMetaImpl::CreateTableFile(TableFileSchema &file_schema) { file_schema.updated_time_ = file_schema.created_on_; file_schema.engine_type_ = table_schema.engine_type_; file_schema.nlist_ = table_schema.nlist_; + file_schema.index_file_size_ = table_schema.index_file_size_; file_schema.metric_type_ = table_schema.metric_type_; utils::GetTableFilePath(options_, file_schema); @@ -949,8 +950,9 @@ Status MySQLMetaImpl::FilesToIndex(TableFilesSchema &files) { groups[table_file.table_id_] = table_schema; } - table_file.metric_type_ = groups[table_file.table_id_].metric_type_; table_file.nlist_ = groups[table_file.table_id_].nlist_; + table_file.index_file_size_ = groups[table_file.table_id_].index_file_size_; + table_file.metric_type_ = groups[table_file.table_id_].metric_type_; table_file.dimension_ = groups[table_file.table_id_].dimension_; utils::GetTableFilePath(options_, table_file); @@ -1043,10 +1045,12 @@ Status MySQLMetaImpl::FilesToSearch(const std::string &table_id, table_file.engine_type_ = resRow["engine_type"]; - table_file.metric_type_ = table_schema.metric_type_; - table_file.nlist_ = table_schema.nlist_; + table_file.index_file_size_ = table_schema.index_file_size_; + + table_file.metric_type_ = table_schema.metric_type_; + std::string file_id; resRow["file_id"].to_string(file_id); table_file.file_id_ = file_id; @@ -1155,10 +1159,12 @@ Status MySQLMetaImpl::FilesToSearch(const std::string &table_id, table_file.engine_type_ = resRow["engine_type"]; - table_file.metric_type_ = table_schema.metric_type_; - table_file.nlist_ = table_schema.nlist_; + table_file.index_file_size_ = table_schema.index_file_size_; + + table_file.metric_type_ = table_schema.metric_type_; + std::string file_id; resRow["file_id"].to_string(file_id); table_file.file_id_ = file_id; @@ -1255,10 +1261,12 @@ Status MySQLMetaImpl::FilesToMerge(const std::string &table_id, table_file.engine_type_ = resRow["engine_type"]; - table_file.metric_type_ = table_schema.metric_type_; - table_file.nlist_ = table_schema.nlist_; + table_file.index_file_size_ = table_schema.index_file_size_; + + table_file.metric_type_ = table_schema.metric_type_; + table_file.created_on_ = resRow["created_on"]; table_file.dimension_ = table_schema.dimension_; @@ -1338,10 +1346,12 @@ Status MySQLMetaImpl::GetTableFiles(const std::string &table_id, file_schema.engine_type_ = resRow["engine_type"]; - file_schema.metric_type_ = table_schema.metric_type_; - file_schema.nlist_ = table_schema.nlist_; + file_schema.index_file_size_ = table_schema.index_file_size_; + + file_schema.metric_type_ = table_schema.metric_type_; + std::string file_id; resRow["file_id"].to_string(file_id); file_schema.file_id_ = file_id; diff --git a/cpp/src/db/meta/SqliteMetaImpl.cpp b/cpp/src/db/meta/SqliteMetaImpl.cpp index 66346cf6c2..99f0b104a9 100644 --- a/cpp/src/db/meta/SqliteMetaImpl.cpp +++ b/cpp/src/db/meta/SqliteMetaImpl.cpp @@ -548,6 +548,7 @@ Status SqliteMetaImpl::CreateTableFile(TableFileSchema &file_schema) { file_schema.updated_time_ = file_schema.created_on_; file_schema.engine_type_ = table_schema.engine_type_; file_schema.nlist_ = table_schema.nlist_; + file_schema.index_file_size_ = table_schema.index_file_size_; file_schema.metric_type_ = table_schema.metric_type_; //multi-threads call sqlite update may get exception('bad logic', etc), so we add a lock here @@ -608,8 +609,9 @@ Status SqliteMetaImpl::FilesToIndex(TableFilesSchema &files) { } groups[table_file.table_id_] = table_schema; } - table_file.metric_type_ = groups[table_file.table_id_].metric_type_; table_file.nlist_ = groups[table_file.table_id_].nlist_; + table_file.index_file_size_ = groups[table_file.table_id_].index_file_size_; + table_file.metric_type_ = groups[table_file.table_id_].metric_type_; table_file.dimension_ = groups[table_file.table_id_].dimension_; files.push_back(table_file); } @@ -703,9 +705,11 @@ Status SqliteMetaImpl::FilesToSearch(const std::string &table_id, table_file.row_count_ = std::get<5>(file); table_file.date_ = std::get<6>(file); table_file.engine_type_ = std::get<7>(file); - table_file.metric_type_ = table_schema.metric_type_; table_file.nlist_ = table_schema.nlist_; + table_file.index_file_size_ = table_schema.index_file_size_; + table_file.metric_type_ = table_schema.metric_type_; table_file.dimension_ = table_schema.dimension_; + utils::GetTableFilePath(options_, table_file); auto dateItr = files.find(table_file.date_); if (dateItr == files.end()) { @@ -782,8 +786,10 @@ Status SqliteMetaImpl::FilesToSearch(const std::string &table_id, table_file.date_ = std::get<6>(file); table_file.engine_type_ = std::get<7>(file); table_file.dimension_ = table_schema.dimension_; - table_file.metric_type_ = table_schema.metric_type_; table_file.nlist_ = table_schema.nlist_; + table_file.index_file_size_ = table_schema.index_file_size_; + table_file.metric_type_ = table_schema.metric_type_; + utils::GetTableFilePath(options_, table_file); auto dateItr = files.find(table_file.date_); if (dateItr == files.end()) { @@ -842,8 +848,10 @@ Status SqliteMetaImpl::FilesToMerge(const std::string &table_id, table_file.date_ = std::get<6>(file); table_file.created_on_ = std::get<7>(file); table_file.dimension_ = table_schema.dimension_; - table_file.metric_type_ = table_schema.metric_type_; table_file.nlist_ = table_schema.nlist_; + table_file.index_file_size_ = table_schema.index_file_size_; + table_file.metric_type_ = table_schema.metric_type_; + utils::GetTableFilePath(options_, table_file); auto dateItr = files.find(table_file.date_); if (dateItr == files.end()) { @@ -892,8 +900,9 @@ Status SqliteMetaImpl::GetTableFiles(const std::string& table_id, file_schema.row_count_ = std::get<4>(file); file_schema.date_ = std::get<5>(file); file_schema.engine_type_ = std::get<6>(file); - file_schema.metric_type_ = table_schema.metric_type_; file_schema.nlist_ = table_schema.nlist_; + file_schema.index_file_size_ = table_schema.index_file_size_; + file_schema.metric_type_ = table_schema.metric_type_; file_schema.created_on_ = std::get<7>(file); file_schema.dimension_ = table_schema.dimension_; diff --git a/cpp/src/server/DBWrapper.cpp b/cpp/src/server/DBWrapper.cpp index 923f8f2861..d5ca579773 100644 --- a/cpp/src/server/DBWrapper.cpp +++ b/cpp/src/server/DBWrapper.cpp @@ -27,19 +27,6 @@ DBWrapper::DBWrapper() { std::string db_slave_path = db_config.GetValue(CONFIG_DB_SLAVE_PATH); StringHelpFunctions::SplitStringByDelimeter(db_slave_path, ";", opt.meta.slave_paths); - int64_t index_size = db_config.GetInt64Value(CONFIG_DB_INDEX_TRIGGER_SIZE); - if(index_size > 0) {//ensure larger than zero, unit is MB - opt.index_trigger_size = (size_t)index_size * engine::ONE_MB; - } - int64_t insert_buffer_size = db_config.GetInt64Value(CONFIG_DB_INSERT_BUFFER_SIZE, 4); - if (insert_buffer_size >= 1) { - opt.insert_buffer_size = insert_buffer_size * engine::ONE_GB; - } - else { - std::cout << "ERROR: insert_buffer_size should be at least 1 GB" << std::endl; - kill(0, SIGUSR1); - } - // cache config ConfigNode& cache_config = ServerConfig::GetInstance().GetConfig(CONFIG_CACHE); opt.insert_cache_immediately_ = cache_config.GetBoolValue(CONFIG_INSERT_CACHE_IMMEDIATELY, false); diff --git a/cpp/src/server/ServerConfig.cpp b/cpp/src/server/ServerConfig.cpp index 71d32452ac..6e740002cf 100644 --- a/cpp/src/server/ServerConfig.cpp +++ b/cpp/src/server/ServerConfig.cpp @@ -79,19 +79,6 @@ ServerError ServerConfig::ValidateConfig() const { return SERVER_INVALID_ARGUMENT; } - uint64_t index_building_threshold = (uint64_t)db_config.GetInt32Value(CONFIG_DB_INDEX_TRIGGER_SIZE, 1024); - index_building_threshold *= MB; - - size_t gpu_mem = 0; - ValidationUtil::GetGpuMemory(gpu_index, gpu_mem); - if(index_building_threshold >= gpu_mem) { - std::cout << "Error: index_building_threshold execeed gpu memory" << std::endl; - return SERVER_INVALID_ARGUMENT; - } else if(index_building_threshold >= gpu_mem/3) { - std::cout << "Warnning: index_building_threshold is greater than 1/3 of gpu memory, " - << "some index type(such as IVFLAT) may cause cuda::bad_alloc() error" << std::endl; - } - //cache config validation ConfigNode cache_config = GetConfig(CONFIG_CACHE); uint64_t cache_cap = (uint64_t)cache_config.GetInt64Value(CONFIG_CPU_CACHE_CAPACITY, 16); diff --git a/cpp/src/server/ServerConfig.h b/cpp/src/server/ServerConfig.h index a800664ebd..2979e35c15 100644 --- a/cpp/src/server/ServerConfig.h +++ b/cpp/src/server/ServerConfig.h @@ -24,7 +24,6 @@ static const char* CONFIG_DB = "db_config"; static const char* CONFIG_DB_URL = "db_backend_url"; static const char* CONFIG_DB_PATH = "db_path"; static const char* CONFIG_DB_SLAVE_PATH = "db_slave_path"; -static const char* CONFIG_DB_INDEX_TRIGGER_SIZE = "index_building_threshold"; static const char* CONFIG_DB_ARCHIVE_DISK = "archive_disk_threshold"; static const char* CONFIG_DB_ARCHIVE_DAYS = "archive_days_threshold"; static const char* CONFIG_DB_INSERT_BUFFER_SIZE = "insert_buffer_size"; diff --git a/cpp/unittest/server/config_test.cpp b/cpp/unittest/server/config_test.cpp index 462b813f26..59669710ba 100644 --- a/cpp/unittest/server/config_test.cpp +++ b/cpp/unittest/server/config_test.cpp @@ -135,12 +135,6 @@ TEST(ConfigTest, SERVER_CONFIG_TEST) { err = config.ValidateConfig(); ASSERT_NE(err, server::SERVER_SUCCESS); - size_t index_building_threshold = (gpu_mem + 1*MB)/MB; - db_config.SetValue(server::CONFIG_DB_INDEX_TRIGGER_SIZE, - std::to_string(index_building_threshold)); - err = config.ValidateConfig(); - ASSERT_NE(err, server::SERVER_SUCCESS); - insert_buffer_size = total_mem/GB + 2; db_config.SetValue(server::CONFIG_DB_INSERT_BUFFER_SIZE, std::to_string(insert_buffer_size)); err = config.ValidateConfig();