diff --git a/configs/milvus.yaml b/configs/milvus.yaml index 643ac70fbf..0281012043 100644 --- a/configs/milvus.yaml +++ b/configs/milvus.yaml @@ -416,7 +416,7 @@ queryNode: buildParallelRate: 0.5 # the ratio of building interim index parallel matched with cpu num multipleChunkedEnable: true # Enable multiple chunked search knowhereScoreConsistency: false # Enable knowhere strong consistency score computation logic - jsonIndexMemoryBudgetInTantivy: 64 # the memory budget for the JSON index In Tantivy + jsonIndexMemoryBudgetInTantivy: 16 # the memory budget for the JSON index In Tantivy jsonIndexCommitInterval: 200 # the commit interval for the JSON index to commit loadMemoryUsageFactor: 1 # The multiply factor of calculating the memory usage while loading segments enableDisk: false # enable querynode load disk index, and search on disk index diff --git a/internal/core/src/common/Consts.h b/internal/core/src/common/Consts.h index 1834433efd..d1bf202347 100644 --- a/internal/core/src/common/Consts.h +++ b/internal/core/src/common/Consts.h @@ -79,6 +79,6 @@ const int64_t DEFAULT_HYBRID_INDEX_BITMAP_CARDINALITY_LIMIT = 100; const size_t MARISA_NULL_KEY_ID = -1; -const int64_t DEFAULT_JSON_INDEX_MEMORY_BUDGET = 67108864; // bytes, 64MB +const int64_t DEFAULT_JSON_INDEX_MEMORY_BUDGET = 16777216; // bytes, 16MB const int64_t DEFAULT_JSON_INDEX_COMMIT_INTERVAL = 200; const bool DEFAULT_JSON_INDEX_ENABLED = true; \ No newline at end of file diff --git a/internal/core/src/exec/expression/Expr.cpp b/internal/core/src/exec/expression/Expr.cpp index 3c67f600a0..850323ad04 100644 --- a/internal/core/src/exec/expression/Expr.cpp +++ b/internal/core/src/exec/expression/Expr.cpp @@ -152,8 +152,6 @@ CompileExpression(const expr::TypedExprPtr& expr, const std::unordered_set& flatten_candidates, bool enable_constant_folding) { ExprPtr result; - std::cout << "CompileExpression context" << context->get_consistency_level() - << std::endl; auto compiled_inputs = CompileInputs(expr, context, flatten_candidates); auto GetTypes = [](const std::vector& exprs) { diff --git a/internal/core/src/index/JsonKeyInvertedIndex.cpp b/internal/core/src/index/JsonKeyInvertedIndex.cpp index dc4cba3a6c..d2e9749141 100644 --- a/internal/core/src/index/JsonKeyInvertedIndex.cpp +++ b/internal/core/src/index/JsonKeyInvertedIndex.cpp @@ -222,7 +222,8 @@ JsonKeyInvertedIndex::Upload(const Config& config) { index_files.reserve(remote_paths_to_size.size() + remote_mem_path_to_size.size()); for (auto& file : remote_paths_to_size) { - index_files.emplace_back(file.first, file.second); + index_files.emplace_back(disk_file_manager_->GetFileName(file.first), + file.second); } for (auto& file : remote_mem_path_to_size) { index_files.emplace_back(file.first, file.second); @@ -240,6 +241,15 @@ JsonKeyInvertedIndex::Load(milvus::tracer::TraceContext ctx, AssertInfo(index_files.has_value(), "index file paths is empty when load json key index"); + for (auto& index_file : index_files.value()) { + boost::filesystem::path p(index_file); + if (!p.has_parent_path()) { + auto remote_prefix = + disk_file_manager_->GetRemoteJsonKeyLogPrefix(); + index_file = remote_prefix + "/" + index_file; + } + } + disk_file_manager_->CacheJsonKeyIndexToDisk(index_files.value()); AssertInfo( tantivy_index_exist(path_.c_str()), "index not exist: {}", path_); diff --git a/internal/core/src/query/ExecPlanNodeVisitor.cpp b/internal/core/src/query/ExecPlanNodeVisitor.cpp index 383aa2097b..122075faa1 100644 --- a/internal/core/src/query/ExecPlanNodeVisitor.cpp +++ b/internal/core/src/query/ExecPlanNodeVisitor.cpp @@ -83,8 +83,6 @@ ExecPlanNodeVisitor::ExecuteTask( plan.plan_node_->ToString(), query_context->get_active_count(), query_context->get_query_timestamp()); - std::cout << "ExecuteTask query_context" - << query_context->get_consistency_level() << std::endl; auto task = milvus::exec::Task::Create(DEFAULT_TASK_ID, plan, 0, query_context); int64_t processed_num = 0; diff --git a/internal/core/src/storage/DiskFileManagerImpl.h b/internal/core/src/storage/DiskFileManagerImpl.h index 99e417561a..896c2b991e 100644 --- a/internal/core/src/storage/DiskFileManagerImpl.h +++ b/internal/core/src/storage/DiskFileManagerImpl.h @@ -133,15 +133,15 @@ class DiskFileManagerImpl : public FileManagerImpl { return added_total_file_size_; } + std::string + GetFileName(const std::string& localfile); + private: int64_t GetIndexBuildId() { return index_meta_.build_id; } - std::string - GetFileName(const std::string& localfile); - std::string GetRemoteIndexPath(const std::string& file_name, int64_t slice_num) const; diff --git a/pkg/util/paramtable/component_param.go b/pkg/util/paramtable/component_param.go index 1d6e396c02..e52356171f 100644 --- a/pkg/util/paramtable/component_param.go +++ b/pkg/util/paramtable/component_param.go @@ -3285,7 +3285,7 @@ user-task-polling: p.JSONIndexMemoryBudgetInTantivy = ParamItem{ Key: "queryNode.segcore.jsonIndexMemoryBudgetInTantivy", Version: "2.5.0", - DefaultValue: "64", + DefaultValue: "16", Doc: "the memory budget for the JSON index In Tantivy", Export: true, }