mirror of
https://gitee.com/milvus-io/milvus.git
synced 2026-01-07 19:31:51 +08:00
f2b83d316b
Chunk cache supports loading raw vectors into memory. issue: https://github.com/milvus-io/milvus/issues/35273 --------- Signed-off-by: bigsheeper <yihao.dai@zilliz.com>
83 lines
2.6 KiB
C++
83 lines
2.6 KiB
C++
// Licensed to the LF AI & Data foundation under one
|
|
// or more contributor license agreements. See the NOTICE file
|
|
// distributed with this work for additional information
|
|
// regarding copyright ownership. The ASF licenses this file
|
|
// to you under the Apache License, Version 2.0 (the
|
|
// "License"); you may not use this file except in compliance
|
|
// with the License. You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
#pragma once
|
|
#include <future>
|
|
#include <unordered_map>
|
|
#include "storage/MmapChunkManager.h"
|
|
#include "mmap/Column.h"
|
|
|
|
namespace milvus::storage {
|
|
|
|
extern std::map<std::string, int> ReadAheadPolicy_Map;
|
|
|
|
class ChunkCache {
|
|
public:
|
|
explicit ChunkCache(const std::string& read_ahead_policy,
|
|
ChunkManagerPtr cm,
|
|
MmapChunkManagerPtr mcm)
|
|
: cm_(cm), mcm_(mcm) {
|
|
auto iter = ReadAheadPolicy_Map.find(read_ahead_policy);
|
|
AssertInfo(iter != ReadAheadPolicy_Map.end(),
|
|
"unrecognized read ahead policy: {}, "
|
|
"should be one of `normal, random, sequential, "
|
|
"willneed, dontneed`",
|
|
read_ahead_policy);
|
|
read_ahead_policy_ = iter->second;
|
|
LOG_INFO("Init ChunkCache with read_ahead_policy: {}",
|
|
read_ahead_policy);
|
|
}
|
|
|
|
~ChunkCache() = default;
|
|
|
|
public:
|
|
std::shared_ptr<ColumnBase>
|
|
Read(const std::string& filepath,
|
|
const MmapChunkDescriptorPtr& descriptor,
|
|
const FieldMeta& field_meta,
|
|
bool mmap_enabled);
|
|
|
|
void
|
|
Remove(const std::string& filepath);
|
|
|
|
void
|
|
Prefetch(const std::string& filepath);
|
|
|
|
private:
|
|
std::shared_ptr<ColumnBase>
|
|
Mmap(const FieldDataPtr& field_data,
|
|
const MmapChunkDescriptorPtr& descriptor,
|
|
const FieldMeta& field_meta,
|
|
bool mmap_enabled);
|
|
|
|
private:
|
|
using ColumnTable = std::unordered_map<
|
|
std::string,
|
|
std::pair<std::promise<std::shared_ptr<ColumnBase>>,
|
|
std::shared_future<std::shared_ptr<ColumnBase>>>>;
|
|
|
|
private:
|
|
mutable std::shared_mutex mutex_;
|
|
int read_ahead_policy_;
|
|
ChunkManagerPtr cm_;
|
|
MmapChunkManagerPtr mcm_;
|
|
ColumnTable columns_;
|
|
};
|
|
|
|
using ChunkCachePtr = std::shared_ptr<milvus::storage::ChunkCache>;
|
|
|
|
} // namespace milvus::storage
|