admin/milvus
1
0
Fork 0
mirror of https://gitee.com/milvus-io/milvus.git synced 2025-12-08 01:58:34 +08:00
Code Issues Packages Projects Releases Wiki Activity

Update vector engine

Browse Source 
Former-commit-id: 4297a3af82574e0f8c4843a6b26969b17fb13387
This commit is contained in:
jinhai 2019-04-10 21:06:48 +08:00
parent 0f307f819a
commit db3f092f16
1 changed files with 21 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

43
pyengine/engine/controller/vector_engine.py
View File

@ -68,8 +68,19 @@ class VectorEngine(object):
pass pass
@staticmethod @staticmethod
def AddVectorToExistFile(group_name): def BuildVectorIndex(group_name, raw_filename, dimension):
pass # Build index
raw_vector_array, raw_vector_id_array = VectorEngine.GetVectorListFromRawFile(group_name)
# create index
index_builder = build_index.FactoryIndex()
index = index_builder().build(dimension, raw_vector_array, raw_vector_id_array)
# TODO(jinhai): store index into Cache
index_filename = raw_filename + '_index'
serialize.write_index(file_name=index_filename, index=index)
UpdateFile(raw_filename, {'row_number': ROW_LIMIT, 'type': 'index', 'filename': index_filename})
@staticmethod @staticmethod
def AddVector(group_name, vectors): def AddVector(group_name, vectors):
@ -90,8 +101,10 @@ class VectorEngine(object):
current_raw_file = file current_raw_file = file
current_raw_row_number = file.row_number current_raw_row_number = file.row_number
print(raw_filename) print(raw_filename)
elif file.type == 'raw':
BuildVectorIndex(group_name, file.filename, group.dimension)
else: else:
print("---- To Build Index") pass
else: else:
pass pass
@ -114,35 +127,21 @@ class VectorEngine(object):
++ table_row_number ++ table_row_number
vector_str_list.append(group_name + '.' + str(vector_id)) vector_str_list.append(group_name + '.' + str(vector_id))
# Build index BuildVectorIndex(group_name, raw_filename, group.dimension)
raw_vector_array, raw_vector_id_array = VectorEngine.GetVectorListFromRawFile(group_name)
d = group.dimension
# create index
index_builder = build_index.FactoryIndex()
index = index_builder().build(d, raw_vector_array, raw_vector_id_array)
# TODO(jinhai): store index into Cache
index_filename = file.filename + '_index'
serialize.write_index(file_name=index_filename, index=index)
UpdateFile(file.filename, {'row_number': ROW_LIMIT, 'type': 'index', 'filename': index_filename})
# create new raw file name # create new raw file name
raw_filename = str(group.file_number + 1) raw_filename = str(group.file_number + 1)
table_row_number = 0 table_row_number = 0
current_raw_file = None
# update file table # If no raw file
MetaManager.CreateRawFile(group_name, raw_filename)
# Append vectors to raw file
if current_raw_file == None: if current_raw_file == None:
# update file table # update file table
MetaManager.CreateRawFile(group_name, raw_filename) MetaManager.CreateRawFile(group_name, raw_filename)
# 1. update db # 1. update db on file number and row number
new_group_file_number = group.file_number + 1 new_group_file_number = group.file_number + 1
new_group_row_number = int(group.row_number) + incoming_row_number new_group_row_number = int(group.row_number) + incoming_row_number - start_row_index
MetaManager.UpdateGroup(group_name, {'file_number': new_group_file_number, 'row_number': new_group_row_number}) MetaManager.UpdateGroup(group_name, {'file_number': new_group_file_number, 'row_number': new_group_row_number})
# 2. store vector into raw files # 2. store vector into raw files