admin/milvus
1
0
Fork 0
mirror of https://gitee.com/milvus-io/milvus.git synced 2025-12-08 01:58:34 +08:00
Code Issues Packages Projects Releases Wiki Activity

[skip e2e] Fix load meta migration (#21584)

Browse Source 
Signed-off-by: longjiquan <jiquan.long@zilliz.com>
This commit is contained in:
Jiquan Long 2023-01-11 19:31:39 +08:00 committed by GitHub
parent 39ed627e96
commit 6d09bbed68
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 24 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
Makefile
View File

@ -96,7 +96,10 @@ binlog:
MIGRATION_PATH = $(PWD)/cmd/tools/migration MIGRATION_PATH = $(PWD)/cmd/tools/migration
meta-migration: meta-migration:
@echo "Building migration tool ..." @echo "Building migration tool ..."
@mkdir -p $(INSTALL_PATH) && go env -w CGO_ENABLED="1" && GO111MODULE=on $(GO) build -o $(INSTALL_PATH)/meta-migration $(MIGRATION_PATH)/main.go 1>/dev/null @source $(PWD)/scripts/setenv.sh && \
mkdir -p $(INSTALL_PATH) && go env -w CGO_ENABLED="1" && \
GO111MODULE=on $(GO) build -ldflags="-r $${RPATH} -X '$(OBJPREFIX).BuildTags=$(BUILD_TAGS)' -X '$(OBJPREFIX).BuildTime=$(BUILD_TIME)' -X '$(OBJPREFIX).GitCommit=$(GIT_COMMIT)' -X '$(OBJPREFIX).GoVersion=$(GO_VERSION)'" \
${APPLE_SILICON_FLAG} -o $(INSTALL_PATH)/meta-migration $(MIGRATION_PATH)/main.go 1>/dev/null
BUILD_TAGS = $(shell git describe --tags --always --dirty="-dev") BUILD_TAGS = $(shell git describe --tags --always --dirty="-dev")
BUILD_TIME = $(shell date -u) BUILD_TIME = $(shell date -u)

17
cmd/tools/migration/meta/210_to_220.go
View File

@ -130,6 +130,7 @@ func (meta *CollectionLoadInfo210) to220() (CollectionLoadInfo220, PartitionLoad
LoadPercentage: 100, LoadPercentage: 100,
Status: querypb.LoadStatus_Loaded, Status: querypb.LoadStatus_Loaded,
ReplicaNumber: loadInfo.ReplicaNumber, ReplicaNumber: loadInfo.ReplicaNumber,
FieldIndexID: make(map[UniqueID]UniqueID),
} }
} }
} }
@ -274,11 +275,13 @@ func combineToSegmentIndexesMeta220(segmentIndexes SegmentIndexesMeta210, indexB
} }
func combineToLoadInfo220(collectionLoadInfo CollectionLoadInfo220, partitionLoadInto PartitionLoadInfo220, fieldIndexes FieldIndexes210) { func combineToLoadInfo220(collectionLoadInfo CollectionLoadInfo220, partitionLoadInto PartitionLoadInfo220, fieldIndexes FieldIndexes210) {
toBeReleased := make([]UniqueID, 0)
for collectionID, loadInfo := range collectionLoadInfo { for collectionID, loadInfo := range collectionLoadInfo {
indexes, ok := fieldIndexes[collectionID] indexes, ok := fieldIndexes[collectionID]
if !ok || len(indexes.indexes) == 0 { if !ok || len(indexes.indexes) == 0 {
log.Warn("release the collection without index", zap.Int64("collectionID", collectionID)) toBeReleased = append(toBeReleased, collectionID)
delete(collectionLoadInfo, collectionID) continue
} }
for _, index := range indexes.indexes { for _, index := range indexes.indexes {
@ -289,15 +292,21 @@ func combineToLoadInfo220(collectionLoadInfo CollectionLoadInfo220, partitionLoa
for collectionID, partitions := range partitionLoadInto { for collectionID, partitions := range partitionLoadInto {
indexes, ok := fieldIndexes[collectionID] indexes, ok := fieldIndexes[collectionID]
if !ok || len(indexes.indexes) == 0 { if !ok || len(indexes.indexes) == 0 {
log.Warn("release the collection without index", zap.Int64("collectionID", collectionID)) toBeReleased = append(toBeReleased, collectionID)
delete(collectionLoadInfo, collectionID) continue
} }
for _, loadInfo := range partitions { for _, loadInfo := range partitions {
for _, index := range indexes.indexes { for _, index := range indexes.indexes {
loadInfo.FieldIndexID[index.GetFiledID()] = index.GetIndexID() loadInfo.FieldIndexID[index.GetFiledID()] = index.GetIndexID()
} }
} }
} }
for _, collectionID := range toBeReleased {
log.Warn("release the collection without index", zap.Int64("collectionID", collectionID))
delete(collectionLoadInfo, collectionID)
}
} }
func From210To220(metas *Meta) (*Meta, error) { func From210To220(metas *Meta) (*Meta, error) {

3
internal/util/sessionutil/session_util.go
View File

@ -241,11 +241,14 @@ func (s *Session) Init(serverName, address string, exclusive bool, triggerKill b
s.Exclusive = exclusive s.Exclusive = exclusive
s.TriggerKill = triggerKill s.TriggerKill = triggerKill
s.checkIDExist() s.checkIDExist()
// TO AVOID PANIC IN MIGRATION SCRIPT.
if !s.useCustomConfig {
serverID, err := s.getServerID() serverID, err := s.getServerID()
if err != nil { if err != nil {
panic(err) panic(err)
} }
s.ServerID = serverID s.ServerID = serverID
}
log.Info("start server", zap.String("name", serverName), zap.String("address", address), zap.Int64("id", s.ServerID)) log.Info("start server", zap.String("name", serverName), zap.String("address", address), zap.Int64("id", s.ServerID))
} }