From 17f1ab71bb760cad114846ed546e8b1b54111a54 Mon Sep 17 00:00:00 2001 From: XuanYang-cn Date: Fri, 27 Jun 2025 15:00:42 +0800 Subject: [PATCH] enhance: Remove not inused BuildIndexInfo (#42926) 1. removed not inuse cgo methods in index_c.h/cpp 2. removed indexcogowrapper/build_index_info.go See also: #39242 Signed-off-by: yangxuan --- internal/core/src/indexbuilder/index_c.cpp | 258 ------------------ internal/core/src/indexbuilder/index_c.h | 87 +----- internal/core/src/indexbuilder/type_c.h | 1 - .../util/indexcgowrapper/build_index_info.go | 204 -------------- 4 files changed, 13 insertions(+), 537 deletions(-) delete mode 100644 internal/util/indexcgowrapper/build_index_info.go diff --git a/internal/core/src/indexbuilder/index_c.cpp b/internal/core/src/indexbuilder/index_c.cpp index 420e53bf5d..65d70a254e 100644 --- a/internal/core/src/indexbuilder/index_c.cpp +++ b/internal/core/src/indexbuilder/index_c.cpp @@ -679,243 +679,6 @@ CleanLocalData(CIndex index) { return status; } -CStatus -NewBuildIndexInfo(CBuildIndexInfo* c_build_index_info, - CStorageConfig c_storage_config) { - try { - auto build_index_info = std::make_unique(); - auto& storage_config = build_index_info->storage_config; - storage_config.address = std::string(c_storage_config.address); - storage_config.bucket_name = std::string(c_storage_config.bucket_name); - storage_config.access_key_id = - std::string(c_storage_config.access_key_id); - storage_config.access_key_value = - std::string(c_storage_config.access_key_value); - storage_config.root_path = std::string(c_storage_config.root_path); - storage_config.storage_type = - std::string(c_storage_config.storage_type); - storage_config.cloud_provider = - std::string(c_storage_config.cloud_provider); - storage_config.iam_endpoint = - std::string(c_storage_config.iam_endpoint); - storage_config.cloud_provider = - std::string(c_storage_config.cloud_provider); - storage_config.useSSL = c_storage_config.useSSL; - storage_config.sslCACert = c_storage_config.sslCACert; - storage_config.useIAM = c_storage_config.useIAM; - storage_config.region = c_storage_config.region; - storage_config.useVirtualHost = c_storage_config.useVirtualHost; - storage_config.requestTimeoutMs = c_storage_config.requestTimeoutMs; - storage_config.gcp_credential_json = - std::string(c_storage_config.gcp_credential_json); - - *c_build_index_info = build_index_info.release(); - auto status = CStatus(); - status.error_code = Success; - status.error_msg = ""; - return status; - } catch (std::exception& e) { - auto status = CStatus(); - status.error_code = UnexpectedError; - status.error_msg = strdup(e.what()); - return status; - } -} - -void -DeleteBuildIndexInfo(CBuildIndexInfo c_build_index_info) { - auto info = (BuildIndexInfo*)c_build_index_info; - delete info; -} - -CStatus -AppendBuildIndexParam(CBuildIndexInfo c_build_index_info, - const uint8_t* serialized_index_params, - const uint64_t len) { - try { - auto build_index_info = (BuildIndexInfo*)c_build_index_info; - auto index_params = - std::make_unique(); - auto res = index_params->ParseFromArray(serialized_index_params, len); - AssertInfo(res, "Unmarshal index params failed"); - for (auto i = 0; i < index_params->params_size(); ++i) { - const auto& param = index_params->params(i); - build_index_info->config[param.key()] = param.value(); - } - - auto status = CStatus(); - status.error_code = Success; - status.error_msg = ""; - return status; - } catch (std::exception& e) { - auto status = CStatus(); - status.error_code = UnexpectedError; - status.error_msg = strdup(e.what()); - return status; - } -} - -CStatus -AppendBuildTypeParam(CBuildIndexInfo c_build_index_info, - const uint8_t* serialized_type_params, - const uint64_t len) { - try { - auto build_index_info = (BuildIndexInfo*)c_build_index_info; - auto type_params = - std::make_unique(); - auto res = type_params->ParseFromArray(serialized_type_params, len); - AssertInfo(res, "Unmarshal index build type params failed"); - for (auto i = 0; i < type_params->params_size(); ++i) { - const auto& param = type_params->params(i); - build_index_info->config[param.key()] = param.value(); - } - - auto status = CStatus(); - status.error_code = Success; - status.error_msg = ""; - return status; - } catch (std::exception& e) { - auto status = CStatus(); - status.error_code = UnexpectedError; - status.error_msg = strdup(e.what()); - return status; - } -} - -CStatus -AppendFieldMetaInfoV2(CBuildIndexInfo c_build_index_info, - int64_t collection_id, - int64_t partition_id, - int64_t segment_id, - int64_t field_id, - const char* field_name, - enum CDataType field_type, - int64_t dim) { - try { - auto build_index_info = (BuildIndexInfo*)c_build_index_info; - build_index_info->collection_id = collection_id; - build_index_info->partition_id = partition_id; - build_index_info->segment_id = segment_id; - build_index_info->field_id = field_id; - build_index_info->field_type = milvus::DataType(field_type); - build_index_info->field_name = field_name; - build_index_info->dim = dim; - - return milvus::SuccessCStatus(); - } catch (std::exception& e) { - return milvus::FailureCStatus(&e); - } -} - -CStatus -AppendFieldMetaInfo(CBuildIndexInfo c_build_index_info, - int64_t collection_id, - int64_t partition_id, - int64_t segment_id, - int64_t field_id, - enum CDataType field_type) { - try { - auto build_index_info = (BuildIndexInfo*)c_build_index_info; - build_index_info->collection_id = collection_id; - build_index_info->partition_id = partition_id; - build_index_info->segment_id = segment_id; - build_index_info->field_id = field_id; - - build_index_info->field_type = milvus::DataType(field_type); - - auto status = CStatus(); - status.error_code = Success; - status.error_msg = ""; - return status; - } catch (std::exception& e) { - auto status = CStatus(); - status.error_code = UnexpectedError; - status.error_msg = strdup(e.what()); - return status; - } -} - -CStatus -AppendIndexMetaInfo(CBuildIndexInfo c_build_index_info, - int64_t index_id, - int64_t build_id, - int64_t version) { - try { - auto build_index_info = (BuildIndexInfo*)c_build_index_info; - build_index_info->index_id = index_id; - build_index_info->index_build_id = build_id; - build_index_info->index_version = version; - - auto status = CStatus(); - status.error_code = Success; - status.error_msg = ""; - return status; - } catch (std::exception& e) { - auto status = CStatus(); - status.error_code = UnexpectedError; - status.error_msg = strdup(e.what()); - return status; - } -} - -CStatus -AppendInsertFilePath(CBuildIndexInfo c_build_index_info, - const char* c_file_path) { - try { - auto build_index_info = (BuildIndexInfo*)c_build_index_info; - std::string insert_file_path(c_file_path); - build_index_info->insert_files.emplace_back(insert_file_path); - - auto status = CStatus(); - status.error_code = Success; - status.error_msg = ""; - return status; - } catch (std::exception& e) { - return milvus::FailureCStatus(&e); - } -} - -CStatus -AppendIndexEngineVersionToBuildInfo(CBuildIndexInfo c_load_index_info, - int32_t index_engine_version) { - try { - auto build_index_info = (BuildIndexInfo*)c_load_index_info; - build_index_info->index_engine_version = index_engine_version; - - auto status = CStatus(); - status.error_code = Success; - status.error_msg = ""; - return status; - } catch (std::exception& e) { - return milvus::FailureCStatus(&e); - } -} - -CStatus -AppendIndexStorageInfo(CBuildIndexInfo c_build_index_info, - const char* c_data_store_path, - const char* c_index_store_path, - int64_t data_store_version) { - try { - auto build_index_info = (BuildIndexInfo*)c_build_index_info; - std::string data_store_path(c_data_store_path), - index_store_path(c_index_store_path); - build_index_info->data_store_path = data_store_path; - build_index_info->index_store_path = index_store_path; - build_index_info->data_store_version = data_store_version; - - auto status = CStatus(); - status.error_code = Success; - status.error_msg = ""; - return status; - } catch (std::exception& e) { - auto status = CStatus(); - status.error_code = UnexpectedError; - status.error_msg = strdup(e.what()); - return status; - } -} - CStatus SerializeIndexAndUpLoad(CIndex index, ProtoLayoutInterface result) { auto status = CStatus(); @@ -936,24 +699,3 @@ SerializeIndexAndUpLoad(CIndex index, ProtoLayoutInterface result) { } return status; } - -CStatus -AppendOptionalFieldDataPath(CBuildIndexInfo c_build_index_info, - const int64_t field_id, - const char* field_name, - const int32_t field_type, - const char* c_file_path) { - try { - auto build_index_info = (BuildIndexInfo*)c_build_index_info; - std::string field_name_str(field_name); - auto& opt_fields_map = build_index_info->opt_fields; - if (opt_fields_map.find(field_id) == opt_fields_map.end()) { - opt_fields_map[field_id] = { - field_name, static_cast(field_type), {}}; - } - std::get<2>(opt_fields_map[field_id]).emplace_back(c_file_path); - return CStatus{Success, ""}; - } catch (std::exception& e) { - return milvus::FailureCStatus(&e); - } -} diff --git a/internal/core/src/indexbuilder/index_c.h b/internal/core/src/indexbuilder/index_c.h index 3dea90028c..ce9b4910fe 100644 --- a/internal/core/src/indexbuilder/index_c.h +++ b/internal/core/src/indexbuilder/index_c.h @@ -21,13 +21,6 @@ extern "C" { #include "common/binary_set_c.h" #include "indexbuilder/type_c.h" -// used only in test -CStatus -CreateIndexV0(enum CDataType dtype, - const char* serialized_type_params, - const char* serialized_index_params, - CIndex* res_index); - CStatus CreateIndex(CIndex* res_index, const uint8_t* serialized_build_index_info, @@ -46,6 +39,19 @@ BuildTextIndex(ProtoLayoutInterface c_binary_set, const uint8_t* serialized_build_index_info, const uint64_t len); +CStatus +CleanLocalData(CIndex index); + +CStatus +SerializeIndexAndUpLoad(CIndex index, ProtoLayoutInterface result); + +// =========== Followings are used only in test ========== +CStatus +CreateIndexV0(enum CDataType dtype, + const char* serialized_type_params, + const char* serialized_index_params, + CIndex* res_index); + CStatus BuildFloatVecIndex(CIndex index, int64_t float_value_num, const float* vectors); @@ -81,73 +87,6 @@ SerializeIndexToBinarySet(CIndex index, CBinarySet* c_binary_set); CStatus LoadIndexFromBinarySet(CIndex index, CBinarySet c_binary_set); -CStatus -CleanLocalData(CIndex index); - -CStatus -NewBuildIndexInfo(CBuildIndexInfo* c_build_index_info, - CStorageConfig c_storage_config); - -void -DeleteBuildIndexInfo(CBuildIndexInfo c_build_index_info); - -CStatus -AppendBuildIndexParam(CBuildIndexInfo c_build_index_info, - const uint8_t* serialized_type_params, - const uint64_t len); - -CStatus -AppendBuildTypeParam(CBuildIndexInfo c_build_index_info, - const uint8_t* serialized_type_params, - const uint64_t len); - -CStatus -AppendFieldMetaInfo(CBuildIndexInfo c_build_index_info, - int64_t collection_id, - int64_t partition_id, - int64_t segment_id, - int64_t field_id, - enum CDataType field_type); - -CStatus -AppendFieldMetaInfoV2(CBuildIndexInfo c_build_index_info, - int64_t collection_id, - int64_t partition_id, - int64_t segment_id, - int64_t field_id, - const char* field_name, - enum CDataType field_type, - int64_t dim); - -CStatus -AppendIndexMetaInfo(CBuildIndexInfo c_build_index_info, - int64_t index_id, - int64_t build_id, - int64_t version); - -CStatus -AppendInsertFilePath(CBuildIndexInfo c_build_index_info, const char* file_path); - -CStatus -AppendIndexEngineVersionToBuildInfo(CBuildIndexInfo c_load_index_info, - int32_t c_index_engine_version); - -CStatus -AppendOptionalFieldDataPath(CBuildIndexInfo c_build_index_info, - const int64_t field_id, - const char* field_name, - const int32_t field_type, - const char* c_file_path); - -CStatus -SerializeIndexAndUpLoad(CIndex index, ProtoLayoutInterface result); - -CStatus -AppendIndexStorageInfo(CBuildIndexInfo c_build_index_info, - const char* c_data_store_path, - const char* c_index_store_path, - int64_t data_store_version); - #ifdef __cplusplus }; #endif diff --git a/internal/core/src/indexbuilder/type_c.h b/internal/core/src/indexbuilder/type_c.h index f0d07fb51d..61001a8775 100644 --- a/internal/core/src/indexbuilder/type_c.h +++ b/internal/core/src/indexbuilder/type_c.h @@ -15,4 +15,3 @@ typedef void* CIndex; typedef void* CIndexQueryResult; -typedef void* CBuildIndexInfo; diff --git a/internal/util/indexcgowrapper/build_index_info.go b/internal/util/indexcgowrapper/build_index_info.go deleted file mode 100644 index 025458673b..0000000000 --- a/internal/util/indexcgowrapper/build_index_info.go +++ /dev/null @@ -1,204 +0,0 @@ -// Licensed to the LF AI & Data foundation under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package indexcgowrapper - -/* -#cgo pkg-config: milvus_core -#include // free -#include "indexbuilder/index_c.h" -*/ -import "C" - -import ( - "fmt" - "unsafe" - - "google.golang.org/protobuf/proto" - - "github.com/milvus-io/milvus-proto/go-api/v2/commonpb" - "github.com/milvus-io/milvus-proto/go-api/v2/schemapb" - "github.com/milvus-io/milvus/pkg/v2/proto/indexcgopb" - "github.com/milvus-io/milvus/pkg/v2/proto/indexpb" -) - -type BuildIndexInfo struct { - cBuildIndexInfo C.CBuildIndexInfo -} - -func NewBuildIndexInfo(config *indexpb.StorageConfig) (*BuildIndexInfo, error) { - var cBuildIndexInfo C.CBuildIndexInfo - - cAddress := C.CString(config.Address) - cBucketName := C.CString(config.BucketName) - cAccessKey := C.CString(config.AccessKeyID) - cAccessValue := C.CString(config.SecretAccessKey) - cRootPath := C.CString(config.RootPath) - cStorageType := C.CString(config.StorageType) - cIamEndPoint := C.CString(config.IAMEndpoint) - cRegion := C.CString(config.Region) - cCloudProvider := C.CString(config.CloudProvider) - cSslCACert := C.CString(config.SslCACert) - cGcpCredentialJSON := C.CString(config.GcpCredentialJSON) - defer C.free(unsafe.Pointer(cAddress)) - defer C.free(unsafe.Pointer(cBucketName)) - defer C.free(unsafe.Pointer(cAccessKey)) - defer C.free(unsafe.Pointer(cAccessValue)) - defer C.free(unsafe.Pointer(cRootPath)) - defer C.free(unsafe.Pointer(cStorageType)) - defer C.free(unsafe.Pointer(cIamEndPoint)) - defer C.free(unsafe.Pointer(cRegion)) - defer C.free(unsafe.Pointer(cCloudProvider)) - defer C.free(unsafe.Pointer(cSslCACert)) - defer C.free(unsafe.Pointer(cGcpCredentialJSON)) - storageConfig := C.CStorageConfig{ - address: cAddress, - bucket_name: cBucketName, - access_key_id: cAccessKey, - access_key_value: cAccessValue, - root_path: cRootPath, - storage_type: cStorageType, - iam_endpoint: cIamEndPoint, - cloud_provider: cCloudProvider, - useSSL: C.bool(config.UseSSL), - sslCACert: cSslCACert, - useIAM: C.bool(config.UseIAM), - region: cRegion, - useVirtualHost: C.bool(config.UseVirtualHost), - requestTimeoutMs: C.int64_t(config.RequestTimeoutMs), - gcp_credential_json: cGcpCredentialJSON, - } - - status := C.NewBuildIndexInfo(&cBuildIndexInfo, storageConfig) - if err := HandleCStatus(&status, "NewBuildIndexInfo failed"); err != nil { - return nil, err - } - return &BuildIndexInfo{cBuildIndexInfo: cBuildIndexInfo}, nil -} - -func DeleteBuildIndexInfo(info *BuildIndexInfo) { - C.DeleteBuildIndexInfo(info.cBuildIndexInfo) -} - -func (bi *BuildIndexInfo) AppendFieldMetaInfo(collectionID int64, partitionID int64, segmentID int64, fieldID int64, fieldType schemapb.DataType) error { - cColID := C.int64_t(collectionID) - cParID := C.int64_t(partitionID) - cSegID := C.int64_t(segmentID) - cFieldID := C.int64_t(fieldID) - cintDType := uint32(fieldType) - status := C.AppendFieldMetaInfo(bi.cBuildIndexInfo, cColID, cParID, cSegID, cFieldID, cintDType) - return HandleCStatus(&status, "appendFieldMetaInfo failed") -} - -func (bi *BuildIndexInfo) AppendFieldMetaInfoV2(collectionID int64, partitionID int64, segmentID int64, fieldID int64, fieldType schemapb.DataType, fieldName string, dim int64) error { - cColID := C.int64_t(collectionID) - cParID := C.int64_t(partitionID) - cSegID := C.int64_t(segmentID) - cFieldID := C.int64_t(fieldID) - cintDType := uint32(fieldType) - cFieldName := C.CString(fieldName) - cDim := C.int64_t(dim) - defer C.free(unsafe.Pointer(cFieldName)) - status := C.AppendFieldMetaInfoV2(bi.cBuildIndexInfo, cColID, cParID, cSegID, cFieldID, cFieldName, cintDType, cDim) - return HandleCStatus(&status, "appendFieldMetaInfo failed") -} - -func (bi *BuildIndexInfo) AppendIndexMetaInfo(indexID int64, buildID int64, indexVersion int64) error { - cIndexID := C.int64_t(indexID) - cBuildID := C.int64_t(buildID) - cIndexVersion := C.int64_t(indexVersion) - - status := C.AppendIndexMetaInfo(bi.cBuildIndexInfo, cIndexID, cBuildID, cIndexVersion) - return HandleCStatus(&status, "appendIndexMetaInfo failed") -} - -func (bi *BuildIndexInfo) AppendIndexStorageInfo(dataStorePath, indexStorePath string, dataStoreVersion int64) error { - cDataStorePath := C.CString(dataStorePath) - defer C.free(unsafe.Pointer(cDataStorePath)) - cIndexStorePath := C.CString(indexStorePath) - defer C.free(unsafe.Pointer(cIndexStorePath)) - cVersion := C.int64_t(dataStoreVersion) - status := C.AppendIndexStorageInfo(bi.cBuildIndexInfo, cDataStorePath, cIndexStorePath, cVersion) - return HandleCStatus(&status, "appendIndexStorageInfo failed") -} - -func (bi *BuildIndexInfo) AppendBuildIndexParam(indexParams map[string]string) error { - if len(indexParams) == 0 { - return nil - } - protoIndexParams := &indexcgopb.IndexParams{ - Params: make([]*commonpb.KeyValuePair, 0), - } - for key, value := range indexParams { - protoIndexParams.Params = append(protoIndexParams.Params, &commonpb.KeyValuePair{Key: key, Value: value}) - } - indexParamsBlob, err := proto.Marshal(protoIndexParams) - if err != nil { - return fmt.Errorf("failed to marshal index params: %s", err) - } - - status := C.AppendBuildIndexParam(bi.cBuildIndexInfo, (*C.uint8_t)(unsafe.Pointer(&indexParamsBlob[0])), (C.uint64_t)(len(indexParamsBlob))) - return HandleCStatus(&status, "appendBuildIndexParam failed") -} - -func (bi *BuildIndexInfo) AppendBuildTypeParam(typeParams map[string]string) error { - if len(typeParams) == 0 { - return nil - } - protoTypeParams := &indexcgopb.TypeParams{ - Params: make([]*commonpb.KeyValuePair, 0), - } - for key, value := range typeParams { - protoTypeParams.Params = append(protoTypeParams.Params, &commonpb.KeyValuePair{Key: key, Value: value}) - } - typeParamsBlob, err := proto.Marshal(protoTypeParams) - if err != nil { - return fmt.Errorf("failed to marshal type params: %s", err) - } - - status := C.AppendBuildTypeParam(bi.cBuildIndexInfo, (*C.uint8_t)(unsafe.Pointer(&typeParamsBlob[0])), (C.uint64_t)(len(typeParamsBlob))) - return HandleCStatus(&status, "appendBuildTypeParam failed") -} - -func (bi *BuildIndexInfo) AppendInsertFile(filePath string) error { - cInsertFilePath := C.CString(filePath) - defer C.free(unsafe.Pointer(cInsertFilePath)) - - status := C.AppendInsertFilePath(bi.cBuildIndexInfo, cInsertFilePath) - return HandleCStatus(&status, "appendInsertFile failed") -} - -func (bi *BuildIndexInfo) AppendIndexEngineVersion(indexEngineVersion int32) error { - cIndexEngineVersion := C.int32_t(indexEngineVersion) - - status := C.AppendIndexEngineVersionToBuildInfo(bi.cBuildIndexInfo, cIndexEngineVersion) - return HandleCStatus(&status, "AppendIndexEngineVersion failed") -} - -func (bi *BuildIndexInfo) AppendOptionalField(optField *indexpb.OptionalFieldInfo) error { - cFieldId := C.int64_t(optField.GetFieldID()) - cFieldType := C.int32_t(optField.GetFieldType()) - cFieldName := C.CString(optField.GetFieldName()) - for _, dataPath := range optField.GetDataPaths() { - cDataPath := C.CString(dataPath) - defer C.free(unsafe.Pointer(cDataPath)) - status := C.AppendOptionalFieldDataPath(bi.cBuildIndexInfo, cFieldId, cFieldName, cFieldType, cDataPath) - if err := HandleCStatus(&status, "AppendOptionalFieldDataPath failed"); err != nil { - return err - } - } - return nil -}