enhance: Remove not inused BuildIndexInfo (#42926)

1. removed not inuse cgo methods in index_c.h/cpp
2. removed indexcogowrapper/build_index_info.go

See also: #39242

Signed-off-by: yangxuan <xuan.yang@zilliz.com>
This commit is contained in:
XuanYang-cn 2025-06-27 15:00:42 +08:00 committed by GitHub
parent 9b06ecb72f
commit 17f1ab71bb
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
4 changed files with 13 additions and 537 deletions

View File

@ -679,243 +679,6 @@ CleanLocalData(CIndex index) {
return status;
}
CStatus
NewBuildIndexInfo(CBuildIndexInfo* c_build_index_info,
CStorageConfig c_storage_config) {
try {
auto build_index_info = std::make_unique<BuildIndexInfo>();
auto& storage_config = build_index_info->storage_config;
storage_config.address = std::string(c_storage_config.address);
storage_config.bucket_name = std::string(c_storage_config.bucket_name);
storage_config.access_key_id =
std::string(c_storage_config.access_key_id);
storage_config.access_key_value =
std::string(c_storage_config.access_key_value);
storage_config.root_path = std::string(c_storage_config.root_path);
storage_config.storage_type =
std::string(c_storage_config.storage_type);
storage_config.cloud_provider =
std::string(c_storage_config.cloud_provider);
storage_config.iam_endpoint =
std::string(c_storage_config.iam_endpoint);
storage_config.cloud_provider =
std::string(c_storage_config.cloud_provider);
storage_config.useSSL = c_storage_config.useSSL;
storage_config.sslCACert = c_storage_config.sslCACert;
storage_config.useIAM = c_storage_config.useIAM;
storage_config.region = c_storage_config.region;
storage_config.useVirtualHost = c_storage_config.useVirtualHost;
storage_config.requestTimeoutMs = c_storage_config.requestTimeoutMs;
storage_config.gcp_credential_json =
std::string(c_storage_config.gcp_credential_json);
*c_build_index_info = build_index_info.release();
auto status = CStatus();
status.error_code = Success;
status.error_msg = "";
return status;
} catch (std::exception& e) {
auto status = CStatus();
status.error_code = UnexpectedError;
status.error_msg = strdup(e.what());
return status;
}
}
void
DeleteBuildIndexInfo(CBuildIndexInfo c_build_index_info) {
auto info = (BuildIndexInfo*)c_build_index_info;
delete info;
}
CStatus
AppendBuildIndexParam(CBuildIndexInfo c_build_index_info,
const uint8_t* serialized_index_params,
const uint64_t len) {
try {
auto build_index_info = (BuildIndexInfo*)c_build_index_info;
auto index_params =
std::make_unique<milvus::proto::indexcgo::IndexParams>();
auto res = index_params->ParseFromArray(serialized_index_params, len);
AssertInfo(res, "Unmarshal index params failed");
for (auto i = 0; i < index_params->params_size(); ++i) {
const auto& param = index_params->params(i);
build_index_info->config[param.key()] = param.value();
}
auto status = CStatus();
status.error_code = Success;
status.error_msg = "";
return status;
} catch (std::exception& e) {
auto status = CStatus();
status.error_code = UnexpectedError;
status.error_msg = strdup(e.what());
return status;
}
}
CStatus
AppendBuildTypeParam(CBuildIndexInfo c_build_index_info,
const uint8_t* serialized_type_params,
const uint64_t len) {
try {
auto build_index_info = (BuildIndexInfo*)c_build_index_info;
auto type_params =
std::make_unique<milvus::proto::indexcgo::TypeParams>();
auto res = type_params->ParseFromArray(serialized_type_params, len);
AssertInfo(res, "Unmarshal index build type params failed");
for (auto i = 0; i < type_params->params_size(); ++i) {
const auto& param = type_params->params(i);
build_index_info->config[param.key()] = param.value();
}
auto status = CStatus();
status.error_code = Success;
status.error_msg = "";
return status;
} catch (std::exception& e) {
auto status = CStatus();
status.error_code = UnexpectedError;
status.error_msg = strdup(e.what());
return status;
}
}
CStatus
AppendFieldMetaInfoV2(CBuildIndexInfo c_build_index_info,
int64_t collection_id,
int64_t partition_id,
int64_t segment_id,
int64_t field_id,
const char* field_name,
enum CDataType field_type,
int64_t dim) {
try {
auto build_index_info = (BuildIndexInfo*)c_build_index_info;
build_index_info->collection_id = collection_id;
build_index_info->partition_id = partition_id;
build_index_info->segment_id = segment_id;
build_index_info->field_id = field_id;
build_index_info->field_type = milvus::DataType(field_type);
build_index_info->field_name = field_name;
build_index_info->dim = dim;
return milvus::SuccessCStatus();
} catch (std::exception& e) {
return milvus::FailureCStatus(&e);
}
}
CStatus
AppendFieldMetaInfo(CBuildIndexInfo c_build_index_info,
int64_t collection_id,
int64_t partition_id,
int64_t segment_id,
int64_t field_id,
enum CDataType field_type) {
try {
auto build_index_info = (BuildIndexInfo*)c_build_index_info;
build_index_info->collection_id = collection_id;
build_index_info->partition_id = partition_id;
build_index_info->segment_id = segment_id;
build_index_info->field_id = field_id;
build_index_info->field_type = milvus::DataType(field_type);
auto status = CStatus();
status.error_code = Success;
status.error_msg = "";
return status;
} catch (std::exception& e) {
auto status = CStatus();
status.error_code = UnexpectedError;
status.error_msg = strdup(e.what());
return status;
}
}
CStatus
AppendIndexMetaInfo(CBuildIndexInfo c_build_index_info,
int64_t index_id,
int64_t build_id,
int64_t version) {
try {
auto build_index_info = (BuildIndexInfo*)c_build_index_info;
build_index_info->index_id = index_id;
build_index_info->index_build_id = build_id;
build_index_info->index_version = version;
auto status = CStatus();
status.error_code = Success;
status.error_msg = "";
return status;
} catch (std::exception& e) {
auto status = CStatus();
status.error_code = UnexpectedError;
status.error_msg = strdup(e.what());
return status;
}
}
CStatus
AppendInsertFilePath(CBuildIndexInfo c_build_index_info,
const char* c_file_path) {
try {
auto build_index_info = (BuildIndexInfo*)c_build_index_info;
std::string insert_file_path(c_file_path);
build_index_info->insert_files.emplace_back(insert_file_path);
auto status = CStatus();
status.error_code = Success;
status.error_msg = "";
return status;
} catch (std::exception& e) {
return milvus::FailureCStatus(&e);
}
}
CStatus
AppendIndexEngineVersionToBuildInfo(CBuildIndexInfo c_load_index_info,
int32_t index_engine_version) {
try {
auto build_index_info = (BuildIndexInfo*)c_load_index_info;
build_index_info->index_engine_version = index_engine_version;
auto status = CStatus();
status.error_code = Success;
status.error_msg = "";
return status;
} catch (std::exception& e) {
return milvus::FailureCStatus(&e);
}
}
CStatus
AppendIndexStorageInfo(CBuildIndexInfo c_build_index_info,
const char* c_data_store_path,
const char* c_index_store_path,
int64_t data_store_version) {
try {
auto build_index_info = (BuildIndexInfo*)c_build_index_info;
std::string data_store_path(c_data_store_path),
index_store_path(c_index_store_path);
build_index_info->data_store_path = data_store_path;
build_index_info->index_store_path = index_store_path;
build_index_info->data_store_version = data_store_version;
auto status = CStatus();
status.error_code = Success;
status.error_msg = "";
return status;
} catch (std::exception& e) {
auto status = CStatus();
status.error_code = UnexpectedError;
status.error_msg = strdup(e.what());
return status;
}
}
CStatus
SerializeIndexAndUpLoad(CIndex index, ProtoLayoutInterface result) {
auto status = CStatus();
@ -936,24 +699,3 @@ SerializeIndexAndUpLoad(CIndex index, ProtoLayoutInterface result) {
}
return status;
}
CStatus
AppendOptionalFieldDataPath(CBuildIndexInfo c_build_index_info,
const int64_t field_id,
const char* field_name,
const int32_t field_type,
const char* c_file_path) {
try {
auto build_index_info = (BuildIndexInfo*)c_build_index_info;
std::string field_name_str(field_name);
auto& opt_fields_map = build_index_info->opt_fields;
if (opt_fields_map.find(field_id) == opt_fields_map.end()) {
opt_fields_map[field_id] = {
field_name, static_cast<milvus::DataType>(field_type), {}};
}
std::get<2>(opt_fields_map[field_id]).emplace_back(c_file_path);
return CStatus{Success, ""};
} catch (std::exception& e) {
return milvus::FailureCStatus(&e);
}
}

View File

@ -21,13 +21,6 @@ extern "C" {
#include "common/binary_set_c.h"
#include "indexbuilder/type_c.h"
// used only in test
CStatus
CreateIndexV0(enum CDataType dtype,
const char* serialized_type_params,
const char* serialized_index_params,
CIndex* res_index);
CStatus
CreateIndex(CIndex* res_index,
const uint8_t* serialized_build_index_info,
@ -46,6 +39,19 @@ BuildTextIndex(ProtoLayoutInterface c_binary_set,
const uint8_t* serialized_build_index_info,
const uint64_t len);
CStatus
CleanLocalData(CIndex index);
CStatus
SerializeIndexAndUpLoad(CIndex index, ProtoLayoutInterface result);
// =========== Followings are used only in test ==========
CStatus
CreateIndexV0(enum CDataType dtype,
const char* serialized_type_params,
const char* serialized_index_params,
CIndex* res_index);
CStatus
BuildFloatVecIndex(CIndex index, int64_t float_value_num, const float* vectors);
@ -81,73 +87,6 @@ SerializeIndexToBinarySet(CIndex index, CBinarySet* c_binary_set);
CStatus
LoadIndexFromBinarySet(CIndex index, CBinarySet c_binary_set);
CStatus
CleanLocalData(CIndex index);
CStatus
NewBuildIndexInfo(CBuildIndexInfo* c_build_index_info,
CStorageConfig c_storage_config);
void
DeleteBuildIndexInfo(CBuildIndexInfo c_build_index_info);
CStatus
AppendBuildIndexParam(CBuildIndexInfo c_build_index_info,
const uint8_t* serialized_type_params,
const uint64_t len);
CStatus
AppendBuildTypeParam(CBuildIndexInfo c_build_index_info,
const uint8_t* serialized_type_params,
const uint64_t len);
CStatus
AppendFieldMetaInfo(CBuildIndexInfo c_build_index_info,
int64_t collection_id,
int64_t partition_id,
int64_t segment_id,
int64_t field_id,
enum CDataType field_type);
CStatus
AppendFieldMetaInfoV2(CBuildIndexInfo c_build_index_info,
int64_t collection_id,
int64_t partition_id,
int64_t segment_id,
int64_t field_id,
const char* field_name,
enum CDataType field_type,
int64_t dim);
CStatus
AppendIndexMetaInfo(CBuildIndexInfo c_build_index_info,
int64_t index_id,
int64_t build_id,
int64_t version);
CStatus
AppendInsertFilePath(CBuildIndexInfo c_build_index_info, const char* file_path);
CStatus
AppendIndexEngineVersionToBuildInfo(CBuildIndexInfo c_load_index_info,
int32_t c_index_engine_version);
CStatus
AppendOptionalFieldDataPath(CBuildIndexInfo c_build_index_info,
const int64_t field_id,
const char* field_name,
const int32_t field_type,
const char* c_file_path);
CStatus
SerializeIndexAndUpLoad(CIndex index, ProtoLayoutInterface result);
CStatus
AppendIndexStorageInfo(CBuildIndexInfo c_build_index_info,
const char* c_data_store_path,
const char* c_index_store_path,
int64_t data_store_version);
#ifdef __cplusplus
};
#endif

View File

@ -15,4 +15,3 @@
typedef void* CIndex;
typedef void* CIndexQueryResult;
typedef void* CBuildIndexInfo;

View File

@ -1,204 +0,0 @@
// Licensed to the LF AI & Data foundation under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package indexcgowrapper
/*
#cgo pkg-config: milvus_core
#include <stdlib.h> // free
#include "indexbuilder/index_c.h"
*/
import "C"
import (
"fmt"
"unsafe"
"google.golang.org/protobuf/proto"
"github.com/milvus-io/milvus-proto/go-api/v2/commonpb"
"github.com/milvus-io/milvus-proto/go-api/v2/schemapb"
"github.com/milvus-io/milvus/pkg/v2/proto/indexcgopb"
"github.com/milvus-io/milvus/pkg/v2/proto/indexpb"
)
type BuildIndexInfo struct {
cBuildIndexInfo C.CBuildIndexInfo
}
func NewBuildIndexInfo(config *indexpb.StorageConfig) (*BuildIndexInfo, error) {
var cBuildIndexInfo C.CBuildIndexInfo
cAddress := C.CString(config.Address)
cBucketName := C.CString(config.BucketName)
cAccessKey := C.CString(config.AccessKeyID)
cAccessValue := C.CString(config.SecretAccessKey)
cRootPath := C.CString(config.RootPath)
cStorageType := C.CString(config.StorageType)
cIamEndPoint := C.CString(config.IAMEndpoint)
cRegion := C.CString(config.Region)
cCloudProvider := C.CString(config.CloudProvider)
cSslCACert := C.CString(config.SslCACert)
cGcpCredentialJSON := C.CString(config.GcpCredentialJSON)
defer C.free(unsafe.Pointer(cAddress))
defer C.free(unsafe.Pointer(cBucketName))
defer C.free(unsafe.Pointer(cAccessKey))
defer C.free(unsafe.Pointer(cAccessValue))
defer C.free(unsafe.Pointer(cRootPath))
defer C.free(unsafe.Pointer(cStorageType))
defer C.free(unsafe.Pointer(cIamEndPoint))
defer C.free(unsafe.Pointer(cRegion))
defer C.free(unsafe.Pointer(cCloudProvider))
defer C.free(unsafe.Pointer(cSslCACert))
defer C.free(unsafe.Pointer(cGcpCredentialJSON))
storageConfig := C.CStorageConfig{
address: cAddress,
bucket_name: cBucketName,
access_key_id: cAccessKey,
access_key_value: cAccessValue,
root_path: cRootPath,
storage_type: cStorageType,
iam_endpoint: cIamEndPoint,
cloud_provider: cCloudProvider,
useSSL: C.bool(config.UseSSL),
sslCACert: cSslCACert,
useIAM: C.bool(config.UseIAM),
region: cRegion,
useVirtualHost: C.bool(config.UseVirtualHost),
requestTimeoutMs: C.int64_t(config.RequestTimeoutMs),
gcp_credential_json: cGcpCredentialJSON,
}
status := C.NewBuildIndexInfo(&cBuildIndexInfo, storageConfig)
if err := HandleCStatus(&status, "NewBuildIndexInfo failed"); err != nil {
return nil, err
}
return &BuildIndexInfo{cBuildIndexInfo: cBuildIndexInfo}, nil
}
func DeleteBuildIndexInfo(info *BuildIndexInfo) {
C.DeleteBuildIndexInfo(info.cBuildIndexInfo)
}
func (bi *BuildIndexInfo) AppendFieldMetaInfo(collectionID int64, partitionID int64, segmentID int64, fieldID int64, fieldType schemapb.DataType) error {
cColID := C.int64_t(collectionID)
cParID := C.int64_t(partitionID)
cSegID := C.int64_t(segmentID)
cFieldID := C.int64_t(fieldID)
cintDType := uint32(fieldType)
status := C.AppendFieldMetaInfo(bi.cBuildIndexInfo, cColID, cParID, cSegID, cFieldID, cintDType)
return HandleCStatus(&status, "appendFieldMetaInfo failed")
}
func (bi *BuildIndexInfo) AppendFieldMetaInfoV2(collectionID int64, partitionID int64, segmentID int64, fieldID int64, fieldType schemapb.DataType, fieldName string, dim int64) error {
cColID := C.int64_t(collectionID)
cParID := C.int64_t(partitionID)
cSegID := C.int64_t(segmentID)
cFieldID := C.int64_t(fieldID)
cintDType := uint32(fieldType)
cFieldName := C.CString(fieldName)
cDim := C.int64_t(dim)
defer C.free(unsafe.Pointer(cFieldName))
status := C.AppendFieldMetaInfoV2(bi.cBuildIndexInfo, cColID, cParID, cSegID, cFieldID, cFieldName, cintDType, cDim)
return HandleCStatus(&status, "appendFieldMetaInfo failed")
}
func (bi *BuildIndexInfo) AppendIndexMetaInfo(indexID int64, buildID int64, indexVersion int64) error {
cIndexID := C.int64_t(indexID)
cBuildID := C.int64_t(buildID)
cIndexVersion := C.int64_t(indexVersion)
status := C.AppendIndexMetaInfo(bi.cBuildIndexInfo, cIndexID, cBuildID, cIndexVersion)
return HandleCStatus(&status, "appendIndexMetaInfo failed")
}
func (bi *BuildIndexInfo) AppendIndexStorageInfo(dataStorePath, indexStorePath string, dataStoreVersion int64) error {
cDataStorePath := C.CString(dataStorePath)
defer C.free(unsafe.Pointer(cDataStorePath))
cIndexStorePath := C.CString(indexStorePath)
defer C.free(unsafe.Pointer(cIndexStorePath))
cVersion := C.int64_t(dataStoreVersion)
status := C.AppendIndexStorageInfo(bi.cBuildIndexInfo, cDataStorePath, cIndexStorePath, cVersion)
return HandleCStatus(&status, "appendIndexStorageInfo failed")
}
func (bi *BuildIndexInfo) AppendBuildIndexParam(indexParams map[string]string) error {
if len(indexParams) == 0 {
return nil
}
protoIndexParams := &indexcgopb.IndexParams{
Params: make([]*commonpb.KeyValuePair, 0),
}
for key, value := range indexParams {
protoIndexParams.Params = append(protoIndexParams.Params, &commonpb.KeyValuePair{Key: key, Value: value})
}
indexParamsBlob, err := proto.Marshal(protoIndexParams)
if err != nil {
return fmt.Errorf("failed to marshal index params: %s", err)
}
status := C.AppendBuildIndexParam(bi.cBuildIndexInfo, (*C.uint8_t)(unsafe.Pointer(&indexParamsBlob[0])), (C.uint64_t)(len(indexParamsBlob)))
return HandleCStatus(&status, "appendBuildIndexParam failed")
}
func (bi *BuildIndexInfo) AppendBuildTypeParam(typeParams map[string]string) error {
if len(typeParams) == 0 {
return nil
}
protoTypeParams := &indexcgopb.TypeParams{
Params: make([]*commonpb.KeyValuePair, 0),
}
for key, value := range typeParams {
protoTypeParams.Params = append(protoTypeParams.Params, &commonpb.KeyValuePair{Key: key, Value: value})
}
typeParamsBlob, err := proto.Marshal(protoTypeParams)
if err != nil {
return fmt.Errorf("failed to marshal type params: %s", err)
}
status := C.AppendBuildTypeParam(bi.cBuildIndexInfo, (*C.uint8_t)(unsafe.Pointer(&typeParamsBlob[0])), (C.uint64_t)(len(typeParamsBlob)))
return HandleCStatus(&status, "appendBuildTypeParam failed")
}
func (bi *BuildIndexInfo) AppendInsertFile(filePath string) error {
cInsertFilePath := C.CString(filePath)
defer C.free(unsafe.Pointer(cInsertFilePath))
status := C.AppendInsertFilePath(bi.cBuildIndexInfo, cInsertFilePath)
return HandleCStatus(&status, "appendInsertFile failed")
}
func (bi *BuildIndexInfo) AppendIndexEngineVersion(indexEngineVersion int32) error {
cIndexEngineVersion := C.int32_t(indexEngineVersion)
status := C.AppendIndexEngineVersionToBuildInfo(bi.cBuildIndexInfo, cIndexEngineVersion)
return HandleCStatus(&status, "AppendIndexEngineVersion failed")
}
func (bi *BuildIndexInfo) AppendOptionalField(optField *indexpb.OptionalFieldInfo) error {
cFieldId := C.int64_t(optField.GetFieldID())
cFieldType := C.int32_t(optField.GetFieldType())
cFieldName := C.CString(optField.GetFieldName())
for _, dataPath := range optField.GetDataPaths() {
cDataPath := C.CString(dataPath)
defer C.free(unsafe.Pointer(cDataPath))
status := C.AppendOptionalFieldDataPath(bi.cBuildIndexInfo, cFieldId, cFieldName, cFieldType, cDataPath)
if err := HandleCStatus(&status, "AppendOptionalFieldDataPath failed"); err != nil {
return err
}
}
return nil
}