From c3f15c6b952c881c9a9a5a5e6008df6c04cfc5ea Mon Sep 17 00:00:00 2001 From: Enwei Jiao Date: Mon, 11 Sep 2023 20:43:17 +0800 Subject: [PATCH] Refactor duplicate error class into one place (#26985) Signed-off-by: Enwei Jiao --- Makefile | 17 +- internal/core/src/CMakeLists.txt | 3 - internal/core/src/common/BitsetView.h | 2 +- internal/core/src/common/CGoHelper.h | 37 -- internal/core/src/common/CMakeLists.txt | 6 +- .../src/{exceptions => common}/EasyAssert.cpp | 1 - .../src/{exceptions => common}/EasyAssert.h | 28 +- internal/core/src/common/FieldMeta.h | 3 +- internal/core/src/{utils => common}/File.h | 4 +- internal/core/src/common/Json.h | 2 +- .../core/src/common/RangeSearchHelper.cpp | 2 +- internal/core/src/common/SystemProperty.cpp | 2 +- internal/core/src/common/SystemProperty.h | 1 - internal/core/src/common/Tracer.cpp | 6 +- internal/core/src/common/Utils.h | 2 +- internal/core/src/common/binary_set_c.cpp | 13 +- internal/core/src/common/protobuf_utils.h | 4 +- internal/core/src/common/type_c.h | 7 +- internal/core/src/config/CMakeLists.txt | 9 +- internal/core/src/config/ConfigKnowhere.cpp | 2 +- internal/core/src/exceptions/CMakeLists.txt | 23 - internal/core/src/index/CMakeLists.txt | 13 +- internal/core/src/index/Index.h | 2 +- internal/core/src/index/ScalarIndex-inl.h | 2 +- internal/core/src/index/ScalarIndex.h | 2 +- internal/core/src/index/StringIndexMarisa.cpp | 8 +- internal/core/src/index/StringIndexMarisa.h | 4 - internal/core/src/index/Utils.cpp | 4 +- internal/core/src/index/VectorMemIndex.cpp | 4 +- .../core/src/indexbuilder/VecIndexCreator.cpp | 2 +- internal/core/src/indexbuilder/index_c.cpp | 3 +- internal/core/src/mmap/Column.h | 4 +- internal/core/src/mmap/Utils.h | 2 +- internal/core/src/query/CMakeLists.txt | 3 +- internal/core/src/query/PlanImpl.h | 4 +- internal/core/src/query/PlanProto.cpp | 5 +- internal/core/src/query/Relational.h | 2 +- internal/core/src/query/SubSearchResult.cpp | 2 +- .../core/src/query/deprecated/BinaryQuery.cpp | 324 ----------- .../core/src/query/deprecated/BinaryQuery.h | 45 -- .../core/src/query/deprecated/BooleanQuery.h | 87 --- .../core/src/query/deprecated/GeneralQuery.h | 148 ----- .../src/query/deprecated/ParserDeprecated.cpp | 295 ---------- .../src/query/deprecated/ParserDeprecated.h | 23 - .../src/query/deprecated/ValidationUtil.cpp | 523 ------------------ .../src/query/deprecated/ValidationUtil.h | 71 --- .../src/query/generated/ExecPlanNodeVisitor.h | 2 +- .../src/query/generated/ShowExprVisitor.h | 2 +- .../src/query/generated/ShowPlanNodeVisitor.h | 4 +- .../query/generated/VerifyPlanNodeVisitor.h | 2 +- .../src/query/visitors/ExecExprVisitor.cpp | 2 +- .../query/visitors/ExecPlanNodeVisitor.cpp | 2 +- .../query/visitors/ShowPlanNodeVisitor.cpp | 4 +- internal/core/src/segcore/ConcurrentVector.h | 2 +- internal/core/src/segcore/Reduce.cpp | 2 +- internal/core/src/segcore/Reduce.h | 9 +- internal/core/src/segcore/ScalarIndex.cpp | 2 +- .../core/src/segcore/SealedIndexingRecord.h | 2 +- internal/core/src/segcore/SegcoreConfig.cpp | 2 +- internal/core/src/segcore/SegcoreConfig.h | 4 +- internal/core/src/segcore/SegmentGrowing.h | 1 - .../core/src/segcore/SegmentGrowingImpl.cpp | 5 +- .../core/src/segcore/SegmentGrowingImpl.h | 6 +- internal/core/src/segcore/SegmentInterface.h | 2 +- .../core/src/segcore/SegmentSealedImpl.cpp | 8 +- internal/core/src/segcore/SegmentSealedImpl.h | 3 +- .../core/src/segcore/load_field_data_c.cpp | 7 +- internal/core/src/segcore/load_index_c.cpp | 38 +- internal/core/src/segcore/plan_c.cpp | 15 +- internal/core/src/segcore/reduce_c.cpp | 7 +- internal/core/src/segcore/segment_c.cpp | 25 +- internal/core/src/storage/BinlogReader.cpp | 14 +- internal/core/src/storage/BinlogReader.h | 9 +- internal/core/src/storage/CMakeLists.txt | 11 +- internal/core/src/storage/DataCodec.cpp | 2 +- internal/core/src/storage/Event.cpp | 15 +- .../core/src/storage/FieldDataInterface.h | 2 +- internal/core/src/storage/IndexData.cpp | 2 +- internal/core/src/storage/InsertData.cpp | 2 +- .../core/src/storage/MinioChunkManager.cpp | 2 +- internal/core/src/storage/PayloadReader.cpp | 2 +- internal/core/src/storage/PayloadStream.cpp | 2 +- internal/core/src/storage/PayloadWriter.cpp | 2 +- internal/core/src/storage/Util.cpp | 2 +- internal/core/src/storage/parquet_c.cpp | 63 ++- internal/core/src/storage/storage_c.cpp | 10 +- internal/core/src/utils/CMakeLists.txt | 17 - internal/core/src/utils/Error.h | 146 ----- internal/core/src/utils/Json.h | 35 -- internal/core/src/utils/Status.cpp | 127 ----- internal/core/src/utils/Status.h | 79 --- internal/core/unittest/bench/CMakeLists.txt | 3 +- internal/core/unittest/test_data_codec.cpp | 6 +- internal/core/unittest/test_index_c_api.cpp | 74 +-- internal/core/unittest/test_indexing.cpp | 10 +- internal/core/unittest/test_parquet_c.cpp | 2 + internal/core/unittest/test_query.cpp | 4 +- internal/core/unittest/test_tracer.cpp | 2 +- internal/core/unittest/test_utils/DataGen.h | 19 +- 99 files changed, 303 insertions(+), 2275 deletions(-) delete mode 100644 internal/core/src/common/CGoHelper.h rename internal/core/src/{exceptions => common}/EasyAssert.cpp (97%) rename internal/core/src/{exceptions => common}/EasyAssert.h (85%) rename internal/core/src/{utils => common}/File.h (96%) delete mode 100644 internal/core/src/exceptions/CMakeLists.txt delete mode 100644 internal/core/src/query/deprecated/BinaryQuery.cpp delete mode 100644 internal/core/src/query/deprecated/BinaryQuery.h delete mode 100644 internal/core/src/query/deprecated/BooleanQuery.h delete mode 100644 internal/core/src/query/deprecated/GeneralQuery.h delete mode 100644 internal/core/src/query/deprecated/ParserDeprecated.cpp delete mode 100644 internal/core/src/query/deprecated/ParserDeprecated.h delete mode 100644 internal/core/src/query/deprecated/ValidationUtil.cpp delete mode 100644 internal/core/src/query/deprecated/ValidationUtil.h delete mode 100644 internal/core/src/utils/CMakeLists.txt delete mode 100644 internal/core/src/utils/Error.h delete mode 100644 internal/core/src/utils/Json.h delete mode 100644 internal/core/src/utils/Status.cpp delete mode 100644 internal/core/src/utils/Status.h diff --git a/Makefile b/Makefile index c25e35aaec..d5651eca60 100644 --- a/Makefile +++ b/Makefile @@ -68,7 +68,7 @@ getdeps: echo "Installing mockery v$(MOCKERY_VERSION) to ./bin/" && GOBIN=$(INSTALL_PATH) go install github.com/vektra/mockery/v2@v$(MOCKERY_VERSION); \ else \ echo "Mockery v$(MOCKERY_VERSION) already installed"; \ - fi + fi tools/bin/revive: tools/check/go.mod cd tools/check; \ @@ -296,13 +296,10 @@ gpu-install: milvus-gpu clean: @echo "Cleaning up all the generated files" - @find . -name '*.test' | xargs rm -fv - @find . -name '*~' | xargs rm -fv @rm -rf bin/ @rm -rf lib/ @rm -rf $(GOPATH)/bin/milvus @rm -rf cmake_build - @rm -rf cwrapper_build @rm -rf internal/core/output milvus-tools: print-build-info @@ -336,9 +333,9 @@ generate-mockery-types: getdeps # Proxy $(INSTALL_PATH)/mockery --name=ProxyComponent --dir=$(PWD)/internal/types --output=$(PWD)/internal/mocks --filename=mock_proxy.go --with-expecter --structname=MockProxy # QueryCoord - $(INSTALL_PATH)/mockery --name=QueryCoordComponent --dir=$(PWD)/internal/types --output=$(PWD)/internal/mocks --filename=mock_querycoord.go --with-expecter --structname=MockQueryCoord + $(INSTALL_PATH)/mockery --name=QueryCoordComponent --dir=$(PWD)/internal/types --output=$(PWD)/internal/mocks --filename=mock_querycoord.go --with-expecter --structname=MockQueryCoord # QueryNode - $(INSTALL_PATH)/mockery --name=QueryNodeComponent --dir=$(PWD)/internal/types --output=$(PWD)/internal/mocks --filename=mock_querynode.go --with-expecter --structname=MockQueryNode + $(INSTALL_PATH)/mockery --name=QueryNodeComponent --dir=$(PWD)/internal/types --output=$(PWD)/internal/mocks --filename=mock_querynode.go --with-expecter --structname=MockQueryNode # DataCoord $(INSTALL_PATH)/mockery --name=DataCoordComponent --dir=$(PWD)/internal/types --output=$(PWD)/internal/mocks --filename=mock_datacoord.go --with-expecter --structname=MockDataCoord # DataNode @@ -384,9 +381,9 @@ generate-mockery-datanode: getdeps $(INSTALL_PATH)/mockery --name=Allocator --dir=$(PWD)/internal/datanode/allocator --output=$(PWD)/internal/datanode/allocator --filename=mock_allocator.go --with-expecter --structname=MockAllocator --outpkg=allocator --inpackage generate-mockery-metastore: getdeps - $(INSTALL_PATH)/mockery --name=RootCoordCatalog --dir=$(PWD)/internal/metastore --output=$(PWD)/internal/metastore/mocks --filename=mock_rootcoord_catalog.go --with-expecter --structname=RootCoordCatalog --outpkg=mocks - $(INSTALL_PATH)/mockery --name=DataCoordCatalog --dir=$(PWD)/internal/metastore --output=$(PWD)/internal/metastore/mocks --filename=mock_datacoord_catalog.go --with-expecter --structname=DataCoordCatalog --outpkg=mocks - $(INSTALL_PATH)/mockery --name=QueryCoordCatalog --dir=$(PWD)/internal/metastore --output=$(PWD)/internal/metastore/mocks --filename=mock_querycoord_catalog.go --with-expecter --structname=QueryCoordCatalog --outpkg=mocks + $(INSTALL_PATH)/mockery --name=RootCoordCatalog --dir=$(PWD)/internal/metastore --output=$(PWD)/internal/metastore/mocks --filename=mock_rootcoord_catalog.go --with-expecter --structname=RootCoordCatalog --outpkg=mocks + $(INSTALL_PATH)/mockery --name=DataCoordCatalog --dir=$(PWD)/internal/metastore --output=$(PWD)/internal/metastore/mocks --filename=mock_datacoord_catalog.go --with-expecter --structname=DataCoordCatalog --outpkg=mocks + $(INSTALL_PATH)/mockery --name=QueryCoordCatalog --dir=$(PWD)/internal/metastore --output=$(PWD)/internal/metastore/mocks --filename=mock_querycoord_catalog.go --with-expecter --structname=QueryCoordCatalog --outpkg=mocks generate-mockery-utils: getdeps # dependency.Factory @@ -404,4 +401,4 @@ generate-mockery-pkg: $(MAKE) -C pkg generate-mockery generate-mockery: generate-mockery-types generate-mockery-kv generate-mockery-rootcoord generate-mockery-proxy generate-mockery-querycoord generate-mockery-querynode generate-mockery-datacoord generate-mockery-pkg - + diff --git a/internal/core/src/CMakeLists.txt b/internal/core/src/CMakeLists.txt index 70bc827fde..bdebcedf44 100644 --- a/internal/core/src/CMakeLists.txt +++ b/internal/core/src/CMakeLists.txt @@ -23,10 +23,7 @@ endif() include_directories(${MILVUS_ENGINE_SRC}) include_directories(${MILVUS_THIRDPARTY_SRC}) -set(FOUND_OPENBLAS "unknown") add_subdirectory( pb ) -add_subdirectory( exceptions ) -add_subdirectory( utils ) add_subdirectory( log ) add_subdirectory( config ) add_subdirectory( common ) diff --git a/internal/core/src/common/BitsetView.h b/internal/core/src/common/BitsetView.h index 777c3ef6bb..57ce86a97e 100644 --- a/internal/core/src/common/BitsetView.h +++ b/internal/core/src/common/BitsetView.h @@ -22,7 +22,7 @@ #include #include "common/Types.h" -#include "exceptions/EasyAssert.h" +#include "common/EasyAssert.h" #include "knowhere/bitsetview.h" namespace milvus { diff --git a/internal/core/src/common/CGoHelper.h b/internal/core/src/common/CGoHelper.h deleted file mode 100644 index 8f8ce80032..0000000000 --- a/internal/core/src/common/CGoHelper.h +++ /dev/null @@ -1,37 +0,0 @@ -// Licensed to the LF AI & Data foundation under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#pragma once - -#include -#include - -#include "common/type_c.h" - -namespace milvus { - -inline CStatus -SuccessCStatus() { - return CStatus{Success, ""}; -} - -inline CStatus -FailureCStatus(ErrorCode error_code, const std::string_view str) { - auto str_dup = strdup(str.data()); - return CStatus{error_code, str_dup}; -} - -} // namespace milvus diff --git a/internal/core/src/common/CMakeLists.txt b/internal/core/src/common/CMakeLists.txt index 8996aa7348..7fdac692ed 100644 --- a/internal/core/src/common/CMakeLists.txt +++ b/internal/core/src/common/CMakeLists.txt @@ -20,14 +20,16 @@ set(COMMON_SRC Common.cpp RangeSearchHelper.cpp Tracer.cpp - IndexMeta.cpp) + IndexMeta.cpp + EasyAssert.cpp +) add_library(milvus_common SHARED ${COMMON_SRC}) target_link_libraries(milvus_common milvus_config - milvus_utils milvus_log + milvus_proto yaml-cpp boost_bitset_ext simdjson diff --git a/internal/core/src/exceptions/EasyAssert.cpp b/internal/core/src/common/EasyAssert.cpp similarity index 97% rename from internal/core/src/exceptions/EasyAssert.cpp rename to internal/core/src/common/EasyAssert.cpp index 77d80ccd6d..5d58cc5ad7 100644 --- a/internal/core/src/exceptions/EasyAssert.cpp +++ b/internal/core/src/common/EasyAssert.cpp @@ -16,7 +16,6 @@ #include #include "EasyAssert.h" -// #define BOOST_STACKTRACE_USE_BACKTRACE #include #include diff --git a/internal/core/src/exceptions/EasyAssert.h b/internal/core/src/common/EasyAssert.h similarity index 85% rename from internal/core/src/exceptions/EasyAssert.h rename to internal/core/src/common/EasyAssert.h index cabacdad5c..36f730ebaf 100644 --- a/internal/core/src/exceptions/EasyAssert.h +++ b/internal/core/src/common/EasyAssert.h @@ -22,10 +22,26 @@ #include #include #include "pb/common.pb.h" +#include "common/type_c.h" /* Paste this on the file if you want to debug. */ namespace milvus { -using ErrorCodeEnum = proto::common::ErrorCode; +enum ErrorCodeEnum { + Success = 0, + UnexpectedError = 1, + BuildIndexError = 2, + IllegalArgument = 5, +}; + +inline CStatus +SuccessCStatus() { + return CStatus{Success, ""}; +} + +inline CStatus +FailureCStatus(int code, const std::string& msg) { + return CStatus{code, strdup(msg.data())}; +} namespace impl { void EasyAssertInfo(bool value, @@ -39,6 +55,11 @@ EasyAssertInfo(bool value, class SegcoreError : public std::runtime_error { public: + static SegcoreError + success() { + return SegcoreError(ErrorCodeEnum::Success, ""); + } + SegcoreError(ErrorCodeEnum error_code, const std::string& error_msg) : std::runtime_error(error_msg), error_code_(error_code) { } @@ -48,6 +69,11 @@ class SegcoreError : public std::runtime_error { return error_code_; } + bool + ok() { + return error_code_ == ErrorCodeEnum::Success; + } + private: ErrorCodeEnum error_code_; }; diff --git a/internal/core/src/common/FieldMeta.h b/internal/core/src/common/FieldMeta.h index 7b2806b026..636186035f 100644 --- a/internal/core/src/common/FieldMeta.h +++ b/internal/core/src/common/FieldMeta.h @@ -21,8 +21,7 @@ #include #include "common/Types.h" -#include "exceptions/EasyAssert.h" -#include "utils/Status.h" +#include "common/EasyAssert.h" namespace milvus { diff --git a/internal/core/src/utils/File.h b/internal/core/src/common/File.h similarity index 96% rename from internal/core/src/utils/File.h rename to internal/core/src/common/File.h index 6ce8ae5722..3622bcbf03 100644 --- a/internal/core/src/utils/File.h +++ b/internal/core/src/common/File.h @@ -12,7 +12,7 @@ #pragma once #include -#include "exceptions/EasyAssert.h" +#include "common/EasyAssert.h" #include "fmt/core.h" #include #include @@ -62,4 +62,4 @@ class File { } int fd_{-1}; }; -} // namespace milvus \ No newline at end of file +} // namespace milvus diff --git a/internal/core/src/common/Json.h b/internal/core/src/common/Json.h index 91df2b9414..892652955a 100644 --- a/internal/core/src/common/Json.h +++ b/internal/core/src/common/Json.h @@ -25,7 +25,7 @@ #include #include -#include "exceptions/EasyAssert.h" +#include "common/EasyAssert.h" #include "simdjson.h" #include "fmt/core.h" #include "simdjson/common_defs.h" diff --git a/internal/core/src/common/RangeSearchHelper.cpp b/internal/core/src/common/RangeSearchHelper.cpp index 36c68f35bd..9e51dac1e6 100644 --- a/internal/core/src/common/RangeSearchHelper.cpp +++ b/internal/core/src/common/RangeSearchHelper.cpp @@ -88,7 +88,7 @@ ReGenRangeSearchResult(DatasetPtr data_set, pq(cmp); auto capacity = std::min(lims[i + 1] - lims[i], topk); - for (int j = lims[i]; j < lims[i + 1]; j++) { + for (size_t j = lims[i]; j < lims[i + 1]; j++) { auto curr = ResultPair(dist[j], id[j]); if (pq.size() < capacity) { pq.push(curr); diff --git a/internal/core/src/common/SystemProperty.cpp b/internal/core/src/common/SystemProperty.cpp index 49f889a88c..ad42ea541a 100644 --- a/internal/core/src/common/SystemProperty.cpp +++ b/internal/core/src/common/SystemProperty.cpp @@ -18,7 +18,7 @@ #include "SystemProperty.h" #include "Consts.h" -#include "exceptions/EasyAssert.h" +#include "common/EasyAssert.h" namespace milvus { class SystemPropertyImpl : public SystemProperty { diff --git a/internal/core/src/common/SystemProperty.h b/internal/core/src/common/SystemProperty.h index 373e4e42a3..b0aac7fcb6 100644 --- a/internal/core/src/common/SystemProperty.h +++ b/internal/core/src/common/SystemProperty.h @@ -19,7 +19,6 @@ #include #include "Types.h" -#include "utils/Json.h" namespace milvus { diff --git a/internal/core/src/common/Tracer.cpp b/internal/core/src/common/Tracer.cpp index 69fc899fc8..21a4c63709 100644 --- a/internal/core/src/common/Tracer.cpp +++ b/internal/core/src/common/Tracer.cpp @@ -11,6 +11,8 @@ #include "log/Log.h" #include "Tracer.h" +#include + #include "opentelemetry/exporters/ostream/span_exporter_factory.h" #include "opentelemetry/exporters/jaeger/jaeger_exporter_factory.h" #include "opentelemetry/exporters/otlp/otlp_grpc_exporter_factory.h" @@ -103,7 +105,7 @@ thread_local std::shared_ptr local_span; void SetRootSpan(std::shared_ptr span) { if (enable_trace) { - local_span = span; + local_span = std::move(span); } } @@ -123,7 +125,7 @@ AddEvent(std::string event_label) { bool isEmptyID(const uint8_t* id, int length) { - for (size_t i = 0; i < length; i++) { + for (int i = 0; i < length; i++) { if (id[i] != 0) { return false; } diff --git a/internal/core/src/common/Utils.h b/internal/core/src/common/Utils.h index d90a72deba..58b9f0da5e 100644 --- a/internal/core/src/common/Utils.h +++ b/internal/core/src/common/Utils.h @@ -28,7 +28,7 @@ #include "common/FieldMeta.h" #include "common/LoadInfo.h" #include "common/Types.h" -#include "exceptions/EasyAssert.h" +#include "common/EasyAssert.h" #include "knowhere/dataset.h" #include "knowhere/expected.h" #include "simdjson.h" diff --git a/internal/core/src/common/binary_set_c.cpp b/internal/core/src/common/binary_set_c.cpp index 8495ae73f6..20a239bc28 100644 --- a/internal/core/src/common/binary_set_c.cpp +++ b/internal/core/src/common/binary_set_c.cpp @@ -14,6 +14,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include "common/EasyAssert.h" #include "knowhere/binaryset.h" #include "common/binary_set_c.h" @@ -23,12 +24,12 @@ NewBinarySet(CBinarySet* c_binary_set) { auto binary_set = std::make_unique(); *c_binary_set = binary_set.release(); auto status = CStatus(); - status.error_code = Success; + status.error_code = milvus::ErrorCodeEnum::Success; status.error_msg = ""; return status; } catch (std::exception& e) { auto status = CStatus(); - status.error_code = UnexpectedError; + status.error_code = milvus::ErrorCodeEnum::UnexpectedError; status.error_msg = strdup(e.what()); return status; } @@ -55,10 +56,10 @@ AppendIndexBinary(CBinarySet c_binary_set, std::shared_ptr data(dup); binary_set->Append(index_key, data, index_size); - status.error_code = Success; + status.error_code = milvus::ErrorCodeEnum::Success; status.error_msg = ""; } catch (std::exception& e) { - status.error_code = UnexpectedError; + status.error_code = milvus::ErrorCodeEnum::UnexpectedError; status.error_msg = strdup(e.what()); } return status; @@ -100,11 +101,11 @@ CopyBinarySetValue(void* data, const char* key, CBinarySet c_binary_set) { auto binary_set = (knowhere::BinarySet*)c_binary_set; try { auto binary = binary_set->GetByName(key); - status.error_code = Success; + status.error_code = milvus::ErrorCodeEnum::Success; status.error_msg = ""; memcpy((uint8_t*)data, binary->data.get(), binary->size); } catch (std::exception& e) { - status.error_code = UnexpectedError; + status.error_code = milvus::ErrorCodeEnum::UnexpectedError; status.error_msg = strdup(e.what()); } return status; diff --git a/internal/core/src/common/protobuf_utils.h b/internal/core/src/common/protobuf_utils.h index 387b168de9..4a6502fc4f 100644 --- a/internal/core/src/common/protobuf_utils.h +++ b/internal/core/src/common/protobuf_utils.h @@ -21,7 +21,7 @@ #include #include "pb/schema.pb.h" -#include "exceptions/EasyAssert.h" +#include "common/EasyAssert.h" using std::string; @@ -38,4 +38,4 @@ RepeatedKeyValToMap( } return mapping; } -} //namespace milvus \ No newline at end of file +} //namespace milvus diff --git a/internal/core/src/common/type_c.h b/internal/core/src/common/type_c.h index e751da7eb3..fdc2a1fcd9 100644 --- a/internal/core/src/common/type_c.h +++ b/internal/core/src/common/type_c.h @@ -31,12 +31,6 @@ enum SegmentType { typedef enum SegmentType SegmentType; -enum ErrorCode { - Success = 0, - UnexpectedError = 1, - IllegalArgument = 5, -}; - // pure C don't support that we use schemapb.DataType directly. // Note: the value of all enumerations must match the corresponding schemapb.DataType. // TODO: what if there are increments in schemapb.DataType. @@ -108,4 +102,5 @@ typedef struct CTraceContext { } CTraceContext; #ifdef __cplusplus } + #endif diff --git a/internal/core/src/config/CMakeLists.txt b/internal/core/src/config/CMakeLists.txt index a167af69b3..36f3ccc6a7 100644 --- a/internal/core/src/config/CMakeLists.txt +++ b/internal/core/src/config/CMakeLists.txt @@ -20,13 +20,8 @@ if ( EMBEDDED_MILVUS ) add_compile_definitions( EMBEDDED_MILVUS ) endif() -set(CONFIG_SRC - ConfigKnowhere.cpp - ) +set(CONFIG_SRC ConfigKnowhere.cpp) add_library(milvus_config STATIC ${CONFIG_SRC}) -target_link_libraries(milvus_config - milvus_exceptions - knowhere - ) +target_link_libraries(milvus_config knowhere) diff --git a/internal/core/src/config/ConfigKnowhere.cpp b/internal/core/src/config/ConfigKnowhere.cpp index 646ecce967..99b5ce80e6 100644 --- a/internal/core/src/config/ConfigKnowhere.cpp +++ b/internal/core/src/config/ConfigKnowhere.cpp @@ -17,7 +17,7 @@ #include #include "ConfigKnowhere.h" -#include "exceptions/EasyAssert.h" +#include "common/EasyAssert.h" #include "glog/logging.h" #include "log/Log.h" #include "knowhere/comp/thread_pool.h" diff --git a/internal/core/src/exceptions/CMakeLists.txt b/internal/core/src/exceptions/CMakeLists.txt deleted file mode 100644 index b92906ca8f..0000000000 --- a/internal/core/src/exceptions/CMakeLists.txt +++ /dev/null @@ -1,23 +0,0 @@ -# Licensed to the LF AI & Data foundation under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set(exceptions_files - EasyAssert.cpp -) - -add_library(milvus_exceptions STATIC ${exceptions_files}) - -target_link_libraries(milvus_exceptions milvus_proto) diff --git a/internal/core/src/index/CMakeLists.txt b/internal/core/src/index/CMakeLists.txt index 11da14d5bd..19ecde2cf2 100644 --- a/internal/core/src/index/CMakeLists.txt +++ b/internal/core/src/index/CMakeLists.txt @@ -27,17 +27,6 @@ endif () milvus_add_pkg_config("milvus_index") add_library(milvus_index SHARED ${INDEX_FILES}) -set(PLATFORM_LIBS ) -if ( LINUX OR APPLE ) - set(PLATFORM_LIBS marisa) -endif() -if (MSYS) - set(PLATFORM_LIBS -Wl,--allow-multiple-definition) -endif () - -target_link_libraries(milvus_index - milvus_storage - ${PLATFORM_LIBS} - ) +target_link_libraries(milvus_index milvus_storage) install(TARGETS milvus_index DESTINATION "${CMAKE_INSTALL_LIBDIR}") diff --git a/internal/core/src/index/Index.h b/internal/core/src/index/Index.h index 0d1cc55ef7..87d6880934 100644 --- a/internal/core/src/index/Index.h +++ b/internal/core/src/index/Index.h @@ -18,7 +18,7 @@ #include #include -#include "exceptions/EasyAssert.h" +#include "common/EasyAssert.h" #include "knowhere/comp/index_param.h" #include "knowhere/dataset.h" #include "common/Types.h" diff --git a/internal/core/src/index/ScalarIndex-inl.h b/internal/core/src/index/ScalarIndex-inl.h index 5ddf60d219..f4e17790b7 100644 --- a/internal/core/src/index/ScalarIndex-inl.h +++ b/internal/core/src/index/ScalarIndex-inl.h @@ -20,6 +20,7 @@ #include "index/Meta.h" #include "knowhere/dataset.h" +#include "common/Types.h" namespace milvus::index { template @@ -73,7 +74,6 @@ inline void ScalarIndex::BuildWithRawData(size_t n, const void* values, const Config& config) { - // TODO :: use arrow proto::schema::StringArray arr; auto ok = arr.ParseFromArray(values, n); Assert(ok); diff --git a/internal/core/src/index/ScalarIndex.h b/internal/core/src/index/ScalarIndex.h index ec67f79b80..1302c1e2e7 100644 --- a/internal/core/src/index/ScalarIndex.h +++ b/internal/core/src/index/ScalarIndex.h @@ -22,7 +22,7 @@ #include #include "common/Types.h" -#include "exceptions/EasyAssert.h" +#include "common/EasyAssert.h" #include "index/Index.h" namespace milvus::index { diff --git a/internal/core/src/index/StringIndexMarisa.cpp b/internal/core/src/index/StringIndexMarisa.cpp index 69328de68f..e8db07c667 100644 --- a/internal/core/src/index/StringIndexMarisa.cpp +++ b/internal/core/src/index/StringIndexMarisa.cpp @@ -30,8 +30,6 @@ namespace milvus::index { -#if defined(__linux__) || defined(__APPLE__) - StringIndexMarisa::StringIndexMarisa(storage::FileManagerImplPtr file_manager) { if (file_manager != nullptr) { file_manager_ = std::dynamic_pointer_cast( @@ -67,7 +65,7 @@ StringIndexMarisa::Build(const Config& config) { marisa::Keyset keyset; for (auto data : field_datas) { auto slice_num = data->get_num_rows(); - for (size_t i = 0; i < slice_num; ++i) { + for (int64_t i = 0; i < slice_num; ++i) { keyset.push_back( (*static_cast(data->RawValue(i))).c_str()); } @@ -80,7 +78,7 @@ StringIndexMarisa::Build(const Config& config) { int64_t offset = 0; for (auto data : field_datas) { auto slice_num = data->get_num_rows(); - for (size_t i = 0; i < slice_num; ++i) { + for (int64_t i = 0; i < slice_num; ++i) { auto str_id = lookup(*static_cast(data->RawValue(i))); AssertInfo(valid_str_id(str_id), "invalid marisa key"); @@ -390,6 +388,4 @@ StringIndexMarisa::Reverse_Lookup(size_t offset) const { return std::string(agent.key().ptr(), agent.key().length()); } -#endif - } // namespace milvus::index diff --git a/internal/core/src/index/StringIndexMarisa.h b/internal/core/src/index/StringIndexMarisa.h index ca9ee3612e..859fa70376 100644 --- a/internal/core/src/index/StringIndexMarisa.h +++ b/internal/core/src/index/StringIndexMarisa.h @@ -16,8 +16,6 @@ #pragma once -#if defined(__linux__) || defined(__APPLE__) - #include #include "index/StringIndex.h" #include @@ -114,5 +112,3 @@ CreateStringIndexMarisa(storage::FileManagerImplPtr file_manager = nullptr) { } } // namespace milvus::index - -#endif diff --git a/internal/core/src/index/Utils.cpp b/internal/core/src/index/Utils.cpp index 87f5c7fc0e..fe96a2ddfd 100644 --- a/internal/core/src/index/Utils.cpp +++ b/internal/core/src/index/Utils.cpp @@ -30,12 +30,12 @@ #include "index/Meta.h" #include #include -#include "exceptions/EasyAssert.h" +#include "common/EasyAssert.h" #include "knowhere/comp/index_param.h" #include "common/Slice.h" #include "storage/FieldData.h" #include "storage/Util.h" -#include "utils/File.h" +#include "common/File.h" namespace milvus::index { diff --git a/internal/core/src/index/VectorMemIndex.cpp b/internal/core/src/index/VectorMemIndex.cpp index 786295c8b1..4606f8e7b4 100644 --- a/internal/core/src/index/VectorMemIndex.cpp +++ b/internal/core/src/index/VectorMemIndex.cpp @@ -29,7 +29,7 @@ #include "index/Index.h" #include "index/Meta.h" #include "index/Utils.h" -#include "exceptions/EasyAssert.h" +#include "common/EasyAssert.h" #include "config/ConfigKnowhere.h" #include "knowhere/factory.h" #include "knowhere/comp/time_recorder.h" @@ -43,7 +43,7 @@ #include "storage/MemFileManagerImpl.h" #include "storage/ThreadPools.h" #include "storage/Util.h" -#include "utils/File.h" +#include "common/File.h" #include "common/Tracer.h" namespace milvus::index { diff --git a/internal/core/src/indexbuilder/VecIndexCreator.cpp b/internal/core/src/indexbuilder/VecIndexCreator.cpp index 962ff7f0b9..dcc47901de 100644 --- a/internal/core/src/indexbuilder/VecIndexCreator.cpp +++ b/internal/core/src/indexbuilder/VecIndexCreator.cpp @@ -11,7 +11,7 @@ #include -#include "exceptions/EasyAssert.h" +#include "common/EasyAssert.h" #include "indexbuilder/VecIndexCreator.h" #include "index/Utils.h" #include "index/IndexFactory.h" diff --git a/internal/core/src/indexbuilder/index_c.cpp b/internal/core/src/indexbuilder/index_c.cpp index 2e8d3bd233..33ded49fe6 100644 --- a/internal/core/src/indexbuilder/index_c.cpp +++ b/internal/core/src/indexbuilder/index_c.cpp @@ -15,7 +15,7 @@ #include #endif -#include "exceptions/EasyAssert.h" +#include "common/EasyAssert.h" #include "indexbuilder/VecIndexCreator.h" #include "indexbuilder/index_c.h" #include "indexbuilder/IndexFactory.h" @@ -26,6 +26,7 @@ #include "pb/index_cgo_msg.pb.h" #include "storage/Util.h" +using namespace milvus; CStatus CreateIndex(enum CDataType dtype, const char* serialized_type_params, diff --git a/internal/core/src/mmap/Column.h b/internal/core/src/mmap/Column.h index bd1e8d5f0b..4b3ec8aa71 100644 --- a/internal/core/src/mmap/Column.h +++ b/internal/core/src/mmap/Column.h @@ -23,12 +23,12 @@ #include "common/FieldMeta.h" #include "common/Span.h" -#include "exceptions/EasyAssert.h" +#include "common/EasyAssert.h" +#include "common/File.h" #include "fmt/format.h" #include "log/Log.h" #include "mmap/Utils.h" #include "storage/FieldData.h" -#include "utils/File.h" namespace milvus { diff --git a/internal/core/src/mmap/Utils.h b/internal/core/src/mmap/Utils.h index 3f01fe7b4d..d762d336d5 100644 --- a/internal/core/src/mmap/Utils.h +++ b/internal/core/src/mmap/Utils.h @@ -27,7 +27,7 @@ #include "common/FieldMeta.h" #include "mmap/Types.h" #include "storage/Util.h" -#include "utils/File.h" +#include "common/File.h" namespace milvus { diff --git a/internal/core/src/query/CMakeLists.txt b/internal/core/src/query/CMakeLists.txt index c476e476a9..2674cbd520 100644 --- a/internal/core/src/query/CMakeLists.txt +++ b/internal/core/src/query/CMakeLists.txt @@ -10,7 +10,6 @@ # or implied. See the License for the specific language governing permissions and limitations under the License set(MILVUS_QUERY_SRCS - deprecated/BinaryQuery.cpp generated/PlanNode.cpp generated/Expr.cpp visitors/ShowPlanNodeVisitor.cpp @@ -32,6 +31,6 @@ set(MILVUS_QUERY_SRCS add_library(milvus_query ${MILVUS_QUERY_SRCS}) if(USE_DYNAMIC_SIMD) target_link_libraries(milvus_query milvus_index milvus_simd) -else() +else() target_link_libraries(milvus_query milvus_index) endif() diff --git a/internal/core/src/query/PlanImpl.h b/internal/core/src/query/PlanImpl.h index 6fe94a82bc..d015387f63 100644 --- a/internal/core/src/query/PlanImpl.h +++ b/internal/core/src/query/PlanImpl.h @@ -19,8 +19,8 @@ #include "Plan.h" #include "PlanNode.h" -#include "exceptions/EasyAssert.h" -#include "utils/Json.h" +#include "common/EasyAssert.h" +#include "common/Json.h" #include "common/Consts.h" namespace milvus::query { diff --git a/internal/core/src/query/PlanProto.cpp b/internal/core/src/query/PlanProto.cpp index 541f11b0df..bb9c064258 100644 --- a/internal/core/src/query/PlanProto.cpp +++ b/internal/core/src/query/PlanProto.cpp @@ -18,7 +18,7 @@ #include "ExprImpl.h" #include "common/VectorTrait.h" -#include "exceptions/EasyAssert.h" +#include "common/EasyAssert.h" #include "generated/ExtractInfoExprVisitor.h" #include "generated/ExtractInfoPlanNodeVisitor.h" #include "pb/plan.pb.h" @@ -182,7 +182,8 @@ ProtoParser::PlanNodeFromProto(const planpb::PlanNode& plan_node_proto) { search_info.metric_type_ = query_info_proto.metric_type(); search_info.topk_ = query_info_proto.topk(); search_info.round_decimal_ = query_info_proto.round_decimal(); - search_info.search_params_ = json::parse(query_info_proto.search_params()); + search_info.search_params_ = + nlohmann::json::parse(query_info_proto.search_params()); auto plan_node = [&]() -> std::unique_ptr { if (anns_proto.vector_type() == diff --git a/internal/core/src/query/Relational.h b/internal/core/src/query/Relational.h index e96ec60b0b..b520226d57 100644 --- a/internal/core/src/query/Relational.h +++ b/internal/core/src/query/Relational.h @@ -16,7 +16,7 @@ #include "common/Utils.h" #include "common/VectorTrait.h" -#include "exceptions/EasyAssert.h" +#include "common/EasyAssert.h" #include "query/Expr.h" #include "query/Utils.h" diff --git a/internal/core/src/query/SubSearchResult.cpp b/internal/core/src/query/SubSearchResult.cpp index e25b1090ff..d9e34b0b76 100644 --- a/internal/core/src/query/SubSearchResult.cpp +++ b/internal/core/src/query/SubSearchResult.cpp @@ -11,7 +11,7 @@ #include -#include "exceptions/EasyAssert.h" +#include "common/EasyAssert.h" #include "query/SubSearchResult.h" namespace milvus::query { diff --git a/internal/core/src/query/deprecated/BinaryQuery.cpp b/internal/core/src/query/deprecated/BinaryQuery.cpp deleted file mode 100644 index 832eb801c2..0000000000 --- a/internal/core/src/query/deprecated/BinaryQuery.cpp +++ /dev/null @@ -1,324 +0,0 @@ -// Copyright (C) 2019-2020 Zilliz. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software distributed under the License -// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -// or implied. See the License for the specific language governing permissions and limitations under the License. - -#include -#include -#include -#include -#include -#include - -#include "BinaryQuery.h" - -namespace milvus { -namespace query_old { - -BinaryQueryPtr -ConstructBinTree(std::vector queries, - QueryRelation relation, - uint64_t idx) { - if (idx == queries.size()) { - return nullptr; - } else if (idx == queries.size() - 1) { - return queries[idx]->getBinaryQuery(); - } else { - BinaryQueryPtr bquery = std::make_shared(); - bquery->relation = relation; - bquery->left_query = std::make_shared(); - bquery->right_query = std::make_shared(); - bquery->left_query->bin = queries[idx]->getBinaryQuery(); - ++idx; - bquery->right_query->bin = ConstructBinTree(queries, relation, idx); - return bquery; - } -} - -Status -ConstructLeafBinTree(std::vector leaf_queries, - BinaryQueryPtr binary_query, - uint64_t idx) { - if (idx == leaf_queries.size()) { - return Status::OK(); - } - binary_query->left_query = std::make_shared(); - binary_query->right_query = std::make_shared(); - if (leaf_queries.size() == leaf_queries.size() - 1) { - binary_query->left_query->leaf = leaf_queries[idx]; - return Status::OK(); - } else if (idx == leaf_queries.size() - 2) { - binary_query->left_query->leaf = leaf_queries[idx]; - ++idx; - binary_query->right_query->leaf = leaf_queries[idx]; - return Status::OK(); - } else { - binary_query->left_query->bin->relation = binary_query->relation; - binary_query->right_query->leaf = leaf_queries[idx]; - ++idx; - return ConstructLeafBinTree( - leaf_queries, binary_query->left_query->bin, idx); - } -} - -Status -GenBinaryQuery(BooleanQueryPtr query, BinaryQueryPtr& binary_query) { - if (query->getBooleanQueries().size() == 0) { - if (binary_query->relation == QueryRelation::AND || - binary_query->relation == QueryRelation::OR) { - // Put VectorQuery to the end of leaf queries - auto query_size = query->getLeafQueries().size(); - for (uint64_t i = 0; i < query_size; ++i) { - if (query->getLeafQueries()[i]->vector_placeholder.size() > 0) { - std::swap(query->getLeafQueries()[i], - query->getLeafQueries()[0]); - break; - } - } - return ConstructLeafBinTree( - query->getLeafQueries(), binary_query, 0); - } else { - switch (query->getOccur()) { - case Occur::MUST: { - binary_query->relation = QueryRelation::AND; - return GenBinaryQuery(query, binary_query); - } - case Occur::MUST_NOT: - binary_query->is_not = true; - case Occur::SHOULD: { - binary_query->relation = QueryRelation::OR; - return GenBinaryQuery(query, binary_query); - } - default: - return Status::OK(); - } - } - } - - if (query->getBooleanQueries().size() == 1) { - auto bc = query->getBooleanQueries()[0]; - binary_query->left_query = std::make_shared(); - switch (bc->getOccur()) { - case Occur::MUST: { - binary_query->relation = QueryRelation::AND; - return GenBinaryQuery(bc, binary_query); - } - case Occur::MUST_NOT: - binary_query->is_not = true; - case Occur::SHOULD: { - binary_query->relation = QueryRelation::OR; - return GenBinaryQuery(bc, binary_query); - } - default: - return Status::OK(); - } - } - - // Construct binary query for every single boolean query - std::vector must_queries; - std::vector must_not_queries; - std::vector should_queries; - Status status; - for (auto& _query : query->getBooleanQueries()) { - status = GenBinaryQuery(_query, _query->getBinaryQuery()); - if (!status.ok()) { - return status; - } - if (_query->getOccur() == Occur::MUST) { - must_queries.emplace_back(_query); - } else if (_query->getOccur() == Occur::MUST_NOT) { - must_not_queries.emplace_back(_query); - } else { - should_queries.emplace_back(_query); - } - } - - // Construct binary query for combine boolean queries - BinaryQueryPtr must_bquery, should_bquery, must_not_bquery; - uint64_t bquery_num = 0; - if (must_queries.size() > 1) { - // Construct a must binary tree - must_bquery = ConstructBinTree(must_queries, QueryRelation::R1, 0); - ++bquery_num; - } else if (must_queries.size() == 1) { - must_bquery = must_queries[0]->getBinaryQuery(); - ++bquery_num; - } - - if (should_queries.size() > 1) { - // Construct a should binary tree - should_bquery = ConstructBinTree(should_queries, QueryRelation::R2, 0); - ++bquery_num; - } else if (should_queries.size() == 1) { - should_bquery = should_queries[0]->getBinaryQuery(); - ++bquery_num; - } - - if (must_not_queries.size() > 1) { - // Construct a must_not binary tree - must_not_bquery = - ConstructBinTree(must_not_queries, QueryRelation::R1, 0); - ++bquery_num; - } else if (must_not_queries.size() == 1) { - must_not_bquery = must_not_queries[0]->getBinaryQuery(); - ++bquery_num; - } - - binary_query->left_query = std::make_shared(); - binary_query->right_query = std::make_shared(); - BinaryQueryPtr must_should_query = std::make_shared(); - must_should_query->left_query = std::make_shared(); - must_should_query->right_query = std::make_shared(); - if (bquery_num == 3) { - must_should_query->relation = QueryRelation::R3; - must_should_query->left_query->bin = must_bquery; - must_should_query->right_query->bin = should_bquery; - binary_query->relation = QueryRelation::R1; - binary_query->left_query->bin = must_should_query; - binary_query->right_query->bin = must_not_bquery; - } else if (bquery_num == 2) { - if (must_bquery == nullptr) { - // should + must_not - binary_query->relation = QueryRelation::R3; - binary_query->left_query->bin = must_not_bquery; - binary_query->right_query->bin = should_bquery; - } else if (should_bquery == nullptr) { - // must + must_not - binary_query->relation = QueryRelation::R4; - binary_query->left_query->bin = must_bquery; - binary_query->right_query->bin = must_not_bquery; - } else { - // must + should - binary_query->relation = QueryRelation::R3; - binary_query->left_query->bin = must_bquery; - binary_query->right_query->bin = should_bquery; - } - } else { - if (must_bquery != nullptr) { - binary_query = must_bquery; - } else if (should_bquery != nullptr) { - binary_query = should_bquery; - } else { - binary_query = must_not_bquery; - } - } - - return Status::OK(); -} - -uint64_t -BinaryQueryHeight(BinaryQueryPtr& binary_query) { - if (binary_query == nullptr) { - return 1; - } - uint64_t left_height = 0, right_height = 0; - if (binary_query->left_query != nullptr) { - left_height = BinaryQueryHeight(binary_query->left_query->bin); - } - if (binary_query->right_query != nullptr) { - right_height = BinaryQueryHeight(binary_query->right_query->bin); - } - return left_height > right_height ? left_height + 1 : right_height + 1; -} - -/** - * rules: - * 1. The child node of 'should' and 'must_not' can only be 'term query' and 'range query'. - * 2. One layer cannot include bool query and leaf query. - * 3. The direct child node of 'bool' node cannot be 'should' node or 'must_not' node. - * 4. All filters are pre-filtered(Do structure query first, then use the result to do filtering for vector query). - * - */ - -Status -rule_1(BooleanQueryPtr& boolean_query, - std::stack& path_stack) { - auto status = Status::OK(); - if (boolean_query != nullptr) { - path_stack.push(boolean_query); - for (const auto& leaf_query : boolean_query->getLeafQueries()) { - if (!leaf_query->vector_placeholder.empty()) { - while (!path_stack.empty()) { - auto query = path_stack.top(); - if (query->getOccur() == Occur::SHOULD || - query->getOccur() == Occur::MUST_NOT) { - std::string msg = - "The child node of 'should' and 'must_not' can " - "only be 'term query' and 'range query'."; - return Status{SERVER_INVALID_DSL_PARAMETER, msg}; - } - path_stack.pop(); - } - } - } - for (auto query : boolean_query->getBooleanQueries()) { - status = rule_1(query, path_stack); - if (!status.ok()) { - return status; - } - } - } - return status; -} - -Status -rule_2(BooleanQueryPtr& boolean_query) { - auto status = Status::OK(); - if (boolean_query != nullptr) { - if (!boolean_query->getBooleanQueries().empty() && - !boolean_query->getLeafQueries().empty()) { - std::string msg = - "One layer cannot include bool query and leaf query."; - return Status{SERVER_INVALID_DSL_PARAMETER, msg}; - } else { - for (auto query : boolean_query->getBooleanQueries()) { - status = rule_2(query); - if (!status.ok()) { - return status; - } - } - } - } - return status; -} - -Status -ValidateBooleanQuery(BooleanQueryPtr& boolean_query) { - auto status = Status::OK(); - if (boolean_query != nullptr) { - for (auto& query : boolean_query->getBooleanQueries()) { - if (query->getOccur() == Occur::SHOULD || - query->getOccur() == Occur::MUST_NOT) { - std::string msg = - "The direct child node of 'bool' node cannot be 'should' " - "node or 'must_not' node."; - return Status{SERVER_INVALID_DSL_PARAMETER, msg}; - } - } - std::stack path_stack; - status = rule_1(boolean_query, path_stack); - if (!status.ok()) { - return status; - } - status = rule_2(boolean_query); - if (!status.ok()) { - return status; - } - } - return status; -} - -bool -ValidateBinaryQuery(BinaryQueryPtr& binary_query) { - uint64_t height = BinaryQueryHeight(binary_query); - return height > 1; -} - -} // namespace query_old -} // namespace milvus diff --git a/internal/core/src/query/deprecated/BinaryQuery.h b/internal/core/src/query/deprecated/BinaryQuery.h deleted file mode 100644 index d99c49a040..0000000000 --- a/internal/core/src/query/deprecated/BinaryQuery.h +++ /dev/null @@ -1,45 +0,0 @@ -// Copyright (C) 2019-2020 Zilliz. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software distributed under the License -// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -// or implied. See the License for the specific language governing permissions and limitations under the License. - -#pragma once - -#include -#include - -#include "BooleanQuery.h" - -namespace milvus { -namespace query_old { - -BinaryQueryPtr -ConstructBinTree(std::vector clauses, - QueryRelation relation, - uint64_t idx); - -Status -ConstructLeafBinTree(std::vector leaf_clauses, - BinaryQueryPtr binary_query, - uint64_t idx); - -Status -GenBinaryQuery(BooleanQueryPtr clause, BinaryQueryPtr& binary_query); - -uint64_t -BinaryQueryHeight(BinaryQueryPtr& binary_query); - -Status -ValidateBooleanQuery(BooleanQueryPtr& boolean_query); - -bool -ValidateBinaryQuery(BinaryQueryPtr& binary_query); - -} // namespace query_old -} // namespace milvus diff --git a/internal/core/src/query/deprecated/BooleanQuery.h b/internal/core/src/query/deprecated/BooleanQuery.h deleted file mode 100644 index 10618d1294..0000000000 --- a/internal/core/src/query/deprecated/BooleanQuery.h +++ /dev/null @@ -1,87 +0,0 @@ -// Copyright (C) 2019-2020 Zilliz. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software distributed under the License -// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -// or implied. See the License for the specific language governing permissions and limitations under the License. - -#pragma once - -#include -#include - -#include "GeneralQuery.h" -#include "utils/Status.h" - -namespace milvus { -namespace query_old { - -enum class Occur { - INVALID = 0, - MUST, - MUST_NOT, - SHOULD, -}; - -class BooleanQuery { - public: - BooleanQuery() { - } - - explicit BooleanQuery(Occur occur) : occur_(occur) { - } - - Occur - getOccur() { - return occur_; - } - - void - SetOccur(Occur occur) { - occur_ = occur; - } - - void - AddBooleanQuery(std::shared_ptr boolean_clause) { - boolean_clauses_.emplace_back(boolean_clause); - } - - void - AddLeafQuery(LeafQueryPtr leaf_query) { - leaf_queries_.emplace_back(leaf_query); - } - - void - SetLeafQuery(std::vector leaf_queries) { - leaf_queries_ = leaf_queries; - } - - std::vector> - getBooleanQueries() { - return boolean_clauses_; - } - - BinaryQueryPtr& - getBinaryQuery() { - return binary_query_; - } - - std::vector& - getLeafQueries() { - return leaf_queries_; - } - - private: - Occur occur_ = Occur::INVALID; - std::vector> boolean_clauses_; - std::vector leaf_queries_; - BinaryQueryPtr binary_query_ = std::make_shared(); -}; -using BooleanQueryPtr = std::shared_ptr; - -} // namespace query_old -} // namespace milvus diff --git a/internal/core/src/query/deprecated/GeneralQuery.h b/internal/core/src/query/deprecated/GeneralQuery.h deleted file mode 100644 index 55a371d5f3..0000000000 --- a/internal/core/src/query/deprecated/GeneralQuery.h +++ /dev/null @@ -1,148 +0,0 @@ -// Copyright (C) 2019-2020 Zilliz. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software distributed under the License -// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -// or implied. See the License for the specific language governing permissions and limitations under the License. - -#pragma once - -#include -#include -#include -#include -#include -#include - -#include "common/Types.h" -#include "utils/Json.h" - -namespace milvus { -namespace query_old { - -enum class CompareOperator { - LT = 0, - LTE, - EQ, - GT, - GTE, - NE, -}; - -enum class QueryRelation { - INVALID = 0, - R1, - R2, - R3, - R4, - AND, - OR, -}; - -struct QueryColumn { - std::string name; - std::string column_value; -}; - -struct TermQuery { - milvus::json json_obj; - // std::string field_name; - // std::vector field_value; - // float boost; -}; -using TermQueryPtr = std::shared_ptr; - -struct CompareExpr { - CompareOperator compare_operator; - std::string operand; -}; - -struct RangeQuery { - milvus::json json_obj; - // std::string field_name; - // std::vector compare_expr; - // float boost; -}; -using RangeQueryPtr = std::shared_ptr; - -struct VectorRecord { - size_t vector_count; - std::vector float_data; - std::vector binary_data; -}; - -struct VectorQuery { - std::string field_name; - milvus::json extra_params = {}; - int64_t topk; - int64_t nq; - std::string metric_type = ""; - float boost; - VectorRecord query_vector; -}; -using VectorQueryPtr = std::shared_ptr; - -struct LeafQuery; -using LeafQueryPtr = std::shared_ptr; - -struct BinaryQuery; -using BinaryQueryPtr = std::shared_ptr; - -struct GeneralQuery { - LeafQueryPtr leaf; - BinaryQueryPtr bin = std::make_shared(); -}; -using GeneralQueryPtr = std::shared_ptr; - -struct LeafQuery { - TermQueryPtr term_query; - RangeQueryPtr range_query; - std::string vector_placeholder; - float query_boost; -}; - -struct BinaryQuery { - GeneralQueryPtr left_query; - GeneralQueryPtr right_query; - QueryRelation relation; - float query_boost; - bool is_not = false; -}; - -struct Query { - GeneralQueryPtr root; - std::unordered_map vectors; - - std::string collection_id; - std::vector partitions; - std::vector field_names; - std::set index_fields; - std::unordered_map metric_types; - std::string index_type; -}; - -using QueryPtr = std::shared_ptr; - -} // namespace query_old - -namespace query { -struct QueryDeprecated { - int64_t num_queries; // - int topK; // topK of queries - std::string - field_name; // must be fakevec, whose data_type must be VEC_FLOAT(DIM) - std::vector query_raw_data; // must be size of num_queries * DIM -}; - -// std::unique_ptr CreateNaiveQueryPtr(int64_t num_queries, int topK, std::string& field_name, const float* -// raw_data) { -// return std:: -//} - -using QueryDeprecatedPtr = std::shared_ptr; -} // namespace query -} // namespace milvus diff --git a/internal/core/src/query/deprecated/ParserDeprecated.cpp b/internal/core/src/query/deprecated/ParserDeprecated.cpp deleted file mode 100644 index 149683f434..0000000000 --- a/internal/core/src/query/deprecated/ParserDeprecated.cpp +++ /dev/null @@ -1,295 +0,0 @@ -// Copyright (C) 2019-2020 Zilliz. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software distributed under the License -// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -// or implied. See the License for the specific language governing permissions and limitations under the License - -#include -#include -#include "ParserDeprecated.h" - -namespace milvus::wtf { -using google::protobuf::RepeatedField; -using google::protobuf::RepeatedPtrField; -#if 1 -#if 0 -void -CopyRowRecords(const RepeatedPtrField& grpc_records, - const RepeatedField& grpc_id_array, - engine::VectorsData& vectors - ) { - // step 1: copy vector data - int64_t float_data_size = 0, binary_data_size = 0; - - for (auto& record : grpc_records) { - float_data_size += record.float_data_size(); - binary_data_size += record.binary_data().size(); - } - - std::vector float_array(float_data_size, 0.0f); - std::vector binary_array(binary_data_size, 0); - int64_t offset = 0; - if (float_data_size > 0) { - for (auto& record : grpc_records) { - memcpy(&float_array[offset], record.float_data().data(), record.float_data_size() * sizeof(float)); - offset += record.float_data_size(); - } - } else if (binary_data_size > 0) { - for (auto& record : grpc_records) { - memcpy(&binary_array[offset], record.binary_data().data(), record.binary_data().size()); - offset += record.binary_data().size(); - } - } - - // step 2: copy id array - std::vector id_array; - if (grpc_id_array.size() > 0) { - id_array.resize(grpc_id_array.size()); - memcpy(id_array.data(), grpc_id_array.data(), grpc_id_array.size() * sizeof(int64_t)); - } - - // step 3: construct vectors - vectors.vector_count_ = grpc_records.size(); - vectors.float_data_.swap(float_array); - vectors.binary_data_.swap(binary_array); - vectors.id_array_.swap(id_array); -} -#endif - -Status -ProcessLeafQueryJson(const milvus::json& query_json, - query_old::BooleanQueryPtr& query, - std::string& field_name) { -#if 1 - if (query_json.contains("term")) { - auto leaf_query = std::make_shared(); - auto term_query = std::make_shared(); - milvus::json json_obj = query_json["term"]; - JSON_NULL_CHECK(json_obj); - JSON_OBJECT_CHECK(json_obj); - term_query->json_obj = json_obj; - milvus::json::iterator json_it = json_obj.begin(); - field_name = json_it.key(); - leaf_query->term_query = term_query; - query->AddLeafQuery(leaf_query); - } else if (query_json.contains("range")) { - auto leaf_query = std::make_shared(); - auto range_query = std::make_shared(); - milvus::json json_obj = query_json["range"]; - JSON_NULL_CHECK(json_obj); - JSON_OBJECT_CHECK(json_obj); - range_query->json_obj = json_obj; - milvus::json::iterator json_it = json_obj.begin(); - field_name = json_it.key(); - - leaf_query->range_query = range_query; - query->AddLeafQuery(leaf_query); - } else if (query_json.contains("vector")) { - auto leaf_query = std::make_shared(); - auto vector_json = query_json["vector"]; - JSON_NULL_CHECK(vector_json); - - leaf_query->vector_placeholder = vector_json.get(); - query->AddLeafQuery(leaf_query); - } else { - return Status{SERVER_INVALID_ARGUMENT, "Leaf query get wrong key"}; - } -#endif - return Status::OK(); -} - -Status -ProcessBooleanQueryJson(const milvus::json& query_json, - query_old::BooleanQueryPtr& boolean_query, - query_old::QueryPtr& query_ptr) { -#if 1 - if (query_json.empty()) { - return Status{SERVER_INVALID_ARGUMENT, "BoolQuery is null"}; - } - for (auto& el : query_json.items()) { - if (el.key() == "must") { - boolean_query->SetOccur(query_old::Occur::MUST); - auto must_json = el.value(); - if (!must_json.is_array()) { - std::string msg = "Must json string is not an array"; - return Status{SERVER_INVALID_DSL_PARAMETER, msg}; - } - - for (auto& json : must_json) { - auto must_query = std::make_shared(); - if (json.contains("must") || json.contains("should") || - json.contains("must_not")) { - STATUS_CHECK( - ProcessBooleanQueryJson(json, must_query, query_ptr)); - boolean_query->AddBooleanQuery(must_query); - } else { - std::string field_name; - STATUS_CHECK( - ProcessLeafQueryJson(json, boolean_query, field_name)); - if (!field_name.empty()) { - query_ptr->index_fields.insert(field_name); - } - } - } - } else if (el.key() == "should") { - boolean_query->SetOccur(query_old::Occur::SHOULD); - auto should_json = el.value(); - if (!should_json.is_array()) { - std::string msg = "Should json string is not an array"; - return Status{SERVER_INVALID_DSL_PARAMETER, msg}; - } - - for (auto& json : should_json) { - auto should_query = std::make_shared(); - if (json.contains("must") || json.contains("should") || - json.contains("must_not")) { - STATUS_CHECK( - ProcessBooleanQueryJson(json, should_query, query_ptr)); - boolean_query->AddBooleanQuery(should_query); - } else { - std::string field_name; - STATUS_CHECK( - ProcessLeafQueryJson(json, boolean_query, field_name)); - if (!field_name.empty()) { - query_ptr->index_fields.insert(field_name); - } - } - } - } else if (el.key() == "must_not") { - boolean_query->SetOccur(query_old::Occur::MUST_NOT); - auto should_json = el.value(); - if (!should_json.is_array()) { - std::string msg = "Must_not json string is not an array"; - return Status{SERVER_INVALID_DSL_PARAMETER, msg}; - } - - for (auto& json : should_json) { - if (json.contains("must") || json.contains("should") || - json.contains("must_not")) { - auto must_not_query = - std::make_shared(); - STATUS_CHECK(ProcessBooleanQueryJson( - json, must_not_query, query_ptr)); - boolean_query->AddBooleanQuery(must_not_query); - } else { - std::string field_name; - STATUS_CHECK( - ProcessLeafQueryJson(json, boolean_query, field_name)); - if (!field_name.empty()) { - query_ptr->index_fields.insert(field_name); - } - } - } - } else { - std::string msg = - "BoolQuery json string does not include bool query"; - return Status{SERVER_INVALID_DSL_PARAMETER, msg}; - } - } -#endif - return Status::OK(); -} - -Status -DeserializeJsonToBoolQuery(const google::protobuf::RepeatedPtrField< - ::milvus::grpc::VectorParam>& vector_params, - const std::string_view dsl_string, - query_old::BooleanQueryPtr& boolean_query, - query_old::QueryPtr& query_ptr) { -#if 1 - try { - milvus::json dsl_json = Json::parse(dsl_string); - - if (dsl_json.empty()) { - return Status{SERVER_INVALID_ARGUMENT, "Query dsl is null"}; - } - auto status = Status::OK(); - if (vector_params.empty()) { - return Status(SERVER_INVALID_DSL_PARAMETER, - "DSL must include vector query"); - } - for (const auto& vector_param : vector_params) { - const std::string_view vector_string = vector_param.json(); - milvus::json vector_json = Json::parse(vector_string); - milvus::json::iterator it = vector_json.begin(); - std::string placeholder = it.key(); - - auto vector_query = std::make_shared(); - milvus::json::iterator vector_param_it = it.value().begin(); - if (vector_param_it != it.value().end()) { - const std::string_view field_name = vector_param_it.key(); - vector_query->field_name = field_name; - milvus::json param_json = vector_param_it.value(); - int64_t topk = param_json["topk"]; - // STATUS_CHECK(server::ValidateSearchTopk(topk)); - vector_query->topk = topk; - if (param_json.contains("metric_type")) { - std::string metric_type = param_json["metric_type"]; - vector_query->metric_type = metric_type; - query_ptr->metric_types.insert( - {field_name, param_json["metric_type"]}); - } - if (!vector_param_it.value()["params"].empty()) { - vector_query->extra_params = - vector_param_it.value()["params"]; - } - query_ptr->index_fields.insert(field_name); - } - - engine::VectorsData vector_data; - CopyRowRecords( - vector_param.row_record().records(), - google::protobuf::RepeatedField(), - vector_data); - vector_query->query_vector.vector_count = vector_data.vector_count_; - vector_query->query_vector.binary_data.swap( - vector_data.binary_data_); - vector_query->query_vector.float_data.swap(vector_data.float_data_); - - query_ptr->vectors.insert( - std::make_pair(placeholder, vector_query)); - } - if (dsl_json.contains("bool")) { - auto boolean_query_json = dsl_json["bool"]; - JSON_NULL_CHECK(boolean_query_json); - status = ProcessBooleanQueryJson( - boolean_query_json, boolean_query, query_ptr); - if (!status.ok()) { - return Status(SERVER_INVALID_DSL_PARAMETER, - "DSL does not include bool"); - } - } else { - return Status(SERVER_INVALID_DSL_PARAMETER, - "DSL does not include bool query"); - } - return Status::OK(); - } catch (std::exception& e) { - return Status(SERVER_INVALID_DSL_PARAMETER, e.what()); - } -#endif - return Status::OK(); -} - -#endif -query_old::QueryPtr -Transformer(proto::service::Query* request) { - query_old::BooleanQueryPtr boolean_query = - std::make_shared(); - query_old::QueryPtr query_ptr = std::make_shared(); -#if 0 - query_ptr->collection_id = request->collection_name(); - auto status = DeserializeJsonToBoolQuery(request->placeholders(), request->dsl(), boolean_query, query_ptr); - status = query_old::ValidateBooleanQuery(boolean_query); - query_old::GeneralQueryPtr general_query = std::make_shared(); - query_old::GenBinaryQuery(boolean_query, general_query->bin); - query_ptr->root = general_query; -#endif - return query_ptr; -} - -} // namespace milvus::wtf diff --git a/internal/core/src/query/deprecated/ParserDeprecated.h b/internal/core/src/query/deprecated/ParserDeprecated.h deleted file mode 100644 index adf079b56e..0000000000 --- a/internal/core/src/query/deprecated/ParserDeprecated.h +++ /dev/null @@ -1,23 +0,0 @@ -// Copyright (C) 2019-2020 Zilliz. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software distributed under the License -// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -// or implied. See the License for the specific language governing permissions and limitations under the License - -#pragma once -#include "pb/milvus.pb.h" -#include "query/deprecated/BooleanQuery.h" -#include "query/deprecated/BinaryQuery.h" -#include "query/deprecated/GeneralQuery.h" - -namespace milvus::wtf { - -query_old::QueryPtr -Transformer(proto::milvus::SearchRequest* query); - -} // namespace milvus::wtf diff --git a/internal/core/src/query/deprecated/ValidationUtil.cpp b/internal/core/src/query/deprecated/ValidationUtil.cpp deleted file mode 100644 index 49ab41d9af..0000000000 --- a/internal/core/src/query/deprecated/ValidationUtil.cpp +++ /dev/null @@ -1,523 +0,0 @@ -// Copyright (C) 2019-2020 Zilliz. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software distributed under the License -// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -// or implied. See the License for the specific language governing permissions and limitations under the License. - -#include "ValidationUtil.h" -#include "config/ServerConfig.h" -#include "utils/Log.h" -#include "utils/StringHelpFunctions.h" - -#include -#include -#include -#include - -namespace milvus { -namespace server { - -namespace { - -Status -CheckParameterRange(const milvus::json& json_params, - const std::string_view param_name, - int64_t min, - int64_t max, - bool min_close = true, - bool max_closed = true) { - if (json_params.find(param_name) == json_params.end()) { - std::string msg = "Parameter list must contain: "; - msg += param_name; - LOG_SERVER_ERROR_ << msg; - return Status(SERVER_INVALID_ARGUMENT, msg); - } - - try { - int64_t value = json_params[param_name]; - bool min_err = min_close ? value < min : value <= min; - bool max_err = max_closed ? value > max : value >= max; - if (min_err || max_err) { - std::string msg = "Invalid " + param_name + - " value: " + std::to_string(value) + - ". Valid range is " + (min_close ? "[" : "(") + - std::to_string(min) + ", " + std::to_string(max) + - (max_closed ? "]" : ")"); - LOG_SERVER_ERROR_ << msg; - return Status(SERVER_INVALID_ARGUMENT, msg); - } - } catch (std::exception& e) { - std::string msg = "Invalid " + param_name + ": "; - msg += e.what(); - LOG_SERVER_ERROR_ << msg; - return Status(SERVER_INVALID_ARGUMENT, msg); - } - - return Status::OK(); -} - -Status -CheckParameterExistence(const milvus::json& json_params, - const std::string_view param_name) { - if (json_params.find(param_name) == json_params.end()) { - std::string msg = "Parameter list must contain: "; - msg += param_name; - LOG_SERVER_ERROR_ << msg; - return Status(SERVER_INVALID_ARGUMENT, msg); - } - - try { - int64_t value = json_params[param_name]; - if (value < 0) { - std::string msg = - "Invalid " + param_name + " value: " + std::to_string(value); - LOG_SERVER_ERROR_ << msg; - return Status(SERVER_INVALID_ARGUMENT, msg); - } - } catch (std::exception& e) { - std::string msg = "Invalid " + param_name + ": "; - msg += e.what(); - LOG_SERVER_ERROR_ << msg; - return Status(SERVER_INVALID_ARGUMENT, msg); - } - - return Status::OK(); -} - -} // namespace - -Status -ValidateCollectionName(const std::string_view collection_name) { - // Collection name shouldn't be empty. - if (collection_name.empty()) { - std::string msg = "Collection name should not be empty."; - LOG_SERVER_ERROR_ << msg; - return Status(SERVER_INVALID_COLLECTION_NAME, msg); - } - - std::string invalid_msg = - "Invalid collection name: " + collection_name + ". "; - // Collection name size shouldn't exceed engine::MAX_NAME_LENGTH. - if (collection_name.size() > engine::MAX_NAME_LENGTH) { - std::string msg = - invalid_msg + "The length of a collection name must be less than " + - std::to_string(engine::MAX_NAME_LENGTH) + " characters."; - LOG_SERVER_ERROR_ << msg; - return Status(SERVER_INVALID_COLLECTION_NAME, msg); - } - - // Collection name first character should be underscore or character. - char first_char = collection_name[0]; - if (first_char != '_' && std::isalpha(first_char) == 0) { - std::string msg = invalid_msg + - "The first character of a collection name must be an " - "underscore or letter."; - LOG_SERVER_ERROR_ << msg; - return Status(SERVER_INVALID_COLLECTION_NAME, msg); - } - - int64_t table_name_size = collection_name.size(); - for (int64_t i = 1; i < table_name_size; ++i) { - char name_char = collection_name[i]; - if (name_char != '_' && name_char != '$' && - std::isalnum(name_char) == 0) { - std::string msg = invalid_msg + - "Collection name can only contain numbers, " - "letters, and underscores."; - LOG_SERVER_ERROR_ << msg; - return Status(SERVER_INVALID_COLLECTION_NAME, msg); - } - } - - return Status::OK(); -} - -Status -ValidateFieldName(const std::string_view field_name) { - // Field name shouldn't be empty. - if (field_name.empty()) { - std::string msg = "Field name should not be empty."; - LOG_SERVER_ERROR_ << msg; - return Status(SERVER_INVALID_FIELD_NAME, msg); - } - - std::string invalid_msg = "Invalid field name: " + field_name + ". "; - // Field name size shouldn't exceed engine::MAX_NAME_LENGTH. - if (field_name.size() > engine::MAX_NAME_LENGTH) { - std::string msg = - invalid_msg + "The length of a field name must be less than " + - std::to_string(engine::MAX_NAME_LENGTH) + " characters."; - LOG_SERVER_ERROR_ << msg; - return Status(SERVER_INVALID_FIELD_NAME, msg); - } - - // Field name first character should be underscore or character. - char first_char = field_name[0]; - if (first_char != '_' && std::isalpha(first_char) == 0) { - std::string msg = invalid_msg + - "The first character of a field name must be an " - "underscore or letter."; - LOG_SERVER_ERROR_ << msg; - return Status(SERVER_INVALID_FIELD_NAME, msg); - } - - int64_t field_name_size = field_name.size(); - for (int64_t i = 1; i < field_name_size; ++i) { - char name_char = field_name[i]; - if (name_char != '_' && std::isalnum(name_char) == 0) { - std::string msg = invalid_msg + - "Field name cannot only contain numbers, " - "letters, and underscores."; - LOG_SERVER_ERROR_ << msg; - return Status(SERVER_INVALID_FIELD_NAME, msg); - } - } - - return Status::OK(); -} - -Status -ValidateVectorIndexType(std::string& index_type, bool is_binary) { - // Index name shouldn't be empty. - if (index_type.empty()) { - std::string msg = "Index type should not be empty."; - LOG_SERVER_ERROR_ << msg; - return Status(SERVER_INVALID_FIELD_NAME, msg); - } - - // string case insensitive - std::transform( - index_type.begin(), index_type.end(), index_type.begin(), ::toupper); - - static std::set s_vector_index_type = { - knowhere::IndexEnum::INVALID, - knowhere::IndexEnum::INDEX_FAISS_IDMAP, - knowhere::IndexEnum::INDEX_FAISS_IVFFLAT, - knowhere::IndexEnum::INDEX_FAISS_IVFPQ, - knowhere::IndexEnum::INDEX_FAISS_IVFSQ8, - knowhere::IndexEnum::INDEX_HNSW, - }; - - static std::set s_binary_index_types = { - knowhere::IndexEnum::INDEX_FAISS_BIN_IDMAP, - knowhere::IndexEnum::INDEX_FAISS_BIN_IVFFLAT, - }; - - std::set& index_types = - is_binary ? s_binary_index_types : s_vector_index_type; - if (index_types.find(index_type) == index_types.end()) { - std::string msg = "Invalid index type: " + index_type; - LOG_SERVER_ERROR_ << msg; - return Status(SERVER_INVALID_INDEX_TYPE, msg); - } - - return Status::OK(); -} - -Status -ValidateStructuredIndexType(std::string& index_type) { - // Index name shouldn't be empty. - if (index_type.empty()) { - std::string msg = "Index type should not be empty."; - LOG_SERVER_ERROR_ << msg; - return Status(SERVER_INVALID_FIELD_NAME, msg); - } - - // string case insensitive - std::transform( - index_type.begin(), index_type.end(), index_type.begin(), ::toupper); - - static std::set s_index_types = { - engine::DEFAULT_STRUCTURED_INDEX, - }; - - if (s_index_types.find(index_type) == s_index_types.end()) { - std::string msg = "Invalid index type: " + index_type; - LOG_SERVER_ERROR_ << msg; - return Status(SERVER_INVALID_INDEX_TYPE, msg); - } - - return Status::OK(); -} - -Status -ValidateDimension(int64_t dim, bool is_binary) { - if (dim <= 0 || dim > engine::MAX_DIMENSION) { - std::string msg = "Invalid dimension: " + std::to_string(dim) + - ". Should be in range 1 ~ " + - std::to_string(engine::MAX_DIMENSION) + "."; - LOG_SERVER_ERROR_ << msg; - return Status(SERVER_INVALID_VECTOR_DIMENSION, msg); - } - - if (is_binary && (dim % 8) != 0) { - std::string msg = "Invalid dimension: " + std::to_string(dim) + - ". Should be multiple of 8."; - LOG_SERVER_ERROR_ << msg; - return Status(SERVER_INVALID_VECTOR_DIMENSION, msg); - } - - return Status::OK(); -} - -Status -ValidateIndexParams(const milvus::json& index_params, - int64_t dimension, - const std::string_view index_type) { - if (engine::utils::IsFlatIndexType(index_type)) { - return Status::OK(); - } else if (index_type == knowhere::IndexEnum::INDEX_FAISS_IVFFLAT || - index_type == knowhere::IndexEnum::INDEX_FAISS_IVFSQ8 || - index_type == knowhere::IndexEnum::INDEX_FAISS_BIN_IVFFLAT) { - auto status = CheckParameterRange( - index_params, knowhere::IndexParams::nlist, 1, 65536); - if (!status.ok()) { - return status; - } - } else if (index_type == knowhere::IndexEnum::INDEX_FAISS_IVFPQ) { - auto status = CheckParameterRange( - index_params, knowhere::IndexParams::nlist, 1, 65536); - if (!status.ok()) { - return status; - } - - status = - CheckParameterExistence(index_params, knowhere::IndexParams::m); - if (!status.ok()) { - return status; - } - - // special check for 'm' parameter - int64_t m_value = index_params[knowhere::IndexParams::m]; - if (!milvus::knowhere::IVFPQConfAdapter::GetValidCPUM(dimension, - m_value)) { - std::string msg = "Invalid m, dimension can't not be divided by m "; - LOG_SERVER_ERROR_ << msg; - return Status(SERVER_INVALID_ARGUMENT, msg); - } - /*std::vector resset; - milvus::knowhere::IVFPQConfAdapter::GetValidMList(dimension, resset); - int64_t m_value = index_params[knowhere::IndexParams::m]; - if (resset.empty()) { - std::string msg = "Invalid collection dimension, unable to get reasonable values for 'm'"; - LOG_SERVER_ERROR_ << msg; - return Status(SERVER_INVALID_COLLECTION_DIMENSION, msg); - } - - auto iter = std::find(std::begin(resset), std::end(resset), m_value); - if (iter == std::end(resset)) { - std::string msg = - "Invalid " + std::string(knowhere::IndexParams::m) + ", must be one of the following values: "; - for (size_t i = 0; i < resset.size(); i++) { - if (i != 0) { - msg += ","; - } - msg += std::to_string(resset[i]); - } - - LOG_SERVER_ERROR_ << msg; - return Status(SERVER_INVALID_ARGUMENT, msg); - }*/ - } else if (index_type == knowhere::IndexEnum::INDEX_HNSW) { - auto status = - CheckParameterRange(index_params, knowhere::IndexParams::M, 4, 64); - if (!status.ok()) { - return status; - } - status = CheckParameterRange( - index_params, knowhere::IndexParams::efConstruction, 8, 512); - if (!status.ok()) { - return status; - } - } - - return Status::OK(); -} - -Status -ValidateSegmentRowCount(int64_t segment_row_count) { - int64_t min = config.engine.build_index_threshold(); - int max = engine::MAX_SEGMENT_ROW_COUNT; - if (segment_row_count < min || segment_row_count > max) { - std::string msg = - "Invalid segment row count: " + std::to_string(segment_row_count) + - ". " + "Should be in range " + std::to_string(min) + " ~ " + - std::to_string(max) + "."; - LOG_SERVER_ERROR_ << msg; - return Status(SERVER_INVALID_SEGMENT_ROW_COUNT, msg); - } - return Status::OK(); -} - -Status -ValidateIndexMetricType(const std::string_view metric_type, - const std::string_view index_type) { - if (engine::utils::IsFlatIndexType(index_type)) { - // pass - } else if (index_type == knowhere::IndexEnum::INDEX_FAISS_BIN_IVFFLAT) { - // binary - if (metric_type != knowhere::Metric::HAMMING && - metric_type != knowhere::Metric::JACCARD) { - std::string msg = "Index metric type " + metric_type + - " does not match index type " + index_type; - LOG_SERVER_ERROR_ << msg; - return Status(SERVER_INVALID_ARGUMENT, msg); - } - } else { - // float - if (metric_type != knowhere::Metric::L2 && - metric_type != knowhere::Metric::IP) { - std::string msg = "Index metric type " + metric_type + - " does not match index type " + index_type; - LOG_SERVER_ERROR_ << msg; - return Status(SERVER_INVALID_ARGUMENT, msg); - } - } - - return Status::OK(); -} - -Status -ValidateSearchMetricType(const std::string_view metric_type, bool is_binary) { - if (is_binary) { - // binary - if (metric_type == knowhere::Metric::L2 || - metric_type == knowhere::Metric::IP) { - std::string msg = - "Cannot search binary entities with index metric type " + - metric_type; - LOG_SERVER_ERROR_ << msg; - return Status(SERVER_INVALID_ARGUMENT, msg); - } - } else { - // float - if (metric_type == knowhere::Metric::HAMMING || - metric_type == knowhere::Metric::JACCARD) { - std::string msg = - "Cannot search float entities with index metric type " + - metric_type; - LOG_SERVER_ERROR_ << msg; - return Status(SERVER_INVALID_ARGUMENT, msg); - } - } - - return Status::OK(); -} - -Status -ValidateSearchTopk(int64_t top_k) { - if (top_k <= 0 || top_k > QUERY_MAX_TOPK) { - std::string msg = "Invalid topk: " + std::to_string(top_k) + ". " + - "The topk must be within the range of 1 ~ 16384."; - LOG_SERVER_ERROR_ << msg; - return Status(SERVER_INVALID_TOPK, msg); - } - - return Status::OK(); -} - -Status -ValidatePartitionTags(const std::vector& partition_tags) { - for (const std::string_view tag : partition_tags) { - // Partition nametag shouldn't be empty. - if (tag.empty()) { - std::string msg = "Partition tag should not be empty."; - LOG_SERVER_ERROR_ << msg; - return Status(SERVER_INVALID_PARTITION_TAG, msg); - } - - std::string invalid_msg = "Invalid partition tag: " + tag + ". "; - // Partition tag size shouldn't exceed 255. - if (tag.size() > engine::MAX_NAME_LENGTH) { - std::string msg = invalid_msg + - "The length of a partition tag must be less than " - "255 characters."; - LOG_SERVER_ERROR_ << msg; - return Status(SERVER_INVALID_PARTITION_TAG, msg); - } - - // Partition tag first character should be underscore or character. - char first_char = tag[0]; - if (first_char != '_' && std::isalnum(first_char) == 0) { - std::string msg = invalid_msg + - "The first character of a partition tag must be " - "an underscore or letter."; - LOG_SERVER_ERROR_ << msg; - return Status(SERVER_INVALID_PARTITION_TAG, msg); - } - - int64_t tag_size = tag.size(); - for (int64_t i = 1; i < tag_size; ++i) { - char name_char = tag[i]; - if (name_char != '_' && name_char != '$' && - std::isalnum(name_char) == 0) { - std::string msg = invalid_msg + - "Partition tag can only contain numbers, " - "letters, and underscores."; - LOG_SERVER_ERROR_ << msg; - return Status(SERVER_INVALID_PARTITION_TAG, msg); - } - } - -#if 0 - // trim side-blank of tag, only compare valid characters - // for example: " ab cd " is treated as "ab cd" - std::string valid_tag = tag; - StringHelpFunctions::TrimStringBlank(valid_tag); - if (valid_tag.empty()) { - std::string msg = "Invalid partition tag: " + valid_tag + ". " + "Partition tag should not be empty."; - LOG_SERVER_ERROR_ << msg; - return Status(SERVER_INVALID_PARTITION_TAG, msg); - } - - // max length of partition tag - if (valid_tag.length() > engine::MAX_NAME_LENGTH) { - std::string msg = "Invalid partition tag: " + valid_tag + ". " + - "Partition tag exceed max length: " + std::to_string(engine::MAX_NAME_LENGTH); - LOG_SERVER_ERROR_ << msg; - return Status(SERVER_INVALID_PARTITION_TAG, msg); - } -#endif - } - - return Status::OK(); -} - -Status -ValidateInsertDataSize(const InsertParam& insert_param) { - int64_t chunk_size = 0; - for (auto& pair : insert_param.fields_data_) { - for (auto& data : pair.second) { - chunk_size += data.second; - } - } - - if (chunk_size > engine::MAX_INSERT_DATA_SIZE) { - std::string msg = - "The amount of data inserted each time cannot exceed " + - std::to_string(engine::MAX_INSERT_DATA_SIZE / engine::MB) + " MB"; - return Status(SERVER_INVALID_ROWRECORD_ARRAY, msg); - } - - return Status::OK(); -} - -Status -ValidateCompactThreshold(double threshold) { - if (threshold > 1.0 || threshold < 0.0) { - std::string msg = - "Invalid compact threshold: " + std::to_string(threshold) + - ". Should be in range [0.0, 1.0]"; - return Status(SERVER_INVALID_ROWRECORD_ARRAY, msg); - } - - return Status::OK(); -} - -} // namespace server -} // namespace milvus diff --git a/internal/core/src/query/deprecated/ValidationUtil.h b/internal/core/src/query/deprecated/ValidationUtil.h deleted file mode 100644 index ae0ae09be4..0000000000 --- a/internal/core/src/query/deprecated/ValidationUtil.h +++ /dev/null @@ -1,71 +0,0 @@ -// Copyright (C) 2019-2020 Zilliz. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software distributed under the License -// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -// or implied. See the License for the specific language governing permissions and limitations under the License. - -#pragma once - -#include "db/Types.h" -#include "server/delivery/request/Types.h" -#include "utils/Json.h" -#include "utils/Status.h" - -#include -#include - -namespace milvus { -namespace server { - -constexpr int64_t QUERY_MAX_TOPK = 16384; -constexpr int64_t GPU_QUERY_MAX_TOPK = 2048; -constexpr int64_t GPU_QUERY_MAX_NPROBE = 2048; - -extern Status -ValidateCollectionName(const std::string_view collection_name); - -extern Status -ValidateFieldName(const std::string_view field_name); - -extern Status -ValidateDimension(int64_t dimension, bool is_binary); - -extern Status -ValidateVectorIndexType(std::string& index_type, bool is_binary); - -extern Status -ValidateStructuredIndexType(std::string& index_type); - -extern Status -ValidateIndexParams(const milvus::json& index_params, - int64_t dimension, - const std::string_view index_type); - -extern Status -ValidateSegmentRowCount(int64_t segment_row_count); - -extern Status -ValidateIndexMetricType(const std::string_view metric_type, - const std::string_view index_type); - -extern Status -ValidateSearchMetricType(const std::string_view metric_type, bool is_binary); - -extern Status -ValidateSearchTopk(int64_t top_k); - -extern Status -ValidatePartitionTags(const std::vector& partition_tags); - -extern Status -ValidateInsertDataSize(const InsertParam& insert_param); - -extern Status -ValidateCompactThreshold(double threshold); -} // namespace server -} // namespace milvus diff --git a/internal/core/src/query/generated/ExecPlanNodeVisitor.h b/internal/core/src/query/generated/ExecPlanNodeVisitor.h index a48446967a..cd1aa91ce1 100644 --- a/internal/core/src/query/generated/ExecPlanNodeVisitor.h +++ b/internal/core/src/query/generated/ExecPlanNodeVisitor.h @@ -12,7 +12,7 @@ #pragma once // Generated File // DO NOT EDIT -#include "utils/Json.h" +#include "common/Json.h" #include "query/PlanImpl.h" #include "segcore/SegmentGrowing.h" #include diff --git a/internal/core/src/query/generated/ShowExprVisitor.h b/internal/core/src/query/generated/ShowExprVisitor.h index 9c5ca313b4..64532a00dd 100644 --- a/internal/core/src/query/generated/ShowExprVisitor.h +++ b/internal/core/src/query/generated/ShowExprVisitor.h @@ -77,6 +77,6 @@ class ShowExprVisitor : public ExprVisitor { } private: - std::optional json_opt_; + std::optional json_opt_; }; } // namespace milvus::query diff --git a/internal/core/src/query/generated/ShowPlanNodeVisitor.h b/internal/core/src/query/generated/ShowPlanNodeVisitor.h index d57b2fd08e..4a8743763b 100644 --- a/internal/core/src/query/generated/ShowPlanNodeVisitor.h +++ b/internal/core/src/query/generated/ShowPlanNodeVisitor.h @@ -12,8 +12,8 @@ #pragma once // Generated File // DO NOT EDIT -#include "exceptions/EasyAssert.h" -#include "utils/Json.h" +#include "common/EasyAssert.h" +#include "common/Json.h" #include #include diff --git a/internal/core/src/query/generated/VerifyPlanNodeVisitor.h b/internal/core/src/query/generated/VerifyPlanNodeVisitor.h index f93f0c0590..6b9653d278 100644 --- a/internal/core/src/query/generated/VerifyPlanNodeVisitor.h +++ b/internal/core/src/query/generated/VerifyPlanNodeVisitor.h @@ -12,7 +12,7 @@ #pragma once // Generated File // DO NOT EDIT -#include "utils/Json.h" +#include "common/Json.h" #include "query/PlanImpl.h" #include "segcore/SegmentGrowing.h" #include diff --git a/internal/core/src/query/visitors/ExecExprVisitor.cpp b/internal/core/src/query/visitors/ExecExprVisitor.cpp index 034e8de2a0..998852613f 100644 --- a/internal/core/src/query/visitors/ExecExprVisitor.cpp +++ b/internal/core/src/query/visitors/ExecExprVisitor.cpp @@ -27,7 +27,7 @@ #include "arrow/type_fwd.h" #include "common/Json.h" #include "common/Types.h" -#include "exceptions/EasyAssert.h" +#include "common/EasyAssert.h" #include "pb/plan.pb.h" #include "query/ExprImpl.h" #include "query/Relational.h" diff --git a/internal/core/src/query/visitors/ExecPlanNodeVisitor.cpp b/internal/core/src/query/visitors/ExecPlanNodeVisitor.cpp index b0e335fcb5..2b1018477f 100644 --- a/internal/core/src/query/visitors/ExecPlanNodeVisitor.cpp +++ b/internal/core/src/query/visitors/ExecPlanNodeVisitor.cpp @@ -17,7 +17,7 @@ #include "query/SubSearchResult.h" #include "query/generated/ExecExprVisitor.h" #include "segcore/SegmentGrowing.h" -#include "utils/Json.h" +#include "common/Json.h" #include "log/Log.h" namespace milvus::query { diff --git a/internal/core/src/query/visitors/ShowPlanNodeVisitor.cpp b/internal/core/src/query/visitors/ShowPlanNodeVisitor.cpp index 43d4787ec7..4325f41539 100644 --- a/internal/core/src/query/visitors/ShowPlanNodeVisitor.cpp +++ b/internal/core/src/query/visitors/ShowPlanNodeVisitor.cpp @@ -11,10 +11,10 @@ #include -#include "exceptions/EasyAssert.h" +#include "common/EasyAssert.h" +#include "common/Json.h" #include "query/generated/ShowExprVisitor.h" #include "query/generated/ShowPlanNodeVisitor.h" -#include "utils/Json.h" namespace milvus::query { #if 0 diff --git a/internal/core/src/segcore/ConcurrentVector.h b/internal/core/src/segcore/ConcurrentVector.h index c34a987141..cf1b5d2405 100644 --- a/internal/core/src/segcore/ConcurrentVector.h +++ b/internal/core/src/segcore/ConcurrentVector.h @@ -30,7 +30,7 @@ #include "common/Span.h" #include "common/Types.h" #include "common/Utils.h" -#include "exceptions/EasyAssert.h" +#include "common/EasyAssert.h" #include "storage/FieldData.h" namespace milvus::segcore { diff --git a/internal/core/src/segcore/Reduce.cpp b/internal/core/src/segcore/Reduce.cpp index c5a48d9221..c1a89e6630 100644 --- a/internal/core/src/segcore/Reduce.cpp +++ b/internal/core/src/segcore/Reduce.cpp @@ -19,7 +19,7 @@ #include "SegmentInterface.h" #include "Utils.h" -#include "exceptions/EasyAssert.h" +#include "common/EasyAssert.h" #include "pkVisitor.h" namespace milvus::segcore { diff --git a/internal/core/src/segcore/Reduce.h b/internal/core/src/segcore/Reduce.h index eeeb8d9fdf..073beb07ef 100644 --- a/internal/core/src/segcore/Reduce.h +++ b/internal/core/src/segcore/Reduce.h @@ -18,7 +18,6 @@ #include #include -#include "utils/Status.h" #include "common/type_c.h" #include "common/QueryResult.h" #include "query/PlanImpl.h" @@ -84,15 +83,15 @@ class ReduceHelper { GetSearchResultDataSlice(int slice_index_); private: - std::vector slice_topKs_; + std::vector& search_results_; + milvus::query::Plan* plan_; + std::vector slice_nqs_; + std::vector slice_topKs_; int64_t total_nq_; int64_t num_segments_; int64_t num_slices_; - milvus::query::Plan* plan_; - std::vector& search_results_; - std::vector slice_nqs_prefix_sum_; // dim0: num_segments_; dim1: total_nq_; dim2: offset diff --git a/internal/core/src/segcore/ScalarIndex.cpp b/internal/core/src/segcore/ScalarIndex.cpp index bc988b0356..c5aaacdd70 100644 --- a/internal/core/src/segcore/ScalarIndex.cpp +++ b/internal/core/src/segcore/ScalarIndex.cpp @@ -9,7 +9,7 @@ // is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express // or implied. See the License for the specific language governing permissions and limitations under the License -#include "exceptions/EasyAssert.h" +#include "common/EasyAssert.h" #include "ScalarIndex.h" namespace milvus::segcore { diff --git a/internal/core/src/segcore/SealedIndexingRecord.h b/internal/core/src/segcore/SealedIndexingRecord.h index a17481f840..5b16115f96 100644 --- a/internal/core/src/segcore/SealedIndexingRecord.h +++ b/internal/core/src/segcore/SealedIndexingRecord.h @@ -19,7 +19,7 @@ #include #include "common/Types.h" -#include "exceptions/EasyAssert.h" +#include "common/EasyAssert.h" #include "index/VectorIndex.h" namespace milvus::segcore { diff --git a/internal/core/src/segcore/SegcoreConfig.cpp b/internal/core/src/segcore/SegcoreConfig.cpp index 292de265ad..2d8e4e1359 100644 --- a/internal/core/src/segcore/SegcoreConfig.cpp +++ b/internal/core/src/segcore/SegcoreConfig.cpp @@ -11,7 +11,7 @@ #include "common/Schema.h" #include "SegcoreConfig.h" -#include "utils/Json.h" +#include "common/Json.h" #include "yaml-cpp/yaml.h" namespace milvus::segcore { diff --git a/internal/core/src/segcore/SegcoreConfig.h b/internal/core/src/segcore/SegcoreConfig.h index dc2360a526..c7f3175119 100644 --- a/internal/core/src/segcore/SegcoreConfig.h +++ b/internal/core/src/segcore/SegcoreConfig.h @@ -15,9 +15,9 @@ #include #include "common/Types.h" +#include "common/Json.h" #include "index/Utils.h" -#include "exceptions/EasyAssert.h" -#include "utils/Json.h" +#include "common/EasyAssert.h" namespace milvus::segcore { diff --git a/internal/core/src/segcore/SegmentGrowing.h b/internal/core/src/segcore/SegmentGrowing.h index 64a7435002..f00b3fc3fc 100644 --- a/internal/core/src/segcore/SegmentGrowing.h +++ b/internal/core/src/segcore/SegmentGrowing.h @@ -18,7 +18,6 @@ #include "common/Schema.h" #include "common/Types.h" #include "query/Plan.h" -#include "query/deprecated/GeneralQuery.h" #include "segcore/SegmentInterface.h" namespace milvus::segcore { diff --git a/internal/core/src/segcore/SegmentGrowingImpl.cpp b/internal/core/src/segcore/SegmentGrowingImpl.cpp index b3b10ca994..c584c6ca2f 100644 --- a/internal/core/src/segcore/SegmentGrowingImpl.cpp +++ b/internal/core/src/segcore/SegmentGrowingImpl.cpp @@ -19,6 +19,7 @@ #include #include "common/Consts.h" +#include "common/EasyAssert.h" #include "common/Types.h" #include "nlohmann/json.hpp" #include "query/PlanNode.h" @@ -222,7 +223,7 @@ SegmentGrowingImpl::LoadFieldData(const LoadFieldDataInfo& infos) { reserved_offset + num_rows); } -Status +SegcoreError SegmentGrowingImpl::Delete(int64_t reserved_begin, int64_t size, const IdArray* ids, @@ -250,7 +251,7 @@ SegmentGrowingImpl::Delete(int64_t reserved_begin, // step 2: fill delete record deleted_record_.push(sort_pks, sort_timestamps.data()); - return Status::OK(); + return SegcoreError::success(); } int64_t diff --git a/internal/core/src/segcore/SegmentGrowingImpl.h b/internal/core/src/segcore/SegmentGrowingImpl.h index bc51c12b32..28db3d6322 100644 --- a/internal/core/src/segcore/SegmentGrowingImpl.h +++ b/internal/core/src/segcore/SegmentGrowingImpl.h @@ -29,10 +29,8 @@ #include "SealedIndexingRecord.h" #include "SegmentGrowing.h" -#include "exceptions/EasyAssert.h" +#include "common/EasyAssert.h" #include "query/PlanNode.h" -#include "query/deprecated/GeneralQuery.h" -#include "utils/Status.h" #include "common/IndexMeta.h" namespace milvus::segcore { @@ -50,7 +48,7 @@ class SegmentGrowingImpl : public SegmentGrowing { const InsertData* insert_data) override; // TODO: add id into delete log, possibly bitmap - Status + SegcoreError Delete(int64_t reserved_offset, int64_t size, const IdArray* pks, diff --git a/internal/core/src/segcore/SegmentInterface.h b/internal/core/src/segcore/SegmentInterface.h index 7dbb41d946..0e1b7aa8ef 100644 --- a/internal/core/src/segcore/SegmentInterface.h +++ b/internal/core/src/segcore/SegmentInterface.h @@ -83,7 +83,7 @@ class SegmentInterface { // virtual int64_t // PreDelete(int64_t size) = 0; - virtual Status + virtual SegcoreError Delete(int64_t reserved_offset, int64_t size, const IdArray* pks, diff --git a/internal/core/src/segcore/SegmentSealedImpl.cpp b/internal/core/src/segcore/SegmentSealedImpl.cpp index 683f77aecb..b5baa6a5b8 100644 --- a/internal/core/src/segcore/SegmentSealedImpl.cpp +++ b/internal/core/src/segcore/SegmentSealedImpl.cpp @@ -24,7 +24,7 @@ #include "Utils.h" #include "Types.h" #include "common/Json.h" -#include "exceptions/EasyAssert.h" +#include "common/EasyAssert.h" #include "mmap/Column.h" #include "common/Consts.h" #include "common/FieldMeta.h" @@ -36,7 +36,7 @@ #include "storage/FieldData.h" #include "storage/Util.h" #include "storage/ThreadPools.h" -#include "utils/File.h" +#include "common/File.h" #include "common/Tracer.h" namespace milvus::segcore { @@ -980,7 +980,7 @@ SegmentSealedImpl::search_ids(const IdArray& id_array, return {std::move(res_id_arr), std::move(res_offsets)}; } -Status +SegcoreError SegmentSealedImpl::Delete(int64_t reserved_offset, // deprecated int64_t size, const IdArray* ids, @@ -1007,7 +1007,7 @@ SegmentSealedImpl::Delete(int64_t reserved_offset, // deprecated } deleted_record_.push(sort_pks, sort_timestamps.data()); - return Status::OK(); + return SegcoreError::success(); } std::string diff --git a/internal/core/src/segcore/SegmentSealedImpl.h b/internal/core/src/segcore/SegmentSealedImpl.h index 2fcb4b86a9..61d60b9eea 100644 --- a/internal/core/src/segcore/SegmentSealedImpl.h +++ b/internal/core/src/segcore/SegmentSealedImpl.h @@ -28,6 +28,7 @@ #include "SealedIndexingRecord.h" #include "SegmentSealed.h" #include "TimestampIndex.h" +#include "common/EasyAssert.h" #include "mmap/Column.h" #include "index/ScalarIndex.h" #include "sys/mman.h" @@ -104,7 +105,7 @@ class SegmentSealedImpl : public SegmentSealed { std::string debug() const override; - Status + SegcoreError Delete(int64_t reserved_offset, int64_t size, const IdArray* pks, diff --git a/internal/core/src/segcore/load_field_data_c.cpp b/internal/core/src/segcore/load_field_data_c.cpp index 882fe1cee5..b0adbf0ed0 100644 --- a/internal/core/src/segcore/load_field_data_c.cpp +++ b/internal/core/src/segcore/load_field_data_c.cpp @@ -14,7 +14,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "common/CGoHelper.h" #include "common/LoadInfo.h" #include "segcore/load_field_data_c.h" @@ -25,7 +24,7 @@ NewLoadFieldDataInfo(CLoadFieldDataInfo* c_load_field_data_info) { *c_load_field_data_info = load_field_data_info.release(); return milvus::SuccessCStatus(); } catch (std::exception& e) { - return milvus::FailureCStatus(UnexpectedError, e.what()); + return milvus::FailureCStatus(milvus::UnexpectedError, e.what()); } } @@ -51,7 +50,7 @@ AppendLoadFieldInfo(CLoadFieldDataInfo c_load_field_data_info, load_field_data_info->field_infos[field_id] = binlog_info; return milvus::SuccessCStatus(); } catch (std::exception& e) { - return milvus::FailureCStatus(UnexpectedError, e.what()); + return milvus::FailureCStatus(milvus::UnexpectedError, e.what()); } } @@ -71,7 +70,7 @@ AppendLoadFieldDataPath(CLoadFieldDataInfo c_load_field_data_info, file_path); return milvus::SuccessCStatus(); } catch (std::exception& e) { - return milvus::FailureCStatus(UnexpectedError, e.what()); + return milvus::FailureCStatus(milvus::UnexpectedError, e.what()); } } diff --git a/internal/core/src/segcore/load_index_c.cpp b/internal/core/src/segcore/load_index_c.cpp index 78cdd0114d..1f6af6c95a 100644 --- a/internal/core/src/segcore/load_index_c.cpp +++ b/internal/core/src/segcore/load_index_c.cpp @@ -12,7 +12,7 @@ #include "segcore/load_index_c.h" #include "common/FieldMeta.h" -#include "exceptions/EasyAssert.h" +#include "common/EasyAssert.h" #include "index/Index.h" #include "index/IndexFactory.h" #include "index/Meta.h" @@ -31,12 +31,12 @@ NewLoadIndexInfo(CLoadIndexInfo* c_load_index_info) { *c_load_index_info = load_index_info.release(); auto status = CStatus(); - status.error_code = Success; + status.error_code = milvus::Success; status.error_msg = ""; return status; } catch (std::exception& e) { auto status = CStatus(); - status.error_code = UnexpectedError; + status.error_code = milvus::UnexpectedError; status.error_msg = strdup(e.what()); return status; } @@ -60,12 +60,12 @@ AppendIndexParam(CLoadIndexInfo c_load_index_info, load_index_info->index_params[index_key] = index_value; auto status = CStatus(); - status.error_code = Success; + status.error_code = milvus::Success; status.error_msg = ""; return status; } catch (std::exception& e) { auto status = CStatus(); - status.error_code = UnexpectedError; + status.error_code = milvus::UnexpectedError; status.error_msg = strdup(e.what()); return status; } @@ -90,12 +90,12 @@ AppendFieldInfo(CLoadIndexInfo c_load_index_info, load_index_info->mmap_dir_path = std::string(mmap_dir_path); auto status = CStatus(); - status.error_code = Success; + status.error_code = milvus::Success; status.error_msg = ""; return status; } catch (std::exception& e) { auto status = CStatus(); - status.error_code = UnexpectedError; + status.error_code = milvus::UnexpectedError; status.error_msg = strdup(e.what()); return status; } @@ -151,12 +151,12 @@ appendVecIndex(CLoadIndexInfo c_load_index_info, CBinarySet c_binary_set) { index_info, file_manager); load_index_info->index->Load(*binary_set, config); auto status = CStatus(); - status.error_code = Success; + status.error_code = milvus::Success; status.error_msg = ""; return status; } catch (std::exception& e) { auto status = CStatus(); - status.error_code = UnexpectedError; + status.error_code = milvus::UnexpectedError; status.error_msg = strdup(e.what()); return status; } @@ -184,12 +184,12 @@ appendScalarIndex(CLoadIndexInfo c_load_index_info, CBinarySet c_binary_set) { nullptr); load_index_info->index->Load(*binary_set); auto status = CStatus(); - status.error_code = Success; + status.error_code = milvus::Success; status.error_msg = ""; return status; } catch (std::exception& e) { auto status = CStatus(); - status.error_code = UnexpectedError; + status.error_code = milvus::UnexpectedError; status.error_msg = strdup(e.what()); return status; } @@ -269,12 +269,12 @@ AppendIndexV2(CLoadIndexInfo c_load_index_info) { load_index_info->index->Load(config); auto status = CStatus(); - status.error_code = Success; + status.error_code = milvus::Success; status.error_msg = ""; return status; } catch (std::exception& e) { auto status = CStatus(); - status.error_code = UnexpectedError; + status.error_code = milvus::UnexpectedError; status.error_msg = strdup(e.what()); return status; } @@ -289,12 +289,12 @@ AppendIndexFilePath(CLoadIndexInfo c_load_index_info, const char* c_file_path) { load_index_info->index_files.emplace_back(index_file_path); auto status = CStatus(); - status.error_code = Success; + status.error_code = milvus::Success; status.error_msg = ""; return status; } catch (std::exception& e) { auto status = CStatus(); - status.error_code = UnexpectedError; + status.error_code = milvus::UnexpectedError; status.error_msg = strdup(e.what()); return status; } @@ -313,12 +313,12 @@ AppendIndexInfo(CLoadIndexInfo c_load_index_info, load_index_info->index_version = version; auto status = CStatus(); - status.error_code = Success; + status.error_code = milvus::Success; status.error_msg = ""; return status; } catch (std::exception& e) { auto status = CStatus(); - status.error_code = UnexpectedError; + status.error_code = milvus::UnexpectedError; status.error_msg = strdup(e.what()); return status; } @@ -338,12 +338,12 @@ CleanLoadedIndex(CLoadIndexInfo c_load_index_info) { load_index_info->index_version); local_chunk_manager->RemoveDir(index_file_path_prefix); auto status = CStatus(); - status.error_code = Success; + status.error_code = milvus::Success; status.error_msg = ""; return status; } catch (std::exception& e) { auto status = CStatus(); - status.error_code = UnexpectedError; + status.error_code = milvus::UnexpectedError; status.error_msg = strdup(e.what()); return status; } diff --git a/internal/core/src/segcore/plan_c.cpp b/internal/core/src/segcore/plan_c.cpp index 8a7892d898..427dc037db 100644 --- a/internal/core/src/segcore/plan_c.cpp +++ b/internal/core/src/segcore/plan_c.cpp @@ -9,7 +9,6 @@ // is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express // or implied. See the License for the specific language governing permissions and limitations under the License -#include "common/CGoHelper.h" #include "pb/segcore.pb.h" #include "query/Plan.h" #include "segcore/Collection.h" @@ -28,7 +27,7 @@ CreateSearchPlanByExpr(CCollection c_col, *col->get_schema(), serialized_expr_plan, size); auto status = CStatus(); - status.error_code = Success; + status.error_code = milvus::Success; status.error_msg = ""; auto plan = (CSearchPlan)res.release(); *res_plan = plan; @@ -41,7 +40,7 @@ CreateSearchPlanByExpr(CCollection c_col, return status; } catch (std::exception& e) { auto status = CStatus(); - status.error_code = UnexpectedError; + status.error_code = milvus::UnexpectedError; status.error_msg = strdup(e.what()); *res_plan = nullptr; return status; @@ -60,14 +59,14 @@ ParsePlaceholderGroup(CSearchPlan c_plan, plan, (const uint8_t*)(placeholder_group_blob), blob_size); auto status = CStatus(); - status.error_code = Success; + status.error_code = milvus::Success; status.error_msg = ""; auto group = (CPlaceholderGroup)res.release(); *res_placeholder_group = group; return status; } catch (std::exception& e) { auto status = CStatus(); - status.error_code = UnexpectedError; + status.error_code = milvus::UnexpectedError; status.error_msg = strdup(e.what()); *res_placeholder_group = nullptr; return status; @@ -94,7 +93,7 @@ GetFieldID(CSearchPlan plan, int64_t* field_id) { *field_id = milvus::query::GetFieldID(p); return milvus::SuccessCStatus(); } catch (std::exception& e) { - return milvus::FailureCStatus(UnexpectedError, strdup(e.what())); + return milvus::FailureCStatus(milvus::UnexpectedError, e.what()); } } @@ -139,7 +138,7 @@ CreateRetrievePlanByExpr(CCollection c_col, *col->get_schema(), serialized_expr_plan, size); auto status = CStatus(); - status.error_code = Success; + status.error_code = milvus::Success; status.error_msg = ""; auto plan = (CRetrievePlan)res.release(); *res_plan = plan; @@ -152,7 +151,7 @@ CreateRetrievePlanByExpr(CCollection c_col, return status; } catch (std::exception& e) { auto status = CStatus(); - status.error_code = UnexpectedError; + status.error_code = milvus::UnexpectedError; status.error_msg = strdup(e.what()); *res_plan = nullptr; return status; diff --git a/internal/core/src/segcore/reduce_c.cpp b/internal/core/src/segcore/reduce_c.cpp index b683fd9021..de30a3a298 100644 --- a/internal/core/src/segcore/reduce_c.cpp +++ b/internal/core/src/segcore/reduce_c.cpp @@ -11,9 +11,8 @@ #include #include "Reduce.h" -#include "common/CGoHelper.h" #include "common/QueryResult.h" -#include "exceptions/EasyAssert.h" +#include "common/EasyAssert.h" #include "query/Plan.h" #include "segcore/reduce_c.h" #include "segcore/Utils.h" @@ -46,7 +45,7 @@ ReduceSearchResultsAndFillData(CSearchResultDataBlobs* cSearchResultDataBlobs, *cSearchResultDataBlobs = reduce_helper.GetSearchResultDataBlobs(); return milvus::SuccessCStatus(); } catch (std::exception& e) { - return milvus::FailureCStatus(UnexpectedError, e.what()); + return milvus::FailureCStatus(milvus::UnexpectedError, e.what()); } } @@ -68,7 +67,7 @@ GetSearchResultDataBlob(CProto* searchResultDataBlob, } catch (std::exception& e) { searchResultDataBlob->proto_blob = nullptr; searchResultDataBlob->proto_size = 0; - return milvus::FailureCStatus(UnexpectedError, e.what()); + return milvus::FailureCStatus(milvus::UnexpectedError, e.what()); } } diff --git a/internal/core/src/segcore/segment_c.cpp b/internal/core/src/segcore/segment_c.cpp index fd7322ee34..bcd4606049 100644 --- a/internal/core/src/segcore/segment_c.cpp +++ b/internal/core/src/segcore/segment_c.cpp @@ -12,7 +12,6 @@ #include "segcore/segment_c.h" #include -#include "common/CGoHelper.h" #include "common/LoadInfo.h" #include "common/Types.h" #include "common/Tracer.h" @@ -92,7 +91,7 @@ Search(CSegmentInterface c_segment, milvus::tracer::CloseRootSpan(); return milvus::SuccessCStatus(); } catch (std::exception& e) { - return milvus::FailureCStatus(UnexpectedError, e.what()); + return milvus::FailureCStatus(milvus::UnexpectedError, e.what()); } } @@ -129,7 +128,7 @@ Retrieve(CSegmentInterface c_segment, span->End(); return milvus::SuccessCStatus(); } catch (std::exception& e) { - return milvus::FailureCStatus(UnexpectedError, e.what()); + return milvus::FailureCStatus(milvus::UnexpectedError, e.what()); } } @@ -191,7 +190,7 @@ Insert(CSegmentInterface c_segment, reserved_offset, size, row_ids, timestamps, insert_data.get()); return milvus::SuccessCStatus(); } catch (std::exception& e) { - return milvus::FailureCStatus(UnexpectedError, e.what()); + return milvus::FailureCStatus(milvus::UnexpectedError, e.what()); } } @@ -202,7 +201,7 @@ PreInsert(CSegmentInterface c_segment, int64_t size, int64_t* offset) { *offset = segment->PreInsert(size); return milvus::SuccessCStatus(); } catch (std::exception& e) { - return milvus::FailureCStatus(UnexpectedError, e.what()); + return milvus::FailureCStatus(milvus::UnexpectedError, e.what()); } } @@ -222,7 +221,7 @@ Delete(CSegmentInterface c_segment, segment->Delete(reserved_offset, size, pks.get(), timestamps); return milvus::SuccessCStatus(); } catch (std::exception& e) { - return milvus::FailureCStatus(UnexpectedError, e.what()); + return milvus::FailureCStatus(milvus::UnexpectedError, e.what()); } } @@ -238,7 +237,7 @@ LoadFieldData(CSegmentInterface c_segment, segment->LoadFieldData(*load_info); return milvus::SuccessCStatus(); } catch (std::exception& e) { - return milvus::FailureCStatus(UnexpectedError, e.what()); + return milvus::FailureCStatus(milvus::UnexpectedError, e.what()); } } @@ -278,7 +277,7 @@ LoadFieldRawData(CSegmentInterface c_segment, segment->LoadFieldData(milvus::FieldId(field_id), field_data_info); return milvus::SuccessCStatus(); } catch (std::exception& e) { - return milvus::FailureCStatus(UnexpectedError, e.what()); + return milvus::FailureCStatus(milvus::UnexpectedError, e.what()); } } @@ -299,7 +298,7 @@ LoadDeletedRecord(CSegmentInterface c_segment, segment_interface->LoadDeletedRecord(load_info); return milvus::SuccessCStatus(); } catch (std::exception& e) { - return milvus::FailureCStatus(UnexpectedError, e.what()); + return milvus::FailureCStatus(milvus::UnexpectedError, e.what()); } } @@ -317,7 +316,7 @@ UpdateSealedSegmentIndex(CSegmentInterface c_segment, segment->LoadIndex(*load_index_info); return milvus::SuccessCStatus(); } catch (std::exception& e) { - return milvus::FailureCStatus(UnexpectedError, e.what()); + return milvus::FailureCStatus(milvus::UnexpectedError, e.what()); } } @@ -334,7 +333,7 @@ UpdateFieldRawDataSize(CSegmentInterface c_segment, milvus::FieldId(field_id), num_rows, field_data_size); return milvus::SuccessCStatus(); } catch (std::exception& e) { - return milvus::FailureCStatus(UnexpectedError, e.what()); + return milvus::FailureCStatus(milvus::UnexpectedError, e.what()); } } @@ -349,7 +348,7 @@ DropFieldData(CSegmentInterface c_segment, int64_t field_id) { segment->DropFieldData(milvus::FieldId(field_id)); return milvus::SuccessCStatus(); } catch (std::exception& e) { - return milvus::FailureCStatus(UnexpectedError, e.what()); + return milvus::FailureCStatus(milvus::UnexpectedError, e.what()); } } @@ -364,6 +363,6 @@ DropSealedSegmentIndex(CSegmentInterface c_segment, int64_t field_id) { segment->DropIndex(milvus::FieldId(field_id)); return milvus::SuccessCStatus(); } catch (std::exception& e) { - return milvus::FailureCStatus(UnexpectedError, e.what()); + return milvus::FailureCStatus(milvus::UnexpectedError, e.what()); } } diff --git a/internal/core/src/storage/BinlogReader.cpp b/internal/core/src/storage/BinlogReader.cpp index fae1a65a90..975ee92ade 100644 --- a/internal/core/src/storage/BinlogReader.cpp +++ b/internal/core/src/storage/BinlogReader.cpp @@ -15,32 +15,34 @@ // limitations under the License. #include "storage/BinlogReader.h" +#include "common/EasyAssert.h" namespace milvus::storage { -Status +milvus::SegcoreError BinlogReader::Read(int64_t nbytes, void* out) { auto remain = size_ - tell_; if (nbytes > remain) { - return Status(SERVER_UNEXPECTED_ERROR, "out range of binlog data"); + return SegcoreError(milvus::UnexpectedError, + "out range of binlog data"); } std::memcpy(out, data_.get() + tell_, nbytes); tell_ += nbytes; - return Status(SERVER_SUCCESS, ""); + return SegcoreError(milvus::Success, ""); } -std::pair> +std::pair> BinlogReader::Read(int64_t nbytes) { auto remain = size_ - tell_; if (nbytes > remain) { return std::make_pair( - Status(SERVER_UNEXPECTED_ERROR, "out range of binlog data"), + SegcoreError(milvus::UnexpectedError, "out range of binlog data"), nullptr); } auto deleter = [&](uint8_t*) {}; // avoid repeated deconstruction auto res = std::shared_ptr(data_.get() + tell_, deleter); tell_ += nbytes; - return std::make_pair(Status(SERVER_SUCCESS, ""), res); + return std::make_pair(SegcoreError(milvus::Success, ""), res); } } // namespace milvus::storage diff --git a/internal/core/src/storage/BinlogReader.h b/internal/core/src/storage/BinlogReader.h index 480362a6af..467407ac80 100644 --- a/internal/core/src/storage/BinlogReader.h +++ b/internal/core/src/storage/BinlogReader.h @@ -19,8 +19,7 @@ #include #include -#include "utils/Status.h" -#include "exceptions/EasyAssert.h" +#include "common/EasyAssert.h" namespace milvus::storage { @@ -31,10 +30,10 @@ class BinlogReader { : data_(binlog_data), size_(length), tell_(0) { } - Status + SegcoreError Read(int64_t nbytes, void* out); - std::pair> + std::pair> Read(int64_t nbytes); int64_t @@ -43,9 +42,9 @@ class BinlogReader { } private: + std::shared_ptr data_; int64_t size_; int64_t tell_; - std::shared_ptr data_; }; using BinlogReaderPtr = std::shared_ptr; diff --git a/internal/core/src/storage/CMakeLists.txt b/internal/core/src/storage/CMakeLists.txt index f49e396754..a69a2d7aeb 100644 --- a/internal/core/src/storage/CMakeLists.txt +++ b/internal/core/src/storage/CMakeLists.txt @@ -31,7 +31,7 @@ set(STORAGE_FILES PayloadReader.cpp PayloadWriter.cpp BinlogReader.cpp - IndexData.cpp + IndexData.cpp InsertData.cpp Event.cpp ThreadPool.cpp @@ -45,12 +45,9 @@ set(STORAGE_FILES add_library(milvus_storage SHARED ${STORAGE_FILES}) -find_package(Boost REQUIRED COMPONENTS filesystem) - -target_link_libraries(milvus_storage PUBLIC - milvus_common - Boost::filesystem - pthread +target_link_libraries(milvus_storage PUBLIC + milvus_common + pthread ${CONAN_LIBS} ) diff --git a/internal/core/src/storage/DataCodec.cpp b/internal/core/src/storage/DataCodec.cpp index 923e8ef57e..5c2fe2a295 100644 --- a/internal/core/src/storage/DataCodec.cpp +++ b/internal/core/src/storage/DataCodec.cpp @@ -20,7 +20,7 @@ #include "storage/InsertData.h" #include "storage/IndexData.h" #include "storage/BinlogReader.h" -#include "exceptions/EasyAssert.h" +#include "common/EasyAssert.h" #include "common/Consts.h" namespace milvus::storage { diff --git a/internal/core/src/storage/Event.cpp b/internal/core/src/storage/Event.cpp index 708481778c..15ab7efa2c 100644 --- a/internal/core/src/storage/Event.cpp +++ b/internal/core/src/storage/Event.cpp @@ -15,10 +15,11 @@ // limitations under the License. #include "storage/Event.h" +#include "nlohmann/json.hpp" #include "storage/PayloadReader.h" #include "storage/PayloadWriter.h" -#include "exceptions/EasyAssert.h" -#include "utils/Json.h" +#include "common/EasyAssert.h" +#include "common/Json.h" #include "common/Consts.h" #include "common/FieldMeta.h" @@ -138,8 +139,8 @@ DescriptorEventDataFixPart::Serialize() { DescriptorEventData::DescriptorEventData(BinlogReaderPtr reader) { fix_part = DescriptorEventDataFixPart(reader); - for (auto i = int8_t(EventType::DescriptorEvent); - i < int8_t(EventType::EventTypeEnd); + for (auto i = static_cast(EventType::DescriptorEvent); + i < static_cast(EventType::EventTypeEnd); i++) { post_header_lengths.push_back(GetEventFixPartSize(EventType(i))); } @@ -152,8 +153,8 @@ DescriptorEventData::DescriptorEventData(BinlogReaderPtr reader) { ast = reader->Read(extra_length, extra_bytes.data()); assert(ast.ok()); - milvus::json json = - milvus::json::parse(extra_bytes.begin(), extra_bytes.end()); + nlohmann::json json = + nlohmann::json::parse(extra_bytes.begin(), extra_bytes.end()); if (json.contains(ORIGIN_SIZE_KEY)) { extras[ORIGIN_SIZE_KEY] = json[ORIGIN_SIZE_KEY]; } @@ -165,7 +166,7 @@ DescriptorEventData::DescriptorEventData(BinlogReaderPtr reader) { std::vector DescriptorEventData::Serialize() { auto fix_part_data = fix_part.Serialize(); - milvus::json extras_json; + nlohmann::json extras_json; for (auto v : extras) { extras_json.emplace(v.first, v.second); } diff --git a/internal/core/src/storage/FieldDataInterface.h b/internal/core/src/storage/FieldDataInterface.h index 3165972ab8..8c7cbf0fa2 100644 --- a/internal/core/src/storage/FieldDataInterface.h +++ b/internal/core/src/storage/FieldDataInterface.h @@ -30,7 +30,7 @@ #include "common/FieldMeta.h" #include "common/Utils.h" #include "common/VectorTrait.h" -#include "exceptions/EasyAssert.h" +#include "common/EasyAssert.h" #include "storage/Exception.h" namespace milvus::storage { diff --git a/internal/core/src/storage/IndexData.cpp b/internal/core/src/storage/IndexData.cpp index fb448a362c..2c14ae5a0d 100644 --- a/internal/core/src/storage/IndexData.cpp +++ b/internal/core/src/storage/IndexData.cpp @@ -15,7 +15,7 @@ // limitations under the License. #include "storage/IndexData.h" -#include "exceptions/EasyAssert.h" +#include "common/EasyAssert.h" #include "common/Consts.h" #include "storage/Event.h" diff --git a/internal/core/src/storage/InsertData.cpp b/internal/core/src/storage/InsertData.cpp index 91cbf093b1..82f2785cfe 100644 --- a/internal/core/src/storage/InsertData.cpp +++ b/internal/core/src/storage/InsertData.cpp @@ -17,7 +17,7 @@ #include "storage/InsertData.h" #include "storage/Event.h" #include "storage/Util.h" -#include "utils/Json.h" +#include "common/Json.h" #include "common/FieldMeta.h" #include "common/Consts.h" diff --git a/internal/core/src/storage/MinioChunkManager.cpp b/internal/core/src/storage/MinioChunkManager.cpp index 430bfbbb1f..89501d82d1 100644 --- a/internal/core/src/storage/MinioChunkManager.cpp +++ b/internal/core/src/storage/MinioChunkManager.cpp @@ -31,7 +31,7 @@ #include "storage/MinioChunkManager.h" #include "storage/AliyunSTSClient.h" #include "storage/AliyunCredentialsProvider.h" -#include "exceptions/EasyAssert.h" +#include "common/EasyAssert.h" #include "log/Log.h" #include "signal.h" diff --git a/internal/core/src/storage/PayloadReader.cpp b/internal/core/src/storage/PayloadReader.cpp index 8f9b190319..54e39cb636 100644 --- a/internal/core/src/storage/PayloadReader.cpp +++ b/internal/core/src/storage/PayloadReader.cpp @@ -15,7 +15,7 @@ // limitations under the License. #include "storage/PayloadReader.h" -#include "exceptions/EasyAssert.h" +#include "common/EasyAssert.h" #include "storage/Util.h" #include "parquet/column_reader.h" #include "arrow/io/api.h" diff --git a/internal/core/src/storage/PayloadStream.cpp b/internal/core/src/storage/PayloadStream.cpp index d7ec9f303c..93ccdef858 100644 --- a/internal/core/src/storage/PayloadStream.cpp +++ b/internal/core/src/storage/PayloadStream.cpp @@ -17,7 +17,7 @@ #include "arrow/api.h" #include "storage/PayloadStream.h" -#include "exceptions/EasyAssert.h" +#include "common/EasyAssert.h" namespace milvus::storage { diff --git a/internal/core/src/storage/PayloadWriter.cpp b/internal/core/src/storage/PayloadWriter.cpp index 7d9def154a..54c47ed81e 100644 --- a/internal/core/src/storage/PayloadWriter.cpp +++ b/internal/core/src/storage/PayloadWriter.cpp @@ -15,7 +15,7 @@ // limitations under the License. #include "storage/PayloadWriter.h" -#include "exceptions/EasyAssert.h" +#include "common/EasyAssert.h" #include "common/FieldMeta.h" #include "storage/Util.h" diff --git a/internal/core/src/storage/Util.cpp b/internal/core/src/storage/Util.cpp index e40fc858e6..e942b70bd2 100644 --- a/internal/core/src/storage/Util.cpp +++ b/internal/core/src/storage/Util.cpp @@ -18,7 +18,7 @@ #include #include "arrow/array/builder_binary.h" #include "arrow/type_fwd.h" -#include "exceptions/EasyAssert.h" +#include "common/EasyAssert.h" #include "common/Consts.h" #include "storage/FieldData.h" #include "storage/FieldDataInterface.h" diff --git a/internal/core/src/storage/parquet_c.cpp b/internal/core/src/storage/parquet_c.cpp index f4935f2ad9..6ac770b954 100644 --- a/internal/core/src/storage/parquet_c.cpp +++ b/internal/core/src/storage/parquet_c.cpp @@ -20,7 +20,6 @@ #include "storage/PayloadReader.h" #include "storage/PayloadWriter.h" #include "storage/FieldData.h" -#include "common/CGoHelper.h" #include "storage/Util.h" using Payload = milvus::storage::Payload; @@ -50,7 +49,8 @@ AddValuesToPayload(CPayloadWriter payloadWriter, const Payload& info) { p->add_payload(info); return milvus::SuccessCStatus(); } catch (std::exception& e) { - return milvus::FailureCStatus(UnexpectedError, e.what()); + return milvus::FailureCStatus(milvus::ErrorCodeEnum::UnexpectedError, + e.what()); } } @@ -117,7 +117,8 @@ AddOneStringToPayload(CPayloadWriter payloadWriter, char* cstr, int str_size) { p->add_one_string_payload(cstr, str_size); return milvus::SuccessCStatus(); } catch (std::exception& e) { - return milvus::FailureCStatus(UnexpectedError, e.what()); + return milvus::FailureCStatus(milvus::ErrorCodeEnum::UnexpectedError, + e.what()); } } @@ -128,7 +129,8 @@ AddOneArrayToPayload(CPayloadWriter payloadWriter, uint8_t* data, int length) { p->add_one_binary_payload(data, length); return milvus::SuccessCStatus(); } catch (std::exception& e) { - return milvus::FailureCStatus(UnexpectedError, e.what()); + return milvus::FailureCStatus(milvus::ErrorCodeEnum::UnexpectedError, + e.what()); } } @@ -139,7 +141,8 @@ AddOneJSONToPayload(CPayloadWriter payloadWriter, uint8_t* data, int length) { p->add_one_binary_payload(data, length); return milvus::SuccessCStatus(); } catch (std::exception& e) { - return milvus::FailureCStatus(UnexpectedError, e.what()); + return milvus::FailureCStatus(milvus::ErrorCodeEnum::UnexpectedError, + e.what()); } } @@ -155,7 +158,8 @@ AddBinaryVectorToPayload(CPayloadWriter payloadWriter, p->add_payload(raw_data_info); return milvus::SuccessCStatus(); } catch (std::exception& e) { - return milvus::FailureCStatus(UnexpectedError, e.what()); + return milvus::FailureCStatus(milvus::ErrorCodeEnum::UnexpectedError, + e.what()); } } @@ -173,7 +177,8 @@ AddFloatVectorToPayload(CPayloadWriter payloadWriter, p->add_payload(raw_data_info); return milvus::SuccessCStatus(); } catch (std::exception& e) { - return milvus::FailureCStatus(UnexpectedError, e.what()); + return milvus::FailureCStatus(milvus::ErrorCodeEnum::UnexpectedError, + e.what()); } } @@ -184,7 +189,8 @@ FinishPayloadWriter(CPayloadWriter payloadWriter) { p->finish(); return milvus::SuccessCStatus(); } catch (std::exception& e) { - return milvus::FailureCStatus(UnexpectedError, e.what()); + return milvus::FailureCStatus(milvus::ErrorCodeEnum::UnexpectedError, + e.what()); } } @@ -240,8 +246,9 @@ NewPayloadReader(int columnType, break; } default: { - return milvus::FailureCStatus(UnexpectedError, - "unsupported data type"); + return milvus::FailureCStatus( + milvus::ErrorCodeEnum::UnexpectedError, + "unsupported data type"); } } @@ -250,7 +257,8 @@ NewPayloadReader(int columnType, *c_reader = (CPayloadReader)(p.release()); return milvus::SuccessCStatus(); } catch (std::exception& e) { - return milvus::FailureCStatus(UnexpectedError, e.what()); + return milvus::FailureCStatus(milvus::ErrorCodeEnum::UnexpectedError, + e.what()); } } @@ -262,7 +270,8 @@ GetBoolFromPayload(CPayloadReader payloadReader, int idx, bool* value) { *value = *reinterpret_cast(field_data->RawValue(idx)); return milvus::SuccessCStatus(); } catch (std::exception& e) { - return milvus::FailureCStatus(UnexpectedError, e.what()); + return milvus::FailureCStatus(milvus::ErrorCodeEnum::UnexpectedError, + e.what()); } } @@ -276,7 +285,8 @@ GetInt8FromPayload(CPayloadReader payloadReader, int8_t** values, int* length) { reinterpret_cast(const_cast(field_data->Data())); return milvus::SuccessCStatus(); } catch (std::exception& e) { - return milvus::FailureCStatus(UnexpectedError, e.what()); + return milvus::FailureCStatus(milvus::ErrorCodeEnum::UnexpectedError, + e.what()); } } @@ -292,7 +302,8 @@ GetInt16FromPayload(CPayloadReader payloadReader, reinterpret_cast(const_cast(field_data->Data())); return milvus::SuccessCStatus(); } catch (std::exception& e) { - return milvus::FailureCStatus(UnexpectedError, e.what()); + return milvus::FailureCStatus(milvus::ErrorCodeEnum::UnexpectedError, + e.what()); } } @@ -308,7 +319,8 @@ GetInt32FromPayload(CPayloadReader payloadReader, reinterpret_cast(const_cast(field_data->Data())); return milvus::SuccessCStatus(); } catch (std::exception& e) { - return milvus::FailureCStatus(UnexpectedError, e.what()); + return milvus::FailureCStatus(milvus::ErrorCodeEnum::UnexpectedError, + e.what()); } } @@ -324,7 +336,8 @@ GetInt64FromPayload(CPayloadReader payloadReader, reinterpret_cast(const_cast(field_data->Data())); return milvus::SuccessCStatus(); } catch (std::exception& e) { - return milvus::FailureCStatus(UnexpectedError, e.what()); + return milvus::FailureCStatus(milvus::ErrorCodeEnum::UnexpectedError, + e.what()); } } @@ -338,7 +351,8 @@ GetFloatFromPayload(CPayloadReader payloadReader, float** values, int* length) { reinterpret_cast(const_cast(field_data->Data())); return milvus::SuccessCStatus(); } catch (std::exception& e) { - return milvus::FailureCStatus(UnexpectedError, e.what()); + return milvus::FailureCStatus(milvus::ErrorCodeEnum::UnexpectedError, + e.what()); } } @@ -354,7 +368,8 @@ GetDoubleFromPayload(CPayloadReader payloadReader, reinterpret_cast(const_cast(field_data->Data())); return milvus::SuccessCStatus(); } catch (std::exception& e) { - return milvus::FailureCStatus(UnexpectedError, e.what()); + return milvus::FailureCStatus(milvus::ErrorCodeEnum::UnexpectedError, + e.what()); } } @@ -371,7 +386,8 @@ GetOneStringFromPayload(CPayloadReader payloadReader, *str_size = field_data->Size(idx); return milvus::SuccessCStatus(); } catch (std::exception& e) { - return milvus::FailureCStatus(UnexpectedError, e.what()); + return milvus::FailureCStatus(milvus::ErrorCodeEnum::UnexpectedError, + e.what()); } } @@ -388,7 +404,8 @@ GetBinaryVectorFromPayload(CPayloadReader payloadReader, *length = field_data->get_num_rows(); return milvus::SuccessCStatus(); } catch (std::exception& e) { - return milvus::FailureCStatus(UnexpectedError, e.what()); + return milvus::FailureCStatus(milvus::ErrorCodeEnum::UnexpectedError, + e.what()); } } @@ -405,7 +422,8 @@ GetFloatVectorFromPayload(CPayloadReader payloadReader, *length = field_data->get_num_rows(); return milvus::SuccessCStatus(); } catch (std::exception& e) { - return milvus::FailureCStatus(UnexpectedError, e.what()); + return milvus::FailureCStatus(milvus::ErrorCodeEnum::UnexpectedError, + e.what()); } } @@ -427,6 +445,7 @@ ReleasePayloadReader(CPayloadReader payloadReader) { milvus::storage::ReleaseArrowUnused(); return milvus::SuccessCStatus(); } catch (std::exception& e) { - return milvus::FailureCStatus(UnexpectedError, e.what()); + return milvus::FailureCStatus(milvus::ErrorCodeEnum::UnexpectedError, + e.what()); } } diff --git a/internal/core/src/storage/storage_c.cpp b/internal/core/src/storage/storage_c.cpp index 020c54159a..e78f55cb76 100644 --- a/internal/core/src/storage/storage_c.cpp +++ b/internal/core/src/storage/storage_c.cpp @@ -15,7 +15,6 @@ // limitations under the License. #include "storage/storage_c.h" -#include "common/CGoHelper.h" #include "storage/RemoteChunkManagerSingleton.h" #include "storage/LocalChunkManagerSingleton.h" @@ -33,7 +32,8 @@ GetLocalUsedSize(const char* c_dir, int64_t* size) { } return milvus::SuccessCStatus(); } catch (std::exception& e) { - return milvus::FailureCStatus(UnexpectedError, e.what()); + return milvus::FailureCStatus(milvus::ErrorCodeEnum::UnexpectedError, + e.what()); } } @@ -45,7 +45,8 @@ InitLocalChunkManagerSingleton(const char* c_path) { return milvus::SuccessCStatus(); } catch (std::exception& e) { - return milvus::FailureCStatus(UnexpectedError, e.what()); + return milvus::FailureCStatus(milvus::ErrorCodeEnum::UnexpectedError, + e.what()); } } @@ -74,7 +75,8 @@ InitRemoteChunkManagerSingleton(CStorageConfig c_storage_config) { return milvus::SuccessCStatus(); } catch (std::exception& e) { - return milvus::FailureCStatus(UnexpectedError, e.what()); + return milvus::FailureCStatus(milvus::ErrorCodeEnum::UnexpectedError, + e.what()); } } diff --git a/internal/core/src/utils/CMakeLists.txt b/internal/core/src/utils/CMakeLists.txt deleted file mode 100644 index 83934e00f0..0000000000 --- a/internal/core/src/utils/CMakeLists.txt +++ /dev/null @@ -1,17 +0,0 @@ -#------------------------------------------------------------------------------- -# Copyright (C) 2019-2020 Zilliz. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software distributed under the License -# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -# or implied. See the License for the specific language governing permissions and limitations under the License. -#------------------------------------------------------------------------------- - -aux_source_directory( ${MILVUS_ENGINE_SRC}/utils UTILS_FILES ) - -add_library( milvus_utils STATIC ${UTILS_FILES} ) - diff --git a/internal/core/src/utils/Error.h b/internal/core/src/utils/Error.h deleted file mode 100644 index 229b1edbed..0000000000 --- a/internal/core/src/utils/Error.h +++ /dev/null @@ -1,146 +0,0 @@ -// Copyright (C) 2019-2020 Zilliz. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software distributed under the License -// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -// or implied. See the License for the specific language governing permissions and limitations under the License. - -#pragma once - -#include -#include -#include - -namespace milvus { - -using ErrorCode = int32_t; - -constexpr ErrorCode SERVER_SUCCESS = 0; -constexpr ErrorCode SERVER_ERROR_CODE_BASE = 30000; - -constexpr ErrorCode -ToServerErrorCode(const ErrorCode error_code) { - return SERVER_ERROR_CODE_BASE + error_code; -} - -constexpr ErrorCode DB_SUCCESS = 0; -constexpr ErrorCode DB_ERROR_CODE_BASE = 40000; - -constexpr ErrorCode -ToDbErrorCode(const ErrorCode error_code) { - return DB_ERROR_CODE_BASE + error_code; -} - -constexpr ErrorCode KNOWHERE_SUCCESS = 0; -constexpr ErrorCode KNOWHERE_ERROR_CODE_BASE = 50000; - -constexpr ErrorCode -ToKnowhereErrorCode(const ErrorCode error_code) { - return KNOWHERE_ERROR_CODE_BASE + error_code; -} - -constexpr ErrorCode WAL_SUCCESS = 0; -constexpr ErrorCode WAL_ERROR_CODE_BASE = 60000; - -constexpr ErrorCode -ToWalErrorCode(const ErrorCode error_code) { - return WAL_ERROR_CODE_BASE + error_code; -} - -constexpr ErrorCode SS_SUCCESS = 0; -constexpr ErrorCode SS_ERROR_CODE_BASE = 70000; - -constexpr ErrorCode -ToSSErrorCode(const ErrorCode error_code) { - return SS_ERROR_CODE_BASE + error_code; -} - -// server error code -constexpr ErrorCode SERVER_UNEXPECTED_ERROR = ToServerErrorCode(1); -constexpr ErrorCode SERVER_UNSUPPORTED_ERROR = ToServerErrorCode(2); -constexpr ErrorCode SERVER_NULL_POINTER = ToServerErrorCode(3); -constexpr ErrorCode SERVER_INVALID_ARGUMENT = ToServerErrorCode(4); -constexpr ErrorCode SERVER_FILE_NOT_FOUND = ToServerErrorCode(5); -constexpr ErrorCode SERVER_NOT_IMPLEMENT = ToServerErrorCode(6); -constexpr ErrorCode SERVER_CANNOT_CREATE_FOLDER = ToServerErrorCode(8); -constexpr ErrorCode SERVER_CANNOT_CREATE_FILE = ToServerErrorCode(9); -constexpr ErrorCode SERVER_CANNOT_DELETE_FOLDER = ToServerErrorCode(10); -constexpr ErrorCode SERVER_CANNOT_DELETE_FILE = ToServerErrorCode(11); -constexpr ErrorCode SERVER_BUILD_INDEX_ERROR = ToServerErrorCode(12); -constexpr ErrorCode SERVER_CANNOT_OPEN_FILE = ToServerErrorCode(13); -constexpr ErrorCode SERVER_FILE_MAGIC_BYTES_ERROR = ToServerErrorCode(14); -constexpr ErrorCode SERVER_FILE_SUM_BYTES_ERROR = ToServerErrorCode(15); -constexpr ErrorCode SERVER_CANNOT_READ_FILE = ToServerErrorCode(16); - -constexpr ErrorCode SERVER_COLLECTION_NOT_EXIST = ToServerErrorCode(100); -constexpr ErrorCode SERVER_INVALID_COLLECTION_NAME = ToServerErrorCode(101); -constexpr ErrorCode SERVER_INVALID_COLLECTION_DIMENSION = - ToServerErrorCode(102); -constexpr ErrorCode SERVER_INVALID_VECTOR_DIMENSION = ToServerErrorCode(104); -constexpr ErrorCode SERVER_INVALID_INDEX_TYPE = ToServerErrorCode(105); -constexpr ErrorCode SERVER_INVALID_ROWRECORD = ToServerErrorCode(106); -constexpr ErrorCode SERVER_INVALID_ROWRECORD_ARRAY = ToServerErrorCode(107); -constexpr ErrorCode SERVER_INVALID_TOPK = ToServerErrorCode(108); -constexpr ErrorCode SERVER_ILLEGAL_VECTOR_ID = ToServerErrorCode(109); -constexpr ErrorCode SERVER_ILLEGAL_SEARCH_RESULT = ToServerErrorCode(110); -constexpr ErrorCode SERVER_CACHE_FULL = ToServerErrorCode(111); -constexpr ErrorCode SERVER_WRITE_ERROR = ToServerErrorCode(112); -constexpr ErrorCode SERVER_INVALID_NPROBE = ToServerErrorCode(113); -constexpr ErrorCode SERVER_INVALID_INDEX_NLIST = ToServerErrorCode(114); -constexpr ErrorCode SERVER_INVALID_INDEX_METRIC_TYPE = ToServerErrorCode(115); -constexpr ErrorCode SERVER_INVALID_SEGMENT_ROW_COUNT = ToServerErrorCode(116); -constexpr ErrorCode SERVER_OUT_OF_MEMORY = ToServerErrorCode(117); -constexpr ErrorCode SERVER_INVALID_PARTITION_TAG = ToServerErrorCode(118); -constexpr ErrorCode SERVER_INVALID_BINARY_QUERY = ToServerErrorCode(119); -constexpr ErrorCode SERVER_INVALID_DSL_PARAMETER = ToServerErrorCode(120); -constexpr ErrorCode SERVER_INVALID_FIELD_NAME = ToServerErrorCode(121); -constexpr ErrorCode SERVER_INVALID_FIELD_NUM = ToServerErrorCode(122); - -// db error code -constexpr ErrorCode DB_META_TRANSACTION_FAILED = ToDbErrorCode(1); -constexpr ErrorCode DB_ERROR = ToDbErrorCode(2); -constexpr ErrorCode DB_NOT_FOUND = ToDbErrorCode(3); -constexpr ErrorCode DB_ALREADY_EXIST = ToDbErrorCode(4); -constexpr ErrorCode DB_INVALID_PATH = ToDbErrorCode(5); -constexpr ErrorCode DB_INCOMPATIB_META = ToDbErrorCode(6); -constexpr ErrorCode DB_INVALID_META_URI = ToDbErrorCode(7); -constexpr ErrorCode DB_EMPTY_COLLECTION = ToDbErrorCode(8); -constexpr ErrorCode DB_BLOOM_FILTER_ERROR = ToDbErrorCode(9); -constexpr ErrorCode DB_PARTITION_NOT_FOUND = ToDbErrorCode(10); -constexpr ErrorCode DB_OUT_OF_STORAGE = ToDbErrorCode(11); -constexpr ErrorCode DB_META_QUERY_FAILED = ToDbErrorCode(12); -constexpr ErrorCode DB_FILE_NOT_FOUND = ToDbErrorCode(13); -constexpr ErrorCode DB_PERMISSION_ERROR = ToDbErrorCode(14); - -// knowhere error code -constexpr ErrorCode KNOWHERE_ERROR = ToKnowhereErrorCode(1); -constexpr ErrorCode KNOWHERE_INVALID_ARGUMENT = ToKnowhereErrorCode(2); -constexpr ErrorCode KNOWHERE_UNEXPECTED_ERROR = ToKnowhereErrorCode(3); -constexpr ErrorCode KNOWHERE_NO_SPACE = ToKnowhereErrorCode(4); - -// knowhere error code -constexpr ErrorCode WAL_ERROR = ToWalErrorCode(1); -constexpr ErrorCode WAL_META_ERROR = ToWalErrorCode(2); -constexpr ErrorCode WAL_FILE_ERROR = ToWalErrorCode(3); -constexpr ErrorCode WAL_PATH_ERROR = ToWalErrorCode(4); - -// Snapshot error code -constexpr ErrorCode SS_ERROR = ToSSErrorCode(1); -constexpr ErrorCode SS_STALE_ERROR = ToSSErrorCode(2); -constexpr ErrorCode SS_NOT_FOUND_ERROR = ToSSErrorCode(3); -constexpr ErrorCode SS_INVALID_CONTEXT_ERROR = ToSSErrorCode(4); -constexpr ErrorCode SS_DUPLICATED_ERROR = ToSSErrorCode(5); -constexpr ErrorCode SS_NOT_ACTIVE_ERROR = ToSSErrorCode(6); -constexpr ErrorCode SS_CONSTRAINT_CHECK_ERROR = ToSSErrorCode(7); -constexpr ErrorCode SS_INVALID_ARGUMENT_ERROR = ToSSErrorCode(8); -constexpr ErrorCode SS_OPERATION_PENDING = ToSSErrorCode(9); -constexpr ErrorCode SS_TIMEOUT = ToSSErrorCode(10); -constexpr ErrorCode SS_NOT_COMMITTED = ToSSErrorCode(11); -constexpr ErrorCode SS_COLLECTION_DROPPED = ToSSErrorCode(12); -constexpr ErrorCode SS_EMPTY_HOLDER = ToSSErrorCode(13); - -} // namespace milvus diff --git a/internal/core/src/utils/Json.h b/internal/core/src/utils/Json.h deleted file mode 100644 index e4056f7e38..0000000000 --- a/internal/core/src/utils/Json.h +++ /dev/null @@ -1,35 +0,0 @@ -// Copyright (C) 2019-2020 Zilliz. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software distributed under the License -// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -// or implied. See the License for the specific language governing permissions and limitations under the License. - -#pragma once - -#include "nlohmann/json.hpp" - -namespace milvus { - -using json = nlohmann::json; - -#define JSON_NULL_CHECK(json) \ - do { \ - if (json.empty()) { \ - return Status{SERVER_INVALID_ARGUMENT, "Json is null"}; \ - } \ - } while (false) - -#define JSON_OBJECT_CHECK(json) \ - do { \ - if (!json.is_object()) { \ - return Status{SERVER_INVALID_ARGUMENT, \ - "Json is not a json object"}; \ - } \ - } while (false) - -} // namespace milvus diff --git a/internal/core/src/utils/Status.cpp b/internal/core/src/utils/Status.cpp deleted file mode 100644 index 332fe0d987..0000000000 --- a/internal/core/src/utils/Status.cpp +++ /dev/null @@ -1,127 +0,0 @@ -// Copyright (C) 2019-2020 Zilliz. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software distributed under the License -// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -// or implied. See the License for the specific language governing permissions and limitations under the License. - -#include "utils/Status.h" -#include "memory" - -#include - -namespace milvus { - -constexpr int CODE_WIDTH = sizeof(StatusCode); - -Status::Status(StatusCode code, const std::string_view msg) { - // 4 bytes store code - // 4 bytes store message length - // the left bytes store message string - auto length = static_cast(msg.size()); - // auto result = new char[length + sizeof(length) + CODE_WIDTH]; - state_.resize(length + sizeof(length) + CODE_WIDTH); - std::memcpy(state_.data(), &code, CODE_WIDTH); - std::memcpy(state_.data() + CODE_WIDTH, &length, sizeof(length)); - memcpy(state_.data() + sizeof(length) + CODE_WIDTH, msg.data(), length); -} - -Status::~Status() { -} - -Status::Status(const Status& s) { - CopyFrom(s); -} - -Status::Status(Status&& s) noexcept { - MoveFrom(s); -} - -Status& -Status::operator=(const Status& s) { - CopyFrom(s); - return *this; -} - -Status& -Status::operator=(Status&& s) noexcept { - MoveFrom(s); - return *this; -} - -void -Status::CopyFrom(const Status& s) { - state_.clear(); - if (s.state_.empty()) { - return; - } - - uint32_t length = 0; - memcpy(&length, s.state_.data() + CODE_WIDTH, sizeof(length)); - int buff_len = length + sizeof(length) + CODE_WIDTH; - state_.resize(buff_len); - memcpy(state_.data(), s.state_.data(), buff_len); -} - -void -Status::MoveFrom(Status& s) { - state_ = s.state_; - s.state_.clear(); -} - -std::string -Status::message() const { - if (state_.empty()) { - return "OK"; - } - - std::string msg; - uint32_t length = 0; - memcpy(&length, state_.data() + CODE_WIDTH, sizeof(length)); - if (length > 0) { - msg.append(state_.data() + sizeof(length) + CODE_WIDTH, length); - } - - return msg; -} - -std::string -Status::ToString() const { - if (state_.empty()) { - return "OK"; - } - - std::string result; - switch (code()) { - case DB_SUCCESS: - result = "OK "; - break; - case DB_ERROR: - result = "Error: "; - break; - case DB_META_TRANSACTION_FAILED: - result = "Database error: "; - break; - case DB_NOT_FOUND: - result = "Not found: "; - break; - case DB_ALREADY_EXIST: - result = "Already exist: "; - break; - case DB_INVALID_PATH: - result = "Invalid path: "; - break; - default: - result = "Error code(" + std::to_string(code()) + "): "; - break; - } - - result += message(); - return result; -} - -} // namespace milvus diff --git a/internal/core/src/utils/Status.h b/internal/core/src/utils/Status.h deleted file mode 100644 index 73bc395861..0000000000 --- a/internal/core/src/utils/Status.h +++ /dev/null @@ -1,79 +0,0 @@ -// Copyright (C) 2019-2020 Zilliz. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software distributed under the License -// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -// or implied. See the License for the specific language governing permissions and limitations under the License. - -#pragma once - -#include "Error.h" - -#include - -namespace milvus { - -class Status; -#define STATUS_CHECK(func) \ - do { \ - Status s = func; \ - if (!s.ok()) { \ - return s; \ - } \ - } while (false) - -using StatusCode = ErrorCode; - -class Status { - public: - Status(StatusCode code, const std::string_view msg); - Status() = default; - virtual ~Status(); - - Status(const Status& s); - - Status(Status&& s) noexcept; - - Status& - operator=(const Status& s); - - Status& - operator=(Status&& s) noexcept; - - static Status - OK() { - return Status(); - } - - bool - ok() const { - return state_.empty() || code() == 0; - } - - StatusCode - code() const { - return (state_.empty()) ? 0 : *(StatusCode*)(state_.data()); - } - - std::string - message() const; - - std::string - ToString() const; - - private: - inline void - CopyFrom(const Status& s); - - inline void - MoveFrom(Status& s); - - private: - std::string state_; -}; // Status - -} // namespace milvus diff --git a/internal/core/unittest/bench/CMakeLists.txt b/internal/core/unittest/bench/CMakeLists.txt index b59c27926d..6147e37332 100644 --- a/internal/core/unittest/bench/CMakeLists.txt +++ b/internal/core/unittest/bench/CMakeLists.txt @@ -12,7 +12,7 @@ include_directories(${CMAKE_HOME_DIRECTORY}/src) include_directories(${CMAKE_HOME_DIRECTORY}/unittest) -set(bench_srcs +set(bench_srcs bench_naive.cpp bench_search.cpp ) @@ -38,7 +38,6 @@ target_link_libraries(indexbuilder_bench milvus_log pthread knowhere - milvus_utils ) target_link_libraries(indexbuilder_bench benchmark_main) diff --git a/internal/core/unittest/test_data_codec.cpp b/internal/core/unittest/test_data_codec.cpp index 18284354fd..3b2e6234db 100644 --- a/internal/core/unittest/test_data_codec.cpp +++ b/internal/core/unittest/test_data_codec.cpp @@ -21,7 +21,7 @@ #include "storage/IndexData.h" #include "storage/Util.h" #include "common/Consts.h" -#include "utils/Json.h" +#include "common/Json.h" using namespace milvus; @@ -305,8 +305,8 @@ TEST(storage, InsertDataBinaryVector) { TEST(storage, InsertDataFloat16Vector) { std::vector data = {1, 2, 3, 4, 5, 6, 7, 8}; int DIM = 2; - auto field_data = - milvus::storage::CreateFieldData(storage::DataType::VECTOR_FLOAT16, DIM); + auto field_data = milvus::storage::CreateFieldData( + storage::DataType::VECTOR_FLOAT16, DIM); field_data->FillFieldData(data.data(), data.size() / DIM); storage::InsertData insert_data(field_data); diff --git a/internal/core/unittest/test_index_c_api.cpp b/internal/core/unittest/test_index_c_api.cpp index 155b23e951..3b52a6d6da 100644 --- a/internal/core/unittest/test_index_c_api.cpp +++ b/internal/core/unittest/test_index_c_api.cpp @@ -48,34 +48,34 @@ TEST(FloatVecIndex, All) { { status = CreateIndex( dtype, type_params_str.c_str(), index_params_str.c_str(), &index); - ASSERT_EQ(Success, status.error_code); + ASSERT_EQ(milvus::Success, status.error_code); } { status = BuildFloatVecIndex(index, NB * DIM, xb_data.data()); - ASSERT_EQ(Success, status.error_code); + ASSERT_EQ(milvus::Success, status.error_code); } { status = SerializeIndexToBinarySet(index, &binary_set); - ASSERT_EQ(Success, status.error_code); + ASSERT_EQ(milvus::Success, status.error_code); } { status = CreateIndex(dtype, type_params_str.c_str(), index_params_str.c_str(), ©_index); - ASSERT_EQ(Success, status.error_code); + ASSERT_EQ(milvus::Success, status.error_code); } { status = LoadIndexFromBinarySet(copy_index, binary_set); - ASSERT_EQ(Success, status.error_code); + ASSERT_EQ(milvus::Success, status.error_code); } { status = DeleteIndex(index); - ASSERT_EQ(Success, status.error_code); + ASSERT_EQ(milvus::Success, status.error_code); } { status = DeleteIndex(copy_index); - ASSERT_EQ(Success, status.error_code); + ASSERT_EQ(milvus::Success, status.error_code); } { DeleteBinarySet(binary_set); } } @@ -107,34 +107,34 @@ TEST(BinaryVecIndex, All) { { status = CreateIndex( dtype, type_params_str.c_str(), index_params_str.c_str(), &index); - ASSERT_EQ(Success, status.error_code); + ASSERT_EQ(milvus::Success, status.error_code); } { status = BuildBinaryVecIndex(index, NB * DIM / 8, xb_data.data()); - ASSERT_EQ(Success, status.error_code); + ASSERT_EQ(milvus::Success, status.error_code); } { status = SerializeIndexToBinarySet(index, &binary_set); - ASSERT_EQ(Success, status.error_code); + ASSERT_EQ(milvus::Success, status.error_code); } { status = CreateIndex(dtype, type_params_str.c_str(), index_params_str.c_str(), ©_index); - ASSERT_EQ(Success, status.error_code); + ASSERT_EQ(milvus::Success, status.error_code); } { status = LoadIndexFromBinarySet(copy_index, binary_set); - ASSERT_EQ(Success, status.error_code); + ASSERT_EQ(milvus::Success, status.error_code); } { status = DeleteIndex(index); - ASSERT_EQ(Success, status.error_code); + ASSERT_EQ(milvus::Success, status.error_code); } { status = DeleteIndex(copy_index); - ASSERT_EQ(Success, status.error_code); + ASSERT_EQ(milvus::Success, status.error_code); } { DeleteBinarySet(binary_set); } } @@ -166,40 +166,40 @@ TEST(CBoolIndexTest, All) { type_params_str.c_str(), index_params_str.c_str(), &index); - ASSERT_EQ(Success, status.error_code); + ASSERT_EQ(milvus::Success, status.error_code); } { status = BuildScalarIndex( index, half_ds->GetRows(), half_ds->GetTensor()); - ASSERT_EQ(Success, status.error_code); + ASSERT_EQ(milvus::Success, status.error_code); } { status = SerializeIndexToBinarySet(index, &binary_set); - ASSERT_EQ(Success, status.error_code); + ASSERT_EQ(milvus::Success, status.error_code); } { status = CreateIndex(dtype, type_params_str.c_str(), index_params_str.c_str(), ©_index); - ASSERT_EQ(Success, status.error_code); + ASSERT_EQ(milvus::Success, status.error_code); } { status = LoadIndexFromBinarySet(copy_index, binary_set); - ASSERT_EQ(Success, status.error_code); + ASSERT_EQ(milvus::Success, status.error_code); } { status = DeleteIndex(index); - ASSERT_EQ(Success, status.error_code); + ASSERT_EQ(milvus::Success, status.error_code); } { status = DeleteIndex(copy_index); - ASSERT_EQ(Success, status.error_code); + ASSERT_EQ(milvus::Success, status.error_code); } { DeleteBinarySet(binary_set); } } - delete[](char*)(half_ds->GetTensor()); + delete[] (char*)(half_ds->GetTensor()); } // TODO: more scalar type. @@ -224,34 +224,34 @@ TEST(CInt64IndexTest, All) { type_params_str.c_str(), index_params_str.c_str(), &index); - ASSERT_EQ(Success, status.error_code); + ASSERT_EQ(milvus::Success, status.error_code); } { status = BuildScalarIndex(index, arr.size(), arr.data()); - ASSERT_EQ(Success, status.error_code); + ASSERT_EQ(milvus::Success, status.error_code); } { status = SerializeIndexToBinarySet(index, &binary_set); - ASSERT_EQ(Success, status.error_code); + ASSERT_EQ(milvus::Success, status.error_code); } { status = CreateIndex(dtype, type_params_str.c_str(), index_params_str.c_str(), ©_index); - ASSERT_EQ(Success, status.error_code); + ASSERT_EQ(milvus::Success, status.error_code); } { status = LoadIndexFromBinarySet(copy_index, binary_set); - ASSERT_EQ(Success, status.error_code); + ASSERT_EQ(milvus::Success, status.error_code); } { status = DeleteIndex(index); - ASSERT_EQ(Success, status.error_code); + ASSERT_EQ(milvus::Success, status.error_code); } { status = DeleteIndex(copy_index); - ASSERT_EQ(Success, status.error_code); + ASSERT_EQ(milvus::Success, status.error_code); } { DeleteBinarySet(binary_set); } } @@ -283,39 +283,39 @@ TEST(CStringIndexTest, All) { type_params_str.c_str(), index_params_str.c_str(), &index); - ASSERT_EQ(Success, status.error_code); + ASSERT_EQ(milvus::Success, status.error_code); } { status = BuildScalarIndex( index, (str_ds->GetRows()), (str_ds->GetTensor())); - ASSERT_EQ(Success, status.error_code); + ASSERT_EQ(milvus::Success, status.error_code); } { status = SerializeIndexToBinarySet(index, &binary_set); - ASSERT_EQ(Success, status.error_code); + ASSERT_EQ(milvus::Success, status.error_code); } { status = CreateIndex(dtype, type_params_str.c_str(), index_params_str.c_str(), ©_index); - ASSERT_EQ(Success, status.error_code); + ASSERT_EQ(milvus::Success, status.error_code); } { status = LoadIndexFromBinarySet(copy_index, binary_set); - ASSERT_EQ(Success, status.error_code); + ASSERT_EQ(milvus::Success, status.error_code); } { status = DeleteIndex(index); - ASSERT_EQ(Success, status.error_code); + ASSERT_EQ(milvus::Success, status.error_code); } { status = DeleteIndex(copy_index); - ASSERT_EQ(Success, status.error_code); + ASSERT_EQ(milvus::Success, status.error_code); } { DeleteBinarySet(binary_set); } } - delete[](char*)(str_ds->GetTensor()); + delete[] (char*)(str_ds->GetTensor()); } #endif diff --git a/internal/core/unittest/test_indexing.cpp b/internal/core/unittest/test_indexing.cpp index 59a38322c2..1c6fbdc5c1 100644 --- a/internal/core/unittest/test_indexing.cpp +++ b/internal/core/unittest/test_indexing.cpp @@ -16,7 +16,9 @@ #include #include +#include "common/EasyAssert.h" #include "knowhere/comp/index_param.h" +#include "nlohmann/json.hpp" #include "query/SearchBruteForce.h" #include "segcore/Reduce.h" #include "index/IndexFactory.h" @@ -55,7 +57,7 @@ generate_data(int N) { } } // namespace -Status +SegcoreError merge_into(int64_t queries, int64_t topk, float* distances, @@ -90,7 +92,7 @@ merge_into(int64_t queries, std::copy_n(buf_dis.data(), topk, src2_dis); std::copy_n(buf_uids.data(), topk, src2_uids); } - return Status::OK(); + return SegcoreError::success(); } /* @@ -173,7 +175,7 @@ TEST(Indexing, BinaryBruteForce) { auto json = SearchResultToJson(sr); std::cout << json.dump(2); #ifdef __linux__ - auto ref = json::parse(R"( + auto ref = nlohmann::json::parse(R"( [ [ [ "1024->0.000000", "48942->0.642000", "18494->0.644000", "68225->0.644000", "93557->0.644000" ], @@ -190,7 +192,7 @@ TEST(Indexing, BinaryBruteForce) { ] )"); #else // for mac - auto ref = json::parse(R"( + auto ref = nlohmann::json::parse(R"( [ [ [ "1024->0.000000", "59169->0.645000", "98548->0.646000", "3356->0.646000", "90373->0.647000" ], diff --git a/internal/core/unittest/test_parquet_c.cpp b/internal/core/unittest/test_parquet_c.cpp index 1084f04dbf..c25fde3033 100644 --- a/internal/core/unittest/test_parquet_c.cpp +++ b/internal/core/unittest/test_parquet_c.cpp @@ -17,11 +17,13 @@ #include #include +#include "common/EasyAssert.h" #include "storage/parquet_c.h" #include "storage/PayloadReader.h" #include "storage/PayloadWriter.h" namespace wrapper = milvus::storage; +using ErrorCode = milvus::ErrorCodeEnum; static void WriteToFile(CBuffer cb) { diff --git a/internal/core/unittest/test_query.cpp b/internal/core/unittest/test_query.cpp index 7b8798f512..de53f62994 100644 --- a/internal/core/unittest/test_query.cpp +++ b/internal/core/unittest/test_query.cpp @@ -22,6 +22,7 @@ #include "test_utils/AssertUtils.h" #include "test_utils/DataGen.h" +using json = nlohmann::json; using namespace milvus; using namespace milvus::query; using namespace milvus::segcore; @@ -31,9 +32,6 @@ const int64_t ROW_COUNT = 100 * 1000; } TEST(Query, ShowExecutor) { - using namespace milvus::query; - using namespace milvus::segcore; - using namespace milvus; auto metric_type = knowhere::metric::L2; auto node = std::make_unique(); auto schema = std::make_shared(); diff --git a/internal/core/unittest/test_tracer.cpp b/internal/core/unittest/test_tracer.cpp index a9d568bac9..7393b671e6 100644 --- a/internal/core/unittest/test_tracer.cpp +++ b/internal/core/unittest/test_tracer.cpp @@ -15,7 +15,7 @@ #include #include "common/Tracer.h" -#include "exceptions/EasyAssert.h" +#include "common/EasyAssert.h" using namespace milvus; using namespace milvus::tracer; diff --git a/internal/core/unittest/test_utils/DataGen.h b/internal/core/unittest/test_utils/DataGen.h index faf5651167..7cdb722508 100644 --- a/internal/core/unittest/test_utils/DataGen.h +++ b/internal/core/unittest/test_utils/DataGen.h @@ -105,15 +105,13 @@ struct GeneratedData { auto src_data = reinterpret_cast( target_field_data.vectors().binary_vector().data()); std::copy_n(src_data, len, ret.data()); - } else if (field_meta.get_data_type() == - DataType::VECTOR_FLOAT16) { + } else if (field_meta.get_data_type() == + DataType::VECTOR_FLOAT16) { // int len = raw_->num_rows() * field_meta.get_dim() * sizeof(float16); int len = raw_->num_rows() * field_meta.get_dim(); ret.resize(len); - auto src_data = - reinterpret_cast(target_field_data.vectors() - .float16_vector() - .data()); + auto src_data = reinterpret_cast( + target_field_data.vectors().float16_vector().data()); std::copy_n(src_data, len, ret.data()); } else { PanicInfo("unsupported"); @@ -269,7 +267,7 @@ DataGen(SchemaPtr schema, insert_cols(data, N, field_meta); break; } - case DataType::VECTOR_FLOAT16: { + case DataType::VECTOR_FLOAT16: { auto dim = field_meta.get_dim(); vector final(dim * N); for (auto& x : final) { @@ -433,7 +431,8 @@ DataGenForJsonArray(SchemaPtr schema, std::to_string(static_cast(er()))); stringVec.push_back("\"" + std::to_string(er()) + "\""); boolVec.push_back(i % 2 == 0 ? "true" : "false"); - arrayVec.push_back(fmt::format("[{}, {}, {}]", i, i+1, i+2)); + arrayVec.push_back( + fmt::format("[{}, {}, {}]", i, i + 1, i + 2)); } auto str = R"({"int":[)" + join(intVec, ",") + R"(],"double":[)" + join(doubleVec, ",") + @@ -604,7 +603,7 @@ SearchResultToVector(const SearchResult& sr) { return result; } -inline json +inline nlohmann::json SearchResultToJson(const SearchResult& sr) { int64_t num_queries = sr.total_nq_; int64_t topk = sr.unity_topK_; @@ -618,7 +617,7 @@ SearchResultToJson(const SearchResult& sr) { } results.emplace_back(std::move(result)); } - return json{results}; + return nlohmann::json{results}; }; inline storage::FieldDataPtr