From d41835585afa61dff7860220fe9d1e79cab44d4f Mon Sep 17 00:00:00 2001 From: groot Date: Fri, 30 Oct 2020 09:59:11 +0800 Subject: [PATCH] remove dependency of sqlite_orm (#4129) Signed-off-by: yhmo --- core/cmake/DefineOptions.cmake | 2 - core/cmake/ThirdPartyPackages.cmake | 61 ---------------------------- core/src/db/meta/MetaSchema.h | 15 +++++++ core/src/db/meta/SqliteMetaImpl.cpp | 62 ++++++++++++++++++++++------- 4 files changed, 63 insertions(+), 77 deletions(-) diff --git a/core/cmake/DefineOptions.cmake b/core/cmake/DefineOptions.cmake index 22c3b6edcd..244eeb4f19 100644 --- a/core/cmake/DefineOptions.cmake +++ b/core/cmake/DefineOptions.cmake @@ -69,8 +69,6 @@ define_option(MILVUS_WITH_PROMETHEUS "Build with PROMETHEUS library" ON) define_option(MILVUS_WITH_SQLITE "Build with SQLite library" ON) -define_option(MILVUS_WITH_SQLITE_ORM "Build with SQLite ORM library" ON) - define_option(MILVUS_WITH_MYSQLPP "Build with MySQL++" ON) define_option(MILVUS_WITH_YAMLCPP "Build with yaml-cpp library" ON) diff --git a/core/cmake/ThirdPartyPackages.cmake b/core/cmake/ThirdPartyPackages.cmake index 907009551a..aa27f7da22 100644 --- a/core/cmake/ThirdPartyPackages.cmake +++ b/core/cmake/ThirdPartyPackages.cmake @@ -15,7 +15,6 @@ set(MILVUS_THIRDPARTY_DEPENDENCIES MySQLPP Prometheus SQLite - SQLite_ORM yaml-cpp libunwind gperftools @@ -45,8 +44,6 @@ macro(build_dependency DEPENDENCY_NAME) build_prometheus() elseif ("${DEPENDENCY_NAME}" STREQUAL "SQLite") build_sqlite() - elseif ("${DEPENDENCY_NAME}" STREQUAL "SQLite_ORM") - build_sqlite_orm() elseif ("${DEPENDENCY_NAME}" STREQUAL "yaml-cpp") build_yamlcpp() elseif ("${DEPENDENCY_NAME}" STREQUAL "libunwind") @@ -611,64 +608,6 @@ if (MILVUS_WITH_SQLITE) link_directories(SYSTEM ${SQLITE_PREFIX}/lib/) endif () -# ---------------------------------------------------------------------- -# SQLite_ORM - -macro(build_sqlite_orm) - message(STATUS "Building SQLITE_ORM-${SQLITE_ORM_VERSION} from source") - - set(SQLITE_ORM_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/sqlite_orm_ep-prefix") - set(SQLITE_ORM_TAR_NAME "${SQLITE_ORM_PREFIX}/sqlite_orm-${SQLITE_ORM_VERSION}.tar.gz") - set(SQLITE_ORM_INCLUDE_DIR "${SQLITE_ORM_PREFIX}/sqlite_orm-${SQLITE_ORM_VERSION}/include/sqlite_orm") - - if (NOT EXISTS ${SQLITE_ORM_INCLUDE_DIR}) - file(MAKE_DIRECTORY ${SQLITE_ORM_PREFIX}) - - set(IS_EXIST_FILE FALSE) - foreach(url ${SQLITE_ORM_SOURCE_URLS}) - file(DOWNLOAD ${url} - ${SQLITE_ORM_TAR_NAME} - TIMEOUT 60 - STATUS status - LOG log) - list(GET status 0 status_code) - list(GET status 1 status_string) - - if(status_code EQUAL 0) - message(STATUS "Downloading SQLITE_ORM ... done") - set(IS_EXIST_FILE TRUE) - break() - else() - string(APPEND logFailedURLs "error: downloading '${url}' failed - status_code: ${status_code} - status_string: ${status_string} - log: - --- LOG BEGIN --- - ${log} - --- LOG END --- - " - ) - endif() - endforeach() - - if(IS_EXIST_FILE STREQUAL "FALSE") - message(FATAL_ERROR "Each download failed! - ${logFailedURLs} - " - ) - endif() - execute_process(COMMAND ${CMAKE_COMMAND} -E tar -xf ${SQLITE_ORM_TAR_NAME} - WORKING_DIRECTORY ${SQLITE_ORM_PREFIX}) - - endif () - -endmacro() - -if (MILVUS_WITH_SQLITE_ORM) - resolve_dependency(SQLite_ORM) - include_directories(SYSTEM "${SQLITE_ORM_INCLUDE_DIR}") -endif () - # ---------------------------------------------------------------------- # yaml-cpp diff --git a/core/src/db/meta/MetaSchema.h b/core/src/db/meta/MetaSchema.h index 2a5acfb096..adbbe1a7f6 100644 --- a/core/src/db/meta/MetaSchema.h +++ b/core/src/db/meta/MetaSchema.h @@ -29,6 +29,16 @@ class MetaField { return name_; } + const std::string& + type() const { + return type_; + } + + const std::string& + setting() const { + return setting_; + } + std::string ToString() const { return name_ + " " + type_ + " " + setting_; @@ -61,6 +71,11 @@ class MetaSchema { bool IsEqual(const MetaFields& fields) const; + const MetaFields& + Fields() const { + return fields_; + } + private: std::string name_; MetaFields fields_; diff --git a/core/src/db/meta/SqliteMetaImpl.cpp b/core/src/db/meta/SqliteMetaImpl.cpp index 2ead90942b..44ffe2aeee 100644 --- a/core/src/db/meta/SqliteMetaImpl.cpp +++ b/core/src/db/meta/SqliteMetaImpl.cpp @@ -195,20 +195,54 @@ SqliteMetaImpl::ValidateMetaSchema() { throw Exception(DB_ERROR, "Connector is null pointer"); } -// // old meta could be recreated since schema changed, throw exception if meta schema is not compatible -// auto ret = ConnectorPtr->sync_schema_simulate(); -// if (ret.find(META_TABLES) != ret.end() && -// sqlite_orm::sync_schema_result::dropped_and_recreated == ret[META_TABLES]) { -// throw Exception(DB_INCOMPATIB_META, "Meta Tables schema is created by Milvus old version"); -// } -// if (ret.find(META_FIELDS) != ret.end() && -// sqlite_orm::sync_schema_result::dropped_and_recreated == ret[META_FIELDS]) { -// throw Exception(DB_INCOMPATIB_META, "Meta Tables schema is created by Milvus old version"); -// } -// if (ret.find(META_TABLEFILES) != ret.end() && -// sqlite_orm::sync_schema_result::dropped_and_recreated == ret[META_TABLEFILES]) { -// throw Exception(DB_INCOMPATIB_META, "Meta TableFiles schema is created by Milvus old version"); -// } + auto validate_schema = [&](const MetaSchema& schema) -> bool { + auto& fields = schema.Fields(); + for (auto& field : fields) { + const char *data_type = nullptr; + const char *collseq = nullptr; + int not_null = 0, primary_key = 0, autoinc = 0; + int ret = sqlite3_table_column_metadata(db_, nullptr, schema.name().c_str(), field.name().c_str(), + &data_type, &collseq, ¬_null, &primary_key, &autoinc); + if (ret == SQLITE_OK) { + std::string str_type(data_type); + if (str_type != field.type()) { + return false; + } + std::string settings = field.setting(); + if (primary_key) { + auto poz = settings.find("PRIMARY KEY"); + if (poz == std::string::npos) { + return false; + } + } + if (not_null) { + auto poz = settings.find("NOT NULL"); + if (poz == std::string::npos) { + return false; + } + } + } + } + + return true; + }; + + // old meta could be recreated since schema changed, throw exception if meta schema is not compatible + if (!validate_schema(TABLES_SCHEMA)) { + throw Exception(DB_INCOMPATIB_META, "Meta Tables schema is created by Milvus old version"); + } + + if (!validate_schema(TABLEFILES_SCHEMA)) { + throw Exception(DB_INCOMPATIB_META, "Meta TableFiles schema is created by Milvus old version"); + } + + if (!validate_schema(FIELDS_SCHEMA)) { + throw Exception(DB_INCOMPATIB_META, "Meta Fields schema is created by Milvus old version"); + } + + if (!validate_schema(ENVIRONMENT_SCHEMA)) { + throw Exception(DB_INCOMPATIB_META, "Meta Environment schema is created by Milvus old version"); + } } Status