diff --git a/.gitignore b/.gitignore index 6b2d6fc97b..8fda9f2980 100644 --- a/.gitignore +++ b/.gitignore @@ -26,3 +26,6 @@ cmake_build *.lo *.tar.gz *.log +.coverage +*.pyc +cov_html/ diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000000..c4729a2555 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,20 @@ +language: cpp +sudo: required +dist: bionic + +cache: + directories: + - $HOME/.ccache + +addons: + apt: + update: true + +before_install: + - source ci/travis/before-install.sh + +install: + - source $TRAVIS_BUILD_DIR/ci/travis/install_dependency.sh + +script: + - $TRAVIS_BUILD_DIR/ci/travis/travis_build.sh diff --git a/CHANGELOG.md b/CHANGELOG.md index a0667d8a32..e7c52eb5bb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,50 @@ # Changelog Please mark all change in change log and use the ticket from JIRA. +# Milvus 0.6.0 (TODO) + +## Bug +- \#228 - memory usage increased slowly during searching vectors +- \#246 - Exclude src/external folder from code coverage for jenkin ci +- \#248 - Reside src/external in thirdparty +- \#316 - Some files not merged after vectors added +- \#327 - Search does not use GPU when index type is FLAT +- \#340 - Test cases run failed on 0.6.0 +- \#353 - Rename config.h.in to version.h.in +- \#374 - sdk_simple return empty result +- \#377 - Create partition success if tag name only contains spaces +- \#397 - sdk_simple return incorrect result +- \#399 - Create partition should be failed if partition tag existed +- \#412 - Message returned is confused when partition created with null partition name +- \#416 - Drop the same partition success repeatally +- \#440 - Query API in customization still uses old version + +## Feature +- \#12 - Pure CPU version for Milvus +- \#77 - Support table partition +- \#127 - Support new Index type IVFPQ +- \#226 - Experimental shards middleware for Milvus +- \#227 - Support new index types SPTAG-KDT and SPTAG-BKT +- \#346 - Support build index with multiple gpu + +## Improvement +- \#255 - Add ivfsq8 test report detailed version +- \#260 - C++ SDK README +- \#266 - Rpc request source code refactor +- \#275 - Rename C++ SDK IndexType +- \#284 - Change C++ SDK to shared library +- \#306 - Use int64 for all config integer +- \#310 - Add Q&A for 'protocol https not supported or disable in libcurl' issue +- \#314 - add Find FAISS in CMake +- \#322 - Add option to enable / disable prometheus +- \#358 - Add more information in build.sh and install.md +- \#404 - Add virtual method Init() in Pass abstract class +- \#409 - Add a Fallback pass in optimizer +- \#433 - C++ SDK query result is not easy to use +- \#449 - Add ShowPartitions example for C++ SDK + +## Task + # Milvus 0.5.3 (2019-11-13) ## Bug @@ -87,7 +131,7 @@ Please mark all change in change log and use the ticket from JIRA. - MS-658 - Fix SQ8 Hybrid can't search - MS-665 - IVF_SQ8H search crash when no GPU resource in search_resources - \#9 - Change default gpu_cache_capacity to 4 -- \#20 - C++ sdk example get grpc error +- \#20 - C++ sdk example get grpc error - \#23 - Add unittest to improve code coverage - \#31 - make clang-format failed after run build.sh -l - \#39 - Create SQ8H index hang if using github server version @@ -139,7 +183,7 @@ Please mark all change in change log and use the ticket from JIRA. - MS-635 - Add compile option to support customized faiss - MS-660 - add ubuntu_build_deps.sh - \#18 - Add all test cases - + # Milvus 0.4.0 (2019-09-12) ## Bug @@ -348,11 +392,11 @@ Please mark all change in change log and use the ticket from JIRA. - MS-82 - Update server startup welcome message - MS-83 - Update vecwise to Milvus - MS-77 - Performance issue of post-search action -- MS-22 - Enhancement for MemVector size control +- MS-22 - Enhancement for MemVector size control - MS-92 - Unify behavior of debug and release build - MS-98 - Install all unit test to installation directory - MS-115 - Change is_startup of metric_config switch from true to on -- MS-122 - Archive criteria config +- MS-122 - Archive criteria config - MS-124 - HasTable interface - MS-126 - Add more error code - MS-128 - Change default db path diff --git a/README.md b/README.md index d8d5d80b11..110f0b6c1e 100644 --- a/README.md +++ b/README.md @@ -78,6 +78,4 @@ Below is a list of Milvus contributors. We greatly appreciate your contributions ## License -[Apache License 2.0](LICENSE) - - +[Apache License 2.0](LICENSE) \ No newline at end of file diff --git a/README_CN.md b/README_CN.md index df5445d931..374cefa9bd 100644 --- a/README_CN.md +++ b/README_CN.md @@ -78,4 +78,3 @@ Milvus 提供稳定的 [Python](https://github.com/milvus-io/pymilvus)、[Java]( ## 许可协议 [Apache 许可协议2.0版](https://github.com/milvus-io/milvus/blob/master/LICENSE) - diff --git a/README_JP.md b/README_JP.md index 4a1d67738d..65e68c7bec 100644 --- a/README_JP.md +++ b/README_JP.md @@ -72,4 +72,4 @@ C++サンプルコードを実行するために、次のコマンドをつか ## ライセンス -[Apache 2.0ライセンス](LICENSE) +[Apache 2.0ライセンス](LICENSE) \ No newline at end of file diff --git a/ci/jenkins/Jenkinsfile b/ci/jenkins/Jenkinsfile index 8b3ab730f8..8d3953b112 100644 --- a/ci/jenkins/Jenkinsfile +++ b/ci/jenkins/Jenkinsfile @@ -2,7 +2,7 @@ String cron_timezone = "TZ=Asia/Shanghai" String cron_string = BRANCH_NAME == "master" ? "H 0 * * * " : "" -cron_string = BRANCH_NAME == "0.5.2" ? "H 1 * * * " : cron_string +cron_string = BRANCH_NAME == "0.6.0" ? "H 1 * * * " : cron_string pipeline { agent none @@ -33,128 +33,267 @@ pipeline { } stages { - stage("Ubuntu 18.04") { + stage("Ubuntu 18.04 x86_64") { environment { OS_NAME = "ubuntu18.04" - PACKAGE_VERSION = VersionNumber([ - versionNumberString : '${SEMVER}-${LOWER_BUILD_TYPE}-ubuntu18.04-x86_64-${BUILD_DATE_FORMATTED, "yyyyMMdd"}-${BUILDS_TODAY}' - ]); - DOCKER_VERSION = "${SEMVER}-${OS_NAME}-${LOWER_BUILD_TYPE}" + CPU_ARCH = "amd64" } - stages { - stage("Run Build") { - agent { - kubernetes { - label 'build' - defaultContainer 'jnlp' - yamlFile 'ci/jenkins/pod/milvus-build-env-pod.yaml' - } + parallel { + stage ("GPU Version") { + environment { + BINRARY_VERSION = "gpu" + PACKAGE_VERSION = VersionNumber([ + versionNumberString : '${SEMVER}-gpu-${OS_NAME}-${CPU_ARCH}-${LOWER_BUILD_TYPE}-${BUILD_DATE_FORMATTED, "yyyyMMdd"}-${BUILDS_TODAY}' + ]); + DOCKER_VERSION = "${SEMVER}-gpu-${OS_NAME}-${LOWER_BUILD_TYPE}" } stages { - stage('Build') { - steps { - container('milvus-build-env') { - script { - load "${env.WORKSPACE}/ci/jenkins/step/build.groovy" - } + stage("Run Build") { + agent { + kubernetes { + label "${BINRARY_VERSION}-build" + defaultContainer 'jnlp' + yamlFile 'ci/jenkins/pod/milvus-gpu-version-build-env-pod.yaml' } } - } - stage('Code Coverage') { - steps { - container('milvus-build-env') { - script { - load "${env.WORKSPACE}/ci/jenkins/step/coverage.groovy" + + stages { + stage('Build') { + steps { + container('milvus-build-env') { + script { + load "${env.WORKSPACE}/ci/jenkins/step/build.groovy" + } + } } } - } - } - stage('Upload Package') { - steps { - container('milvus-build-env') { - script { - load "${env.WORKSPACE}/ci/jenkins/step/packaging.groovy" + stage('Code Coverage') { + steps { + container('milvus-build-env') { + script { + load "${env.WORKSPACE}/ci/jenkins/step/coverage.groovy" + } + } } } - } - } - } - } - - stage("Publish docker images") { - agent { - kubernetes { - label 'publish' - defaultContainer 'jnlp' - yamlFile 'ci/jenkins/pod/docker-pod.yaml' - } - } - - stages { - stage('Publish') { - steps { - container('publish-images'){ - script { - load "${env.WORKSPACE}/ci/jenkins/step/publishImages.groovy" - } - } - } - } - } - } - - stage("Deploy to Development") { - agent { - kubernetes { - label 'dev-test' - defaultContainer 'jnlp' - yamlFile 'ci/jenkins/pod/testEnvironment.yaml' - } - } - - stages { - stage("Deploy to Dev") { - steps { - container('milvus-test-env') { - script { - load "${env.WORKSPACE}/ci/jenkins/step/deploySingle2Dev.groovy" - } - } - } - } - - stage("Dev Test") { - steps { - container('milvus-test-env') { - script { - boolean isNightlyTest = isTimeTriggeredBuild() - if (isNightlyTest) { - load "${env.WORKSPACE}/ci/jenkins/step/singleDevNightlyTest.groovy" - } else { - load "${env.WORKSPACE}/ci/jenkins/step/singleDevTest.groovy" + stage('Upload Package') { + steps { + container('milvus-build-env') { + script { + load "${env.WORKSPACE}/ci/jenkins/step/package.groovy" + } } } } } } - stage ("Cleanup Dev") { - steps { - container('milvus-test-env') { - script { - load "${env.WORKSPACE}/ci/jenkins/step/cleanupSingleDev.groovy" + stage("Publish docker images") { + agent { + kubernetes { + label "${BINRARY_VERSION}-publish" + defaultContainer 'jnlp' + yamlFile 'ci/jenkins/pod/docker-pod.yaml' + } + } + + stages { + stage('Publish') { + steps { + container('publish-images'){ + script { + load "${env.WORKSPACE}/ci/jenkins/step/publishImages.groovy" + } + } + } + } + } + } + + stage("Deploy to Development") { + agent { + kubernetes { + label "${BINRARY_VERSION}-dev-test" + defaultContainer 'jnlp' + yamlFile 'ci/jenkins/pod/testEnvironment.yaml' + } + } + + stages { + stage("Deploy to Dev") { + steps { + container('milvus-test-env') { + script { + load "${env.WORKSPACE}/ci/jenkins/step/deploySingle2Dev.groovy" + } + } + } + } + + stage("Dev Test") { + steps { + container('milvus-test-env') { + script { + boolean isNightlyTest = isTimeTriggeredBuild() + if (isNightlyTest) { + load "${env.WORKSPACE}/ci/jenkins/step/singleDevNightlyTest.groovy" + } else { + load "${env.WORKSPACE}/ci/jenkins/step/singleDevTest.groovy" + } + } + } + } + } + + stage ("Cleanup Dev") { + steps { + container('milvus-test-env') { + script { + load "${env.WORKSPACE}/ci/jenkins/step/cleanupSingleDev.groovy" + } + } + } + } + } + post { + unsuccessful { + container('milvus-test-env') { + script { + load "${env.WORKSPACE}/ci/jenkins/step/cleanupSingleDev.groovy" + } } } } } } - post { - unsuccessful { - container('milvus-test-env') { - script { - load "${env.WORKSPACE}/ci/jenkins/step/cleanupSingleDev.groovy" + } + + stage ("CPU Version") { + environment { + BINRARY_VERSION = "cpu" + PACKAGE_VERSION = VersionNumber([ + versionNumberString : '${SEMVER}-cpu-${OS_NAME}-${CPU_ARCH}-${LOWER_BUILD_TYPE}-${BUILD_DATE_FORMATTED, "yyyyMMdd"}-${BUILDS_TODAY}' + ]); + DOCKER_VERSION = "${SEMVER}-cpu-${OS_NAME}-${LOWER_BUILD_TYPE}" + } + + stages { + stage("Run Build") { + agent { + kubernetes { + label "${BINRARY_VERSION}-build" + defaultContainer 'jnlp' + yamlFile 'ci/jenkins/pod/milvus-cpu-version-build-env-pod.yaml' + } + } + + stages { + stage('Build') { + steps { + container('milvus-build-env') { + script { + load "${env.WORKSPACE}/ci/jenkins/step/build.groovy" + } + } + } + } + stage('Code Coverage') { + steps { + container('milvus-build-env') { + script { + load "${env.WORKSPACE}/ci/jenkins/step/coverage.groovy" + } + } + } + } + stage('Upload Package') { + steps { + container('milvus-build-env') { + script { + load "${env.WORKSPACE}/ci/jenkins/step/package.groovy" + } + } + } + } + } + } + + stage("Publish docker images") { + agent { + kubernetes { + label "${BINRARY_VERSION}-publish" + defaultContainer 'jnlp' + yamlFile 'ci/jenkins/pod/docker-pod.yaml' + } + } + + stages { + stage('Publish') { + steps { + container('publish-images'){ + script { + load "${env.WORKSPACE}/ci/jenkins/step/publishImages.groovy" + } + } + } + } + } + } + + stage("Deploy to Development") { + agent { + kubernetes { + label "${BINRARY_VERSION}-dev-test" + defaultContainer 'jnlp' + yamlFile 'ci/jenkins/pod/testEnvironment.yaml' + } + } + + stages { + stage("Deploy to Dev") { + steps { + container('milvus-test-env') { + script { + load "${env.WORKSPACE}/ci/jenkins/step/deploySingle2Dev.groovy" + } + } + } + } + + stage("Dev Test") { + steps { + container('milvus-test-env') { + script { + boolean isNightlyTest = isTimeTriggeredBuild() + if (isNightlyTest) { + load "${env.WORKSPACE}/ci/jenkins/step/singleDevNightlyTest.groovy" + } else { + load "${env.WORKSPACE}/ci/jenkins/step/singleDevTest.groovy" + } + } + } + } + } + + stage ("Cleanup Dev") { + steps { + container('milvus-test-env') { + script { + load "${env.WORKSPACE}/ci/jenkins/step/cleanupSingleDev.groovy" + } + } + } + } + } + post { + unsuccessful { + container('milvus-test-env') { + script { + load "${env.WORKSPACE}/ci/jenkins/step/cleanupSingleDev.groovy" + } + } } } } diff --git a/ci/jenkins/pod/milvus-cpu-version-build-env-pod.yaml b/ci/jenkins/pod/milvus-cpu-version-build-env-pod.yaml new file mode 100644 index 0000000000..561bfe8140 --- /dev/null +++ b/ci/jenkins/pod/milvus-cpu-version-build-env-pod.yaml @@ -0,0 +1,34 @@ +apiVersion: v1 +kind: Pod +metadata: + name: milvus-cpu-build-env + labels: + app: milvus + componet: cpu-build-env +spec: + containers: + - name: milvus-build-env + image: registry.zilliz.com/milvus/milvus-cpu-build-env:v0.6.0-ubuntu18.04 + env: + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + command: + - cat + tty: true + resources: + limits: + memory: "32Gi" + cpu: "8.0" + requests: + memory: "16Gi" + cpu: "4.0" + - name: milvus-mysql + image: mysql:5.6 + env: + - name: MYSQL_ROOT_PASSWORD + value: 123456 + ports: + - containerPort: 3306 + name: mysql diff --git a/ci/jenkins/pod/milvus-build-env-pod.yaml b/ci/jenkins/pod/milvus-gpu-version-build-env-pod.yaml similarity index 80% rename from ci/jenkins/pod/milvus-build-env-pod.yaml rename to ci/jenkins/pod/milvus-gpu-version-build-env-pod.yaml index da938d8ba2..422dd72ab2 100644 --- a/ci/jenkins/pod/milvus-build-env-pod.yaml +++ b/ci/jenkins/pod/milvus-gpu-version-build-env-pod.yaml @@ -1,14 +1,14 @@ apiVersion: v1 kind: Pod metadata: - name: milvus-build-env + name: milvus-gpu-build-env labels: app: milvus - componet: build-env + componet: gpu-build-env spec: containers: - name: milvus-build-env - image: registry.zilliz.com/milvus/milvus-build-env:v0.5.1-ubuntu18.04 + image: registry.zilliz.com/milvus/milvus-gpu-build-env:v0.6.0-ubuntu18.04 env: - name: POD_IP valueFrom: diff --git a/ci/jenkins/scripts/packaging.sh b/ci/jenkins/scripts/packaging.sh deleted file mode 100755 index 7687cbf33f..0000000000 --- a/ci/jenkins/scripts/packaging.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/usr/bin/env bash - -set -ex - -pip3 install -r requirements.txt -./yaml_processor.py merge -f /opt/milvus/conf/server_config.yaml -m ../yaml/update_server_config.yaml -i && \ -rm /opt/milvus/conf/server_config.yaml.bak - -if [ -d "/opt/milvus/unittest" ]; then - rm -rf "/opt/milvus/unittest" -fi - -tar -zcvf ./${PROJECT_NAME}-${PACKAGE_VERSION}.tar.gz -C /opt/ milvus \ No newline at end of file diff --git a/ci/jenkins/step/build.groovy b/ci/jenkins/step/build.groovy index 14d0414f4f..6c1da64a82 100644 --- a/ci/jenkins/step/build.groovy +++ b/ci/jenkins/step/build.groovy @@ -1,9 +1,11 @@ timeout(time: 60, unit: 'MINUTES') { - dir ("ci/jenkins/scripts") { - sh "./build.sh -l" + dir ("ci/scripts") { withCredentials([usernamePassword(credentialsId: "${params.JFROG_CREDENTIALS_ID}", usernameVariable: 'USERNAME', passwordVariable: 'PASSWORD')]) { - sh "export JFROG_ARTFACTORY_URL='${params.JFROG_ARTFACTORY_URL}' && export JFROG_USER_NAME='${USERNAME}' && export JFROG_PASSWORD='${PASSWORD}' && ./build.sh -t ${params.BUILD_TYPE} -o /opt/milvus -d /opt/milvus -j -u -c" + if ("${env.BINRARY_VERSION}" == "gpu") { + sh "export JFROG_ARTFACTORY_URL='${params.JFROG_ARTFACTORY_URL}' && export JFROG_USER_NAME='${USERNAME}' && export JFROG_PASSWORD='${PASSWORD}' && ./build.sh -t ${params.BUILD_TYPE} -o /opt/milvus -l -g -j -u -c" + } else { + sh "export JFROG_ARTFACTORY_URL='${params.JFROG_ARTFACTORY_URL}' && export JFROG_USER_NAME='${USERNAME}' && export JFROG_PASSWORD='${PASSWORD}' && ./build.sh -t ${params.BUILD_TYPE} -o /opt/milvus -l -m -j -u -c" + } } } } - diff --git a/ci/jenkins/step/cleanupSingleDev.groovy b/ci/jenkins/step/cleanupSingleDev.groovy index 3b8c1833b5..30325e0c91 100644 --- a/ci/jenkins/step/cleanupSingleDev.groovy +++ b/ci/jenkins/step/cleanupSingleDev.groovy @@ -1,12 +1,12 @@ try { - def helmResult = sh script: "helm status ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-gpu", returnStatus: true + def helmResult = sh script: "helm status ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-${env.BINRARY_VERSION}", returnStatus: true if (!helmResult) { - sh "helm del --purge ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-gpu" + sh "helm del --purge ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-${env.BINRARY_VERSION}" } } catch (exc) { - def helmResult = sh script: "helm status ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-gpu", returnStatus: true + def helmResult = sh script: "helm status ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-${env.BINRARY_VERSION}", returnStatus: true if (!helmResult) { - sh "helm del --purge ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-gpu" + sh "helm del --purge ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-${env.BINRARY_VERSION}" } throw exc } diff --git a/ci/jenkins/step/coverage.groovy b/ci/jenkins/step/coverage.groovy index ff2e3e6fa2..75ac7b48b5 100644 --- a/ci/jenkins/step/coverage.groovy +++ b/ci/jenkins/step/coverage.groovy @@ -1,5 +1,5 @@ timeout(time: 30, unit: 'MINUTES') { - dir ("ci/jenkins/scripts") { + dir ("ci/scripts") { sh "./coverage.sh -o /opt/milvus -u root -p 123456 -t \$POD_IP" // Set some env variables so codecov detection script works correctly withCredentials([[$class: 'StringBinding', credentialsId: "${env.PIPELINE_NAME}-codecov-token", variable: 'CODECOV_TOKEN']]) { diff --git a/ci/jenkins/step/deploySingle2Dev.groovy b/ci/jenkins/step/deploySingle2Dev.groovy index 69be2dfe31..7b479ff44a 100644 --- a/ci/jenkins/step/deploySingle2Dev.groovy +++ b/ci/jenkins/step/deploySingle2Dev.groovy @@ -1,9 +1,9 @@ sh 'helm init --client-only --skip-refresh --stable-repo-url https://kubernetes.oss-cn-hangzhou.aliyuncs.com/charts' sh 'helm repo update' dir ('milvus-helm') { - checkout([$class: 'GitSCM', branches: [[name: "0.5.3"]], userRemoteConfigs: [[url: "https://github.com/milvus-io/milvus-helm.git", name: 'origin', refspec: "+refs/heads/0.5.3:refs/remotes/origin/0.5.3"]]]) - dir ("milvus-gpu") { - sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-gpu -f ci/db_backend/sqlite_values.yaml -f ci/filebeat/values.yaml --namespace milvus ." + checkout([$class: 'GitSCM', branches: [[name: "0.6.0"]], userRemoteConfigs: [[url: "https://github.com/milvus-io/milvus-helm.git", name: 'origin', refspec: "+refs/heads/0.6.0:refs/remotes/origin/0.6.0"]]]) + dir ("milvus") { + sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-${env.BINRARY_VERSION} -f ci/db_backend/sqlite_${env.BINRARY_VERSION}_values.yaml -f ci/filebeat/values.yaml --namespace milvus ." } } diff --git a/ci/jenkins/step/package.groovy b/ci/jenkins/step/package.groovy new file mode 100644 index 0000000000..c07dcd9d88 --- /dev/null +++ b/ci/jenkins/step/package.groovy @@ -0,0 +1,13 @@ +timeout(time: 5, unit: 'MINUTES') { + dir ("ci/jenkins/scripts") { + sh "pip3 install -r requirements.txt" + sh "./yaml_processor.py merge -f /opt/milvus/conf/server_config.yaml -m ../yaml/update_server_config.yaml -i && rm /opt/milvus/conf/server_config.yaml.bak" + } + sh "tar -zcvf ./${PROJECT_NAME}-${PACKAGE_VERSION}.tar.gz -C /opt/ milvus" + withCredentials([usernamePassword(credentialsId: "${params.JFROG_CREDENTIALS_ID}", usernameVariable: 'JFROG_USERNAME', passwordVariable: 'JFROG_PASSWORD')]) { + def uploadStatus = sh(returnStatus: true, script: "curl -u${JFROG_USERNAME}:${JFROG_PASSWORD} -T ./${PROJECT_NAME}-${PACKAGE_VERSION}.tar.gz ${params.JFROG_ARTFACTORY_URL}/milvus/package/${PROJECT_NAME}-${PACKAGE_VERSION}.tar.gz") + if (uploadStatus != 0) { + error("\" ${PROJECT_NAME}-${PACKAGE_VERSION}.tar.gz \" upload to \" ${params.JFROG_ARTFACTORY_URL}/milvus/package/${PROJECT_NAME}-${PACKAGE_VERSION}.tar.gz \" failed!") + } + } +} diff --git a/ci/jenkins/step/packaging.groovy b/ci/jenkins/step/packaging.groovy deleted file mode 100644 index ce1653bb09..0000000000 --- a/ci/jenkins/step/packaging.groovy +++ /dev/null @@ -1,11 +0,0 @@ -timeout(time: 5, unit: 'MINUTES') { - dir ("ci/jenkins/scripts") { - sh "./packaging.sh" - withCredentials([usernamePassword(credentialsId: "${params.JFROG_CREDENTIALS_ID}", usernameVariable: 'JFROG_USERNAME', passwordVariable: 'JFROG_PASSWORD')]) { - def uploadStatus = sh(returnStatus: true, script: "curl -u${JFROG_USERNAME}:${JFROG_PASSWORD} -T ./${PROJECT_NAME}-${PACKAGE_VERSION}.tar.gz ${params.JFROG_ARTFACTORY_URL}/milvus/package/${PROJECT_NAME}-${PACKAGE_VERSION}.tar.gz") - if (uploadStatus != 0) { - error("\" ${PROJECT_NAME}-${PACKAGE_VERSION}.tar.gz \" upload to \" ${params.JFROG_ARTFACTORY_URL}/milvus/package/${PROJECT_NAME}-${PACKAGE_VERSION}.tar.gz \" failed!") - } - } - } -} diff --git a/ci/jenkins/step/publishImages.groovy b/ci/jenkins/step/publishImages.groovy index 62df0c73bf..72e9924c62 100644 --- a/ci/jenkins/step/publishImages.groovy +++ b/ci/jenkins/step/publishImages.groovy @@ -1,6 +1,6 @@ container('publish-images') { timeout(time: 15, unit: 'MINUTES') { - dir ("docker/deploy/${OS_NAME}") { + dir ("docker/deploy/${env.BINRARY_VERSION}/${env.OS_NAME}") { def binaryPackage = "${PROJECT_NAME}-${PACKAGE_VERSION}.tar.gz" withCredentials([usernamePassword(credentialsId: "${params.JFROG_CREDENTIALS_ID}", usernameVariable: 'JFROG_USERNAME', passwordVariable: 'JFROG_PASSWORD')]) { diff --git a/ci/jenkins/step/singleDevNightlyTest.groovy b/ci/jenkins/step/singleDevNightlyTest.groovy index 201132d2c3..cee8a092c1 100644 --- a/ci/jenkins/step/singleDevNightlyTest.groovy +++ b/ci/jenkins/step/singleDevNightlyTest.groovy @@ -1,22 +1,22 @@ timeout(time: 90, unit: 'MINUTES') { dir ("tests/milvus_python_test") { sh 'python3 -m pip install -r requirements.txt' - sh "pytest . --alluredir=\"test_out/dev/single/sqlite\" --ip ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-gpu-milvus-gpu-engine.milvus.svc.cluster.local" + sh "pytest . --alluredir=\"test_out/dev/single/sqlite\" --ip ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-${env.BINRARY_VERSION}-engine.milvus.svc.cluster.local" } // mysql database backend test - load "${env.WORKSPACE}/ci/jenkins/jenkinsfile/cleanupSingleDev.groovy" + load "ci/jenkins/jenkinsfile/cleanupSingleDev.groovy" if (!fileExists('milvus-helm')) { dir ("milvus-helm") { - checkout([$class: 'GitSCM', branches: [[name: "0.5.3"]], userRemoteConfigs: [[url: "https://github.com/milvus-io/milvus-helm.git", name: 'origin', refspec: "+refs/heads/0.5.3:refs/remotes/origin/0.5.3"]]]) + checkout([$class: 'GitSCM', branches: [[name: "0.6.0"]], userRemoteConfigs: [[url: "https://github.com/milvus-io/milvus-helm.git", name: 'origin', refspec: "+refs/heads/0.6.0:refs/remotes/origin/0.6.0"]]]) } } dir ("milvus-helm") { - dir ("milvus-gpu") { - sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-gpu -f ci/db_backend/mysql_values.yaml -f ci/filebeat/values.yaml --namespace milvus ." + dir ("milvus") { + sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-${env.BINRARY_VERSION} -f ci/db_backend/mysql_${env.BINRARY_VERSION}_values.yaml -f ci/filebeat/values.yaml --namespace milvus ." } } dir ("tests/milvus_python_test") { - sh "pytest . --alluredir=\"test_out/dev/single/mysql\" --ip ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-gpu-milvus-gpu-engine.milvus.svc.cluster.local" + sh "pytest . --alluredir=\"test_out/dev/single/mysql\" --ip ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-${env.BINRARY_VERSION}-engine.milvus.svc.cluster.local" } } diff --git a/ci/jenkins/step/singleDevTest.groovy b/ci/jenkins/step/singleDevTest.groovy index 7f3ce392b1..db0fdc0f3b 100644 --- a/ci/jenkins/step/singleDevTest.groovy +++ b/ci/jenkins/step/singleDevTest.groovy @@ -1,24 +1,23 @@ timeout(time: 60, unit: 'MINUTES') { dir ("tests/milvus_python_test") { sh 'python3 -m pip install -r requirements.txt' - sh "pytest . --alluredir=\"test_out/dev/single/sqlite\" --level=1 --ip ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-gpu-milvus-gpu-engine.milvus.svc.cluster.local" + sh "pytest . --alluredir=\"test_out/dev/single/sqlite\" --level=1 --ip ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-${env.BINRARY_VERSION}-engine.milvus.svc.cluster.local" } - // mysql database backend test - // load "${env.WORKSPACE}/ci/jenkins/jenkinsfile/cleanupSingleDev.groovy" - // Remove mysql-version tests: 10-28 + // mysql database backend test + // load "ci/jenkins/jenkinsfile/cleanupSingleDev.groovy" // if (!fileExists('milvus-helm')) { // dir ("milvus-helm") { - // checkout([$class: 'GitSCM', branches: [[name: "0.5.3"]], userRemoteConfigs: [[url: "https://github.com/milvus-io/milvus-helm.git", name: 'origin', refspec: "+refs/heads/0.5.3:refs/remotes/origin/0.5.3"]]]) + // checkout([$class: 'GitSCM', branches: [[name: "0.6.0"]], userRemoteConfigs: [[url: "https://github.com/milvus-io/milvus-helm.git", name: 'origin', refspec: "+refs/heads/0.6.0:refs/remotes/origin/0.6.0"]]]) // } // } // dir ("milvus-helm") { - // dir ("milvus-gpu") { - // sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-gpu -f ci/db_backend/mysql_values.yaml -f ci/filebeat/values.yaml --namespace milvus ." + // dir ("milvus") { + // sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-${env.BINRARY_VERSION} -f ci/db_backend/mysql_${env.BINRARY_VERSION}_values.yaml -f ci/filebeat/values.yaml --namespace milvus ." // } // } // dir ("tests/milvus_python_test") { - // sh "pytest . --alluredir=\"test_out/dev/single/mysql\" --level=1 --ip ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-gpu-milvus-gpu-engine.milvus.svc.cluster.local" + // sh "pytest . --alluredir=\"test_out/dev/single/mysql\" --level=1 --ip ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-${env.BINRARY_VERSION}-engine.milvus.svc.cluster.local" // } } diff --git a/ci/jenkinsfile/packaged_milvus.groovy b/ci/jenkinsfile/packaged_milvus.groovy index f185cb4c5f..1d30e21910 100644 --- a/ci/jenkinsfile/packaged_milvus.groovy +++ b/ci/jenkinsfile/packaged_milvus.groovy @@ -5,9 +5,6 @@ container('milvus-build-env') { gitlabCommitStatus(name: 'Packaged Engine') { if (fileExists('milvus')) { try { - if (fileExists('milvus/unittest')) { - sh "rm -rf ./milvus/unittest" - } sh "tar -zcvf ./${PROJECT_NAME}-engine-${PACKAGE_VERSION}.tar.gz ./milvus" def fileTransfer = load "${env.WORKSPACE}/ci/function/file_transfer.groovy" fileTransfer.FileTransfer("${PROJECT_NAME}-engine-${PACKAGE_VERSION}.tar.gz", "${PROJECT_NAME}/engine/${JOB_NAME}-${BUILD_ID}", 'nas storage') diff --git a/ci/jenkins/scripts/build.sh b/ci/scripts/build.sh similarity index 70% rename from ci/jenkins/scripts/build.sh rename to ci/scripts/build.sh index 2ccdf4a618..eb5205ad26 100755 --- a/ci/jenkins/scripts/build.sh +++ b/ci/scripts/build.sh @@ -1,5 +1,7 @@ #!/bin/bash +set -e + SOURCE="${BASH_SOURCE[0]}" while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )" @@ -8,38 +10,41 @@ while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symli done SCRIPTS_DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )" -CMAKE_BUILD_DIR="${SCRIPTS_DIR}/../../../core/cmake_build" +MILVUS_CORE_DIR="${SCRIPTS_DIR}/../../core" +CORE_BUILD_DIR="${MILVUS_CORE_DIR}/cmake_build" BUILD_TYPE="Debug" BUILD_UNITTEST="OFF" INSTALL_PREFIX="/opt/milvus" +FAISS_ROOT="" +CUSTOMIZATION="OFF" # default use origin faiss BUILD_COVERAGE="OFF" -DB_PATH="/opt/milvus" -PROFILING="OFF" USE_JFROG_CACHE="OFF" RUN_CPPLINT="OFF" -CUSTOMIZATION="OFF" # default use ori faiss +GPU_VERSION="OFF" +WITH_MKL="OFF" CUDA_COMPILER=/usr/local/cuda/bin/nvcc -CUSTOMIZED_FAISS_URL="${FAISS_URL:-NONE}" -wget -q --method HEAD ${CUSTOMIZED_FAISS_URL} -if [ $? -eq 0 ]; then - CUSTOMIZATION="ON" -else - CUSTOMIZATION="OFF" -fi - -while getopts "o:d:t:ulcgjhx" arg +while getopts "o:t:b:f:gxulcjmh" arg do case $arg in o) INSTALL_PREFIX=$OPTARG ;; - d) - DB_PATH=$OPTARG - ;; t) BUILD_TYPE=$OPTARG # BUILD_TYPE ;; + b) + CORE_BUILD_DIR=$OPTARG # CORE_BUILD_DIR + ;; + f) + FAISS_ROOT=$OPTARG # FAISS ROOT PATH + ;; + g) + GPU_VERSION="ON"; + ;; + x) + CUSTOMIZATION="ON"; + ;; u) echo "Build and run unittest cases" ; BUILD_UNITTEST="ON"; @@ -50,31 +55,31 @@ do c) BUILD_COVERAGE="ON" ;; - g) - PROFILING="ON" - ;; j) USE_JFROG_CACHE="ON" ;; - x) - CUSTOMIZATION="OFF" # force use ori faiss + m) + WITH_MKL="ON" ;; h) # help echo " parameter: -o: install prefix(default: /opt/milvus) --d: db data path(default: /opt/milvus) -t: build type(default: Debug) +-b: core code build directory +-f: faiss root path +-g: gpu version +-x: milvus customization (default: OFF) -u: building unit test options(default: OFF) -l: run cpplint, clang-format and clang-tidy(default: OFF) -c: code coverage(default: OFF) --g: profiling(default: OFF) -j: use jfrog cache build directory(default: OFF) +-m: build with MKL(default: OFF) -h: help usage: -./build.sh -p \${INSTALL_PREFIX} -t \${BUILD_TYPE} [-u] [-l] [-r] [-c] [-g] [-j] [-h] +./build.sh -o \${INSTALL_PREFIX} -t \${BUILD_TYPE} -b \${CORE_BUILD_DIR} -f \${FAISS_ROOT} [-g] [-x] [-u] [-l] [-c] [-j] [-m] [-h] " exit 0 ;; @@ -85,31 +90,30 @@ usage: esac done -if [[ ! -d ${CMAKE_BUILD_DIR} ]]; then - mkdir ${CMAKE_BUILD_DIR} +if [[ ! -d ${CORE_BUILD_DIR} ]]; then + mkdir ${CORE_BUILD_DIR} fi -cd ${CMAKE_BUILD_DIR} - -# remove make cache since build.sh -l use default variables -# force update the variables each time -make rebuild_cache +cd ${CORE_BUILD_DIR} CMAKE_CMD="cmake \ --DBUILD_UNIT_TEST=${BUILD_UNITTEST} \ -DCMAKE_INSTALL_PREFIX=${INSTALL_PREFIX} -DCMAKE_BUILD_TYPE=${BUILD_TYPE} \ -DCMAKE_CUDA_COMPILER=${CUDA_COMPILER} \ --DBUILD_COVERAGE=${BUILD_COVERAGE} \ --DMILVUS_DB_PATH=${DB_PATH} \ --DMILVUS_ENABLE_PROFILING=${PROFILING} \ --DUSE_JFROG_CACHE=${USE_JFROG_CACHE} \ +-DMILVUS_GPU_VERSION=${GPU_VERSION} \ -DCUSTOMIZATION=${CUSTOMIZATION} \ --DFAISS_URL=${CUSTOMIZED_FAISS_URL} \ -.." +-DBUILD_UNIT_TEST=${BUILD_UNITTEST} \ +-DBUILD_COVERAGE=${BUILD_COVERAGE} \ +-DUSE_JFROG_CACHE=${USE_JFROG_CACHE} \ +-DFAISS_ROOT=${FAISS_ROOT} \ +-DFAISS_WITH_MKL=${WITH_MKL} \ +-DArrow_SOURCE=AUTO \ +-DFAISS_SOURCE=AUTO \ +${MILVUS_CORE_DIR}" echo ${CMAKE_CMD} ${CMAKE_CMD} + if [[ ${RUN_CPPLINT} == "ON" ]]; then # cpplint check make lint @@ -135,8 +139,8 @@ if [[ ${RUN_CPPLINT} == "ON" ]]; then # exit 1 # fi # echo "clang-tidy check passed!" -else - # compile and build - make -j8 || exit 1 - make install || exit 1 fi + +# compile and build +make -j8 || exit 1 +make install || exit 1 diff --git a/ci/jenkins/scripts/coverage.sh b/ci/scripts/coverage.sh similarity index 85% rename from ci/jenkins/scripts/coverage.sh rename to ci/scripts/coverage.sh index 07ab210d2f..cece6f0dcc 100755 --- a/ci/jenkins/scripts/coverage.sh +++ b/ci/scripts/coverage.sh @@ -9,18 +9,22 @@ done SCRIPTS_DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )" INSTALL_PREFIX="/opt/milvus" -CMAKE_BUILD_DIR="${SCRIPTS_DIR}/../../../core/cmake_build" +MILVUS_CORE_DIR="${SCRIPTS_DIR}/../../core" +CORE_BUILD_DIR="${MILVUS_CORE_DIR}/cmake_build" MYSQL_USER_NAME=root MYSQL_PASSWORD=123456 MYSQL_HOST='127.0.0.1' MYSQL_PORT='3306' -while getopts "o:u:p:t:h" arg +while getopts "o:b:u:p:t:h" arg do case $arg in o) INSTALL_PREFIX=$OPTARG ;; + b) + CORE_BUILD_DIR=$OPTARG # CORE_BUILD_DIR + ;; u) MYSQL_USER_NAME=$OPTARG ;; @@ -35,13 +39,14 @@ do parameter: -o: milvus install prefix(default: /opt/milvus) +-b: core code build directory -u: mysql account -p: mysql password -t: mysql host -h: help usage: -./coverage.sh -o \${INSTALL_PREFIX} -u \${MYSQL_USER} -p \${MYSQL_PASSWORD} -t \${MYSQL_HOST} [-h] +./coverage.sh -o \${INSTALL_PREFIX} -b \${CORE_BUILD_DIR} -u \${MYSQL_USER} -p \${MYSQL_PASSWORD} -t \${MYSQL_HOST} [-h] " exit 0 ;; @@ -63,12 +68,14 @@ FILE_INFO_OUTPUT="output.info" FILE_INFO_OUTPUT_NEW="output_new.info" DIR_LCOV_OUTPUT="lcov_out" -DIR_GCNO="${CMAKE_BUILD_DIR}" +DIR_GCNO="${CORE_BUILD_DIR}" DIR_UNITTEST="${INSTALL_PREFIX}/unittest" +cd ${SCRIPTS_DIR} + # delete old code coverage info files -rm -rf lcov_out -rm -f FILE_INFO_BASE FILE_INFO_MILVUS FILE_INFO_OUTPUT FILE_INFO_OUTPUT_NEW +rm -rf ${DIR_LCOV_OUTPUT} +rm -f ${FILE_INFO_BASE} ${FILE_INFO_MILVUS} ${FILE_INFO_OUTPUT} ${FILE_INFO_OUTPUT_NEW} MYSQL_DB_NAME=milvus_`date +%s%N` @@ -109,7 +116,7 @@ for test in `ls ${DIR_UNITTEST}`; do if [ $? -ne 0 ]; then echo ${args} echo ${DIR_UNITTEST}/${test} "run failed" - exit -1 + exit 1 fi done @@ -132,12 +139,11 @@ ${LCOV_CMD} -r "${FILE_INFO_OUTPUT}" -o "${FILE_INFO_OUTPUT_NEW}" \ "*/src/server/Server.cpp" \ "*/src/server/DBWrapper.cpp" \ "*/src/server/grpc_impl/GrpcServer.cpp" \ - "*/src/external/easyloggingpp/easylogging++.h" \ - "*/src/external/easyloggingpp/easylogging++.cc" + "*/thirdparty/*" if [ $? -ne 0 ]; then echo "gen ${FILE_INFO_OUTPUT_NEW} failed" - exit -2 + exit 2 fi # gen html report diff --git a/ci/travis/before-install.sh b/ci/travis/before-install.sh new file mode 100755 index 0000000000..70133cfe1d --- /dev/null +++ b/ci/travis/before-install.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +set -ex + +if [[ "${TRAVIS_OS_NAME}" == "linux" ]]; then + export CCACHE_COMPRESS=1 + export CCACHE_COMPRESSLEVEL=5 + export CCACHE_COMPILERCHECK=content + export PATH=/usr/lib/ccache/:$PATH + ccache --show-stats +fi + +set +ex diff --git a/ci/travis/install_dependency.sh b/ci/travis/install_dependency.sh new file mode 100755 index 0000000000..e9efd1f441 --- /dev/null +++ b/ci/travis/install_dependency.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash + +set -e + +wget -P /tmp https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2019.PUB + +sudo apt-key add /tmp/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2019.PUB + +echo "deb https://apt.repos.intel.com/mkl all main" | \ + sudo tee /etc/apt/sources.list.d/intel-mkl.list + +sudo wget -O /usr/share/keyrings/apache-arrow-keyring.gpg https://dl.bintray.com/apache/arrow/$(lsb_release --id --short | tr 'A-Z' 'a-z')/apache-arrow-keyring.gpg + +sudo tee /etc/apt/sources.list.d/apache-arrow.list < /dev/null 2>&1 +make rebuild_cache >/dev/null 2>&1 CMAKE_CMD="cmake \ -DBUILD_UNIT_TEST=${BUILD_UNITTEST} \ -DCMAKE_INSTALL_PREFIX=${INSTALL_PREFIX} -DCMAKE_BUILD_TYPE=${BUILD_TYPE} \ +-DFAISS_ROOT=${FAISS_ROOT} \ +-DFAISS_SOURCE=${FAISS_SOURCE} \ -DCMAKE_CUDA_COMPILER=${CUDA_COMPILER} \ -DBUILD_COVERAGE=${BUILD_COVERAGE} \ -DMILVUS_DB_PATH=${DB_PATH} \ -DMILVUS_ENABLE_PROFILING=${PROFILING} \ -DUSE_JFROG_CACHE=${USE_JFROG_CACHE} \ -DCUSTOMIZATION=${CUSTOMIZATION} \ --DFAISS_URL=${CUSTOMIZED_FAISS_URL} \ +-DMILVUS_GPU_VERSION=${GPU_VERSION} \ +-DFAISS_WITH_MKL=${WITH_MKL} \ +-DMILVUS_WITH_PROMETHEUS=${WITH_PROMETHEUS} \ ../" echo ${CMAKE_CMD} ${CMAKE_CMD} if [[ ${MAKE_CLEAN} == "ON" ]]; then - make clean + make clean fi if [[ ${RUN_CPPLINT} == "ON" ]]; then - # cpplint check - make lint - if [ $? -ne 0 ]; then - echo "ERROR! cpplint check failed" - exit 1 - fi - echo "cpplint check passed!" + # cpplint check + make lint + if [ $? -ne 0 ]; then + echo "ERROR! cpplint check failed" + exit 1 + fi + echo "cpplint check passed!" - # clang-format check - make check-clang-format - if [ $? -ne 0 ]; then - echo "ERROR! clang-format check failed" - exit 1 - fi - echo "clang-format check passed!" + # clang-format check + make check-clang-format + if [ $? -ne 0 ]; then + echo "ERROR! clang-format check failed" + exit 1 + fi + echo "clang-format check passed!" # # clang-tidy check # make check-clang-tidy @@ -140,11 +159,11 @@ if [[ ${RUN_CPPLINT} == "ON" ]]; then # echo "clang-tidy check passed!" else - # strip binary symbol - if [[ ${BUILD_TYPE} != "Debug" ]]; then - strip src/milvus_server - fi + # strip binary symbol + if [[ ${BUILD_TYPE} != "Debug" ]]; then + strip src/milvus_server + fi - # compile and build - make -j 8 install || exit 1 -fi \ No newline at end of file + # compile and build + make -j 8 install || exit 1 +fi diff --git a/core/cmake/DefineOptions.cmake b/core/cmake/DefineOptions.cmake index 167b6e9d66..4f4c8b17b5 100644 --- a/core/cmake/DefineOptions.cmake +++ b/core/cmake/DefineOptions.cmake @@ -13,16 +13,16 @@ macro(define_option name description default) endmacro() function(list_join lst glue out) - if("${${lst}}" STREQUAL "") + if ("${${lst}}" STREQUAL "") set(${out} "" PARENT_SCOPE) return() - endif() + endif () list(GET ${lst} 0 joined) list(REMOVE_AT ${lst} 0) - foreach(item ${${lst}}) + foreach (item ${${lst}}) set(joined "${joined}${glue}${item}") - endforeach() + endforeach () set(${out} ${joined} PARENT_SCOPE) endfunction() @@ -35,22 +35,29 @@ macro(define_option_string name description default) set("${name}_OPTION_ENUM" ${ARGN}) list_join("${name}_OPTION_ENUM" "|" "${name}_OPTION_ENUM") - if(NOT ("${${name}_OPTION_ENUM}" STREQUAL "")) + if (NOT ("${${name}_OPTION_ENUM}" STREQUAL "")) set_property(CACHE ${name} PROPERTY STRINGS ${ARGN}) - endif() + endif () endmacro() +#---------------------------------------------------------------------- +set_option_category("GPU version") + +define_option(MILVUS_GPU_VERSION "Build GPU version" OFF) + #---------------------------------------------------------------------- set_option_category("Thirdparty") -set(MILVUS_DEPENDENCY_SOURCE_DEFAULT "AUTO") +set(MILVUS_DEPENDENCY_SOURCE_DEFAULT "BUNDLED") define_option_string(MILVUS_DEPENDENCY_SOURCE - "Method to use for acquiring MILVUS's build dependencies" - "${MILVUS_DEPENDENCY_SOURCE_DEFAULT}" - "AUTO" - "BUNDLED" - "SYSTEM") + "Method to use for acquiring MILVUS's build dependencies" + "${MILVUS_DEPENDENCY_SOURCE_DEFAULT}" + "AUTO" + "BUNDLED" + "SYSTEM") + +define_option(MILVUS_USE_CCACHE "Use ccache when compiling (if available)" ON) define_option(MILVUS_VERBOSE_THIRDPARTY_BUILD "Show output from ExternalProjects rather than just logging to files" ON) @@ -70,33 +77,21 @@ define_option(MILVUS_WITH_YAMLCPP "Build with yaml-cpp library" ON) if (MILVUS_ENABLE_PROFILING STREQUAL "ON") define_option(MILVUS_WITH_LIBUNWIND "Build with libunwind" ON) define_option(MILVUS_WITH_GPERFTOOLS "Build with gperftools" ON) -endif() +endif () define_option(MILVUS_WITH_GRPC "Build with GRPC" ON) define_option(MILVUS_WITH_ZLIB "Build with zlib compression" ON) -#---------------------------------------------------------------------- -if(MSVC) - set_option_category("MSVC") - - define_option(MSVC_LINK_VERBOSE - "Pass verbose linking options when linking libraries and executables" - OFF) - - define_option(MILVUS_USE_STATIC_CRT "Build MILVUS with statically linked CRT" OFF) -endif() - - #---------------------------------------------------------------------- set_option_category("Test and benchmark") unset(MILVUS_BUILD_TESTS CACHE) if (BUILD_UNIT_TEST) define_option(MILVUS_BUILD_TESTS "Build the MILVUS googletest unit tests" ON) -else() +else () define_option(MILVUS_BUILD_TESTS "Build the MILVUS googletest unit tests" OFF) -endif(BUILD_UNIT_TEST) +endif (BUILD_UNIT_TEST) #---------------------------------------------------------------------- macro(config_summary) @@ -108,12 +103,12 @@ macro(config_summary) message(STATUS " Generator: ${CMAKE_GENERATOR}") message(STATUS " Build type: ${CMAKE_BUILD_TYPE}") message(STATUS " Source directory: ${CMAKE_CURRENT_SOURCE_DIR}") - if(${CMAKE_EXPORT_COMPILE_COMMANDS}) + if (${CMAKE_EXPORT_COMPILE_COMMANDS}) message( STATUS " Compile commands: ${CMAKE_CURRENT_BINARY_DIR}/compile_commands.json") - endif() + endif () - foreach(category ${MILVUS_OPTION_CATEGORIES}) + foreach (category ${MILVUS_OPTION_CATEGORIES}) message(STATUS) message(STATUS "${category} options:") @@ -121,50 +116,50 @@ macro(config_summary) set(option_names ${MILVUS_${category}_OPTION_NAMES}) set(max_value_length 0) - foreach(name ${option_names}) + foreach (name ${option_names}) string(LENGTH "\"${${name}}\"" value_length) - if(${max_value_length} LESS ${value_length}) + if (${max_value_length} LESS ${value_length}) set(max_value_length ${value_length}) - endif() - endforeach() + endif () + endforeach () - foreach(name ${option_names}) - if("${${name}_OPTION_TYPE}" STREQUAL "string") + foreach (name ${option_names}) + if ("${${name}_OPTION_TYPE}" STREQUAL "string") set(value "\"${${name}}\"") - else() + else () set(value "${${name}}") - endif() + endif () set(default ${${name}_OPTION_DEFAULT}) set(description ${${name}_OPTION_DESCRIPTION}) string(LENGTH ${description} description_length) - if(${description_length} LESS 70) + if (${description_length} LESS 70) string( SUBSTRING " " ${description_length} -1 description_padding) - else() + else () set(description_padding " ") - endif() + endif () set(comment "[${name}]") - if("${value}" STREQUAL "${default}") + if ("${value}" STREQUAL "${default}") set(comment "[default] ${comment}") - endif() + endif () - if(NOT ("${${name}_OPTION_ENUM}" STREQUAL "")) + if (NOT ("${${name}_OPTION_ENUM}" STREQUAL "")) set(comment "${comment} [${${name}_OPTION_ENUM}]") - endif() + endif () string( SUBSTRING "${value} " 0 ${max_value_length} value) message(STATUS " ${description} ${description_padding} ${value} ${comment}") - endforeach() + endforeach () - endforeach() + endforeach () endmacro() diff --git a/core/cmake/ThirdPartyPackages.cmake b/core/cmake/ThirdPartyPackages.cmake index 657efa2eef..2092879b17 100644 --- a/core/cmake/ThirdPartyPackages.cmake +++ b/core/cmake/ThirdPartyPackages.cmake @@ -164,8 +164,10 @@ endif () macro(resolve_dependency DEPENDENCY_NAME) if (${DEPENDENCY_NAME}_SOURCE STREQUAL "AUTO") - #disable find_package for now - build_dependency(${DEPENDENCY_NAME}) + find_package(${DEPENDENCY_NAME} MODULE) + if(NOT ${${DEPENDENCY_NAME}_FOUND}) + build_dependency(${DEPENDENCY_NAME}) + endif() elseif (${DEPENDENCY_NAME}_SOURCE STREQUAL "BUNDLED") build_dependency(${DEPENDENCY_NAME}) elseif (${DEPENDENCY_NAME}_SOURCE STREQUAL "SYSTEM") diff --git a/core/conf/server_config.template b/core/conf/server_cpu_config.template similarity index 74% rename from core/conf/server_config.template rename to core/conf/server_cpu_config.template index 4a1432eea6..41889f5cef 100644 --- a/core/conf/server_config.template +++ b/core/conf/server_cpu_config.template @@ -27,10 +27,7 @@ metric_config: port: 8080 # port prometheus uses to fetch metrics, must in range [1025, 65534] cache_config: - cpu_cache_capacity: 16 # GB, CPU memory used for cache, must be a positive integer - cpu_cache_threshold: 0.85 # percentage of data that will be kept when cache cleanup is triggered, must be in range (0.0, 1.0] - gpu_cache_capacity: 4 # GB, GPU memory used for cache, must be a positive integer - gpu_cache_threshold: 0.85 # percentage of data that will be kept when cache cleanup is triggered, must be in range (0.0, 1.0] + cpu_cache_capacity: 16 # GB, size of CPU memory used for cache, must be a positive integer cache_insert_data: false # whether to load inserted data into cache, must be a boolean engine_config: @@ -38,8 +35,10 @@ engine_config: # if nq >= use_blas_threshold, use OpenBlas, slower with stable response times gpu_search_threshold: 1000 # threshold beyond which the search computation is executed on GPUs only -resource_config: - search_resources: # define the devices used for search computation, must be in format: cpu or gpux - - cpu +gpu_resource_config: + enable: false # whether to enable GPU resources + cache_capacity: 4 # GB, size of GPU memory per card used for cache, must be a positive integer + search_resources: # define the GPU devices used for search computation, must be in format gpux + - gpu0 + build_index_resources: # define the GPU devices used for index building, must be in format gpux - gpu0 - index_build_device: gpu0 # GPU used for building index, must be in format: gpux \ No newline at end of file diff --git a/core/conf/server_gpu_config.template b/core/conf/server_gpu_config.template new file mode 100644 index 0000000000..531c633da7 --- /dev/null +++ b/core/conf/server_gpu_config.template @@ -0,0 +1,44 @@ +# Default values are used when you make no changes to the following parameters. + +server_config: + address: 0.0.0.0 # milvus server ip address (IPv4) + port: 19530 # milvus server port, must in range [1025, 65534] + deploy_mode: single # deployment type: single, cluster_readonly, cluster_writable + time_zone: UTC+8 # time zone, must be in format: UTC+X + +db_config: + primary_path: @MILVUS_DB_PATH@ # path used to store data and meta + secondary_path: # path used to store data only, split by semicolon + + backend_url: sqlite://:@:/ # URI format: dialect://username:password@host:port/database + # Keep 'dialect://:@:/', and replace other texts with real values + # Replace 'dialect' with 'mysql' or 'sqlite' + + insert_buffer_size: 4 # GB, maximum insert buffer size allowed, must be a positive integer + # sum of insert_buffer_size and cpu_cache_capacity cannot exceed total memory + + preload_table: # preload data at startup, '*' means load all tables, empty value means no preload + # you can specify preload tables like this: table1,table2,table3 + +metric_config: + enable_monitor: false # enable monitoring or not, must be a boolean + collector: prometheus # prometheus + prometheus_config: + port: 8080 # port prometheus uses to fetch metrics, must in range [1025, 65534] + +cache_config: + cpu_cache_capacity: 16 # GB, size of CPU memory used for cache, must be a positive integer + cache_insert_data: false # whether to load inserted data into cache, must be a boolean + +engine_config: + use_blas_threshold: 1100 # if nq < use_blas_threshold, use SSE, faster with fluctuated response times + # if nq >= use_blas_threshold, use OpenBlas, slower with stable response times + gpu_search_threshold: 1000 # threshold beyond which the search computation is executed on GPUs only + +gpu_resource_config: + enable: true # whether to enable GPU resources + cache_capacity: 4 # GB, size of GPU memory per card used for cache, must be a positive integer + search_resources: # define the GPU devices used for search computation, must be in format gpux + - gpu0 + build_index_resources: # define the GPU devices used for index building, must be in format gpux + - gpu0 diff --git a/core/coverage.sh b/core/coverage.sh index 2cb0861de4..833b1b0050 100755 --- a/core/coverage.sh +++ b/core/coverage.sh @@ -122,9 +122,7 @@ ${LCOV_CMD} -r "${FILE_INFO_OUTPUT}" -o "${FILE_INFO_OUTPUT_NEW}" \ "*/src/server/Server.cpp" \ "*/src/server/DBWrapper.cpp" \ "*/src/server/grpc_impl/GrpcServer.cpp" \ - "*/easylogging++.h" \ - "*/easylogging++.cc" \ - "*/src/external/*" + "*/thirdparty/*" if [ $? -ne 0 ]; then echo "generate ${FILE_INFO_OUTPUT_NEW} failed" diff --git a/core/scripts/migration/README.md b/core/scripts/migration/README.md new file mode 100644 index 0000000000..7c318c1393 --- /dev/null +++ b/core/scripts/migration/README.md @@ -0,0 +1,28 @@ +## Data Migration + +####0.3.x +legacy data is not migrate-able for later versions + +####0.4.x +legacy data can be reused directly by 0.5.x + +legacy data can be migrated to 0.6.x + +####0.5.x +legacy data can be migrated to 0.6.x + +####0.6.x +how to migrate legacy 0.4.x/0.5.x data + +for sqlite meta: +```shell + $ sqlite3 [parth_to]/meta.sqlite < sqlite_4_to_6.sql +``` + +for mysql meta: +```shell + $ mysql -h127.0.0.1 -uroot -p123456 -Dmilvus < mysql_4_to_6.sql +``` + + + diff --git a/core/scripts/migration/mysql_4_to_6.sql b/core/scripts/migration/mysql_4_to_6.sql new file mode 100644 index 0000000000..f8a5b1b70b --- /dev/null +++ b/core/scripts/migration/mysql_4_to_6.sql @@ -0,0 +1,4 @@ +alter table Tables add column owner_table VARCHAR(255) DEFAULT '' NOT NULL; +alter table Tables add column partition_tag VARCHAR(255) DEFAULT '' NOT NULL; +alter table Tables add column version VARCHAR(64) DEFAULT '0.6.0' NOT NULL; +update Tables set version='0.6.0'; diff --git a/core/scripts/migration/mysql_6_to_4.sql b/core/scripts/migration/mysql_6_to_4.sql new file mode 100644 index 0000000000..96c60e0280 --- /dev/null +++ b/core/scripts/migration/mysql_6_to_4.sql @@ -0,0 +1,3 @@ +alter table Tables drop column owner_table; +alter table Tables drop column partition_tag; +alter table Tables drop column version; diff --git a/core/scripts/migration/sqlite_4_to_6.sql b/core/scripts/migration/sqlite_4_to_6.sql new file mode 100644 index 0000000000..2069145046 --- /dev/null +++ b/core/scripts/migration/sqlite_4_to_6.sql @@ -0,0 +1,4 @@ +alter table Tables add column 'owner_table' TEXT DEFAULT '' NOT NULL; +alter table Tables add column 'partition_tag' TEXT DEFAULT '' NOT NULL; +alter table Tables add column 'version' TEXT DEFAULT '0.6.0' NOT NULL; +update Tables set version='0.6.0'; diff --git a/core/scripts/migration/sqlite_6_to_4.sql b/core/scripts/migration/sqlite_6_to_4.sql new file mode 100644 index 0000000000..686d276f46 --- /dev/null +++ b/core/scripts/migration/sqlite_6_to_4.sql @@ -0,0 +1,7 @@ +CREATE TABLE 'TempTables' ( 'id' INTEGER PRIMARY KEY NOT NULL , 'table_id' TEXT UNIQUE NOT NULL , 'state' INTEGER NOT NULL , 'dimension' INTEGER NOT NULL , 'created_on' INTEGER NOT NULL , 'flag' INTEGER DEFAULT 0 NOT NULL , 'index_file_size' INTEGER NOT NULL , 'engine_type' INTEGER NOT NULL , 'nlist' INTEGER NOT NULL , 'metric_type' INTEGER NOT NULL); + +INSERT INTO TempTables SELECT id, table_id, state, dimension, created_on, flag, index_file_size, engine_type, nlist, metric_type FROM Tables; + +DROP TABLE Tables; + +ALTER TABLE TempTables RENAME TO Tables; diff --git a/core/src/CMakeLists.txt b/core/src/CMakeLists.txt index ae3a458987..14a75bb539 100644 --- a/core/src/CMakeLists.txt +++ b/core/src/CMakeLists.txt @@ -19,13 +19,15 @@ include_directories(${MILVUS_SOURCE_DIR}) include_directories(${MILVUS_ENGINE_SRC}) +include_directories(${MILVUS_THIRDPARTY_SRC}) -include_directories(${CUDA_TOOLKIT_ROOT_DIR}/include) include_directories(${MILVUS_ENGINE_SRC}/grpc/gen-status) include_directories(${MILVUS_ENGINE_SRC}/grpc/gen-milvus) -#this statement must put here, since the INDEX_INCLUDE_DIRS is defined in code/CMakeList.txt add_subdirectory(index) +if (FAISS_WITH_MKL) + add_compile_definitions("WITH_MKL") +endif () set(INDEX_INCLUDE_DIRS ${INDEX_INCLUDE_DIRS} PARENT_SCOPE) foreach (dir ${INDEX_INCLUDE_DIRS}) @@ -35,6 +37,7 @@ endforeach () aux_source_directory(${MILVUS_ENGINE_SRC}/cache cache_files) aux_source_directory(${MILVUS_ENGINE_SRC}/config config_files) aux_source_directory(${MILVUS_ENGINE_SRC}/metrics metrics_files) +aux_source_directory(${MILVUS_ENGINE_SRC}/metrics/prometheus metrics_prometheus_files) aux_source_directory(${MILVUS_ENGINE_SRC}/db db_main_files) aux_source_directory(${MILVUS_ENGINE_SRC}/db/engine db_engine_files) aux_source_directory(${MILVUS_ENGINE_SRC}/db/insert db_insert_files) @@ -64,15 +67,21 @@ set(scheduler_files ${scheduler_task_files} ) -aux_source_directory(${MILVUS_ENGINE_SRC}/external/easyloggingpp external_easyloggingpp_files) -aux_source_directory(${MILVUS_ENGINE_SRC}/external/nlohmann external_nlohmann_files) -set(external_files - ${external_easyloggingpp_files} - ${external_nlohmann_files} +aux_source_directory(${MILVUS_THIRDPARTY_SRC}/easyloggingpp thirdparty_easyloggingpp_files) +aux_source_directory(${MILVUS_THIRDPARTY_SRC}/nlohmann thirdparty_nlohmann_files) +set(thirdparty_files + ${thirdparty_easyloggingpp_files} + ${thirdparty_nlohmann_files} ) aux_source_directory(${MILVUS_ENGINE_SRC}/server server_files) -aux_source_directory(${MILVUS_ENGINE_SRC}/server/grpc_impl grpc_server_files) +aux_source_directory(${MILVUS_ENGINE_SRC}/server/grpc_impl/request grpc_request_files) +aux_source_directory(${MILVUS_ENGINE_SRC}/server/grpc_impl grpc_impl_files) +set(grpc_server_files + ${grpc_request_files} + ${grpc_impl_files} + ) + aux_source_directory(${MILVUS_ENGINE_SRC}/utils utils_files) aux_source_directory(${MILVUS_ENGINE_SRC}/wrapper wrapper_files) @@ -84,11 +93,16 @@ set(engine_files ${db_insert_files} ${db_meta_files} ${metrics_files} - ${external_files} + ${thirdparty_files} ${utils_files} ${wrapper_files} ) +if (MILVUS_WITH_PROMETHEUS) + set(engine_files ${engine_files} + ${metrics_prometheus_files}) +endif () + set(client_grpc_lib grpcpp_channelz grpc++ @@ -109,35 +123,50 @@ set(boost_lib libboost_serialization.a ) -set(cuda_lib - ${CUDA_TOOLKIT_ROOT_DIR}/lib64/stubs/libnvidia-ml.so - cudart - cublas - ) - set(third_party_libs sqlite ${client_grpc_lib} yaml-cpp - ${prometheus_lib} - ${cuda_lib} mysqlpp zlib ${boost_lib} ) -if (MILVUS_ENABLE_PROFILING STREQUAL "ON") +if (MILVUS_GPU_VERSION) + include_directories(${CUDA_INCLUDE_DIRS}) + link_directories("${CUDA_TOOLKIT_ROOT_DIR}/lib64") + set(cuda_lib + ${CUDA_TOOLKIT_ROOT_DIR}/lib64/stubs/libnvidia-ml.so + cudart + cublas + ) set(third_party_libs ${third_party_libs} - gperftools - libunwind - ) + ${cuda_lib} + ) + aux_source_directory(${MILVUS_ENGINE_SRC}/wrapper/gpu wrapper_gpu_files) + set(engine_files ${engine_files} + ${wrapper_gpu_files} + ) +endif () + +if (MILVUS_ENABLE_PROFILING) + set(third_party_libs ${third_party_libs} + gperftools + libunwind + ) +endif () + +if (MILVUS_WITH_PROMETHEUS) + set(third_party_libs ${third_party_libs} + ${prometheus_lib} + ) endif () -link_directories("${CUDA_TOOLKIT_ROOT_DIR}/lib64") set(engine_libs pthread libgomp.a libgfortran.a + dl ) if (NOT ${CMAKE_SYSTEM_PROCESSOR} MATCHES "aarch64") @@ -147,26 +176,33 @@ if (NOT ${CMAKE_SYSTEM_PROCESSOR} MATCHES "aarch64") ) endif () -cuda_add_library(milvus_engine STATIC ${engine_files}) +add_library(milvus_engine STATIC ${engine_files}) target_link_libraries(milvus_engine knowhere - ${engine_libs} ${third_party_libs} + ${engine_libs} ) -add_library(metrics STATIC ${metrics_files}) +if (MILVUS_WITH_PROMETHEUS) + add_library(metrics STATIC ${metrics_files} ${metrics_prometheus_files}) +else () + add_library(metrics STATIC ${metrics_files}) +endif () set(metrics_lib yaml-cpp - ${prometheus_lib} ) +if (MILVUS_WITH_PROMETHEUS) + set(metrics_lib ${metrics_lib} + ${prometheus_lib} + ) +endif () + target_link_libraries(metrics ${metrics_lib}) set(server_libs milvus_engine - pthread - dl metrics ) diff --git a/core/src/cache/Cache.inl b/core/src/cache/Cache.inl index 3a60dd288f..9ac7ff21e6 100644 --- a/core/src/cache/Cache.inl +++ b/core/src/cache/Cache.inl @@ -15,24 +15,18 @@ // specific language governing permissions and limitations // under the License. - - - namespace milvus { namespace cache { constexpr double DEFAULT_THRESHHOLD_PERCENT = 0.85; -template +template Cache::Cache(int64_t capacity, uint64_t cache_max_count) - : usage_(0), - capacity_(capacity), - freemem_percent_(DEFAULT_THRESHHOLD_PERCENT), - lru_(cache_max_count) { -// AGENT_LOG_DEBUG << "Construct Cache with capacity " << std::to_string(mem_capacity) + : usage_(0), capacity_(capacity), freemem_percent_(DEFAULT_THRESHHOLD_PERCENT), lru_(cache_max_count) { + // AGENT_LOG_DEBUG << "Construct Cache with capacity " << std::to_string(mem_capacity) } -template +template void Cache::set_capacity(int64_t capacity) { if (capacity > 0) { @@ -41,23 +35,23 @@ Cache::set_capacity(int64_t capacity) { } } -template +template size_t Cache::size() const { std::lock_guard lock(mutex_); return lru_.size(); } -template +template bool -Cache::exists(const std::string &key) { +Cache::exists(const std::string& key) { std::lock_guard lock(mutex_); return lru_.exists(key); } -template +template ItemObj -Cache::get(const std::string &key) { +Cache::get(const std::string& key) { std::lock_guard lock(mutex_); if (!lru_.exists(key)) { return nullptr; @@ -66,60 +60,59 @@ Cache::get(const std::string &key) { return lru_.get(key); } -template +template void -Cache::insert(const std::string &key, const ItemObj &item) { +Cache::insert(const std::string& key, const ItemObj& item) { if (item == nullptr) { return; } -// if(item->size() > capacity_) { -// SERVER_LOG_ERROR << "Item size " << item->size() -// << " is too large to insert into cache, capacity " << capacity_; -// return; -// } + // if(item->size() > capacity_) { + // SERVER_LOG_ERROR << "Item size " << item->size() + // << " is too large to insert into cache, capacity " << capacity_; + // return; + // } - //calculate usage + // calculate usage { std::lock_guard lock(mutex_); - //if key already exist, subtract old item size + // if key already exist, subtract old item size if (lru_.exists(key)) { - const ItemObj &old_item = lru_.get(key); + const ItemObj& old_item = lru_.get(key); usage_ -= old_item->Size(); } - //plus new item size + // plus new item size usage_ += item->Size(); } - //if usage exceed capacity, free some items + // if usage exceed capacity, free some items if (usage_ > capacity_) { - SERVER_LOG_DEBUG << "Current usage " << usage_ - << " exceeds cache capacity " << capacity_ + SERVER_LOG_DEBUG << "Current usage " << usage_ << " exceeds cache capacity " << capacity_ << ", start free memory"; free_memory(); } - //insert new item + // insert new item { std::lock_guard lock(mutex_); lru_.put(key, item); - SERVER_LOG_DEBUG << "Insert " << key << " size:" << item->Size() - << " bytes into cache, usage: " << usage_ << " bytes"; + SERVER_LOG_DEBUG << "Insert " << key << " size:" << item->Size() << " bytes into cache, usage: " << usage_ + << " bytes"; } } -template +template void -Cache::erase(const std::string &key) { +Cache::erase(const std::string& key) { std::lock_guard lock(mutex_); if (!lru_.exists(key)) { return; } - const ItemObj &old_item = lru_.get(key); + const ItemObj& old_item = lru_.get(key); usage_ -= old_item->Size(); SERVER_LOG_DEBUG << "Erase " << key << " size: " << old_item->Size(); @@ -127,7 +120,7 @@ Cache::erase(const std::string &key) { lru_.erase(key); } -template +template void Cache::clear() { std::lock_guard lock(mutex_); @@ -137,15 +130,16 @@ Cache::clear() { } /* free memory space when CACHE occupation exceed its capacity */ -template +template void Cache::free_memory() { - if (usage_ <= capacity_) return; + if (usage_ <= capacity_) + return; int64_t threshhold = capacity_ * freemem_percent_; int64_t delta_size = usage_ - threshhold; if (delta_size <= 0) { - delta_size = 1;//ensure at least one item erased + delta_size = 1; // ensure at least one item erased } std::set key_array; @@ -156,8 +150,8 @@ Cache::free_memory() { auto it = lru_.rbegin(); while (it != lru_.rend() && released_size < delta_size) { - auto &key = it->first; - auto &obj_ptr = it->second; + auto& key = it->first; + auto& obj_ptr = it->second; key_array.emplace(key); released_size += obj_ptr->Size(); @@ -167,14 +161,14 @@ Cache::free_memory() { SERVER_LOG_DEBUG << "to be released memory size: " << released_size; - for (auto &key : key_array) { + for (auto& key : key_array) { erase(key); } print(); } -template +template void Cache::print() { size_t cache_count = 0; @@ -188,7 +182,5 @@ Cache::print() { SERVER_LOG_DEBUG << "[Cache capacity]: " << capacity_ << " bytes"; } -} // namespace cache -} // namespace milvus - - +} // namespace cache +} // namespace milvus diff --git a/core/src/cache/CacheMgr.inl b/core/src/cache/CacheMgr.inl index 23b2f0df74..30d2940d87 100644 --- a/core/src/cache/CacheMgr.inl +++ b/core/src/cache/CacheMgr.inl @@ -15,21 +15,18 @@ // specific language governing permissions and limitations // under the License. - - namespace milvus { namespace cache { -template +template CacheMgr::CacheMgr() { } -template +template CacheMgr::~CacheMgr() { - } -template +template uint64_t CacheMgr::ItemCount() const { if (cache_ == nullptr) { @@ -37,12 +34,12 @@ CacheMgr::ItemCount() const { return 0; } - return (uint64_t) (cache_->size()); + return (uint64_t)(cache_->size()); } -template +template bool -CacheMgr::ItemExists(const std::string &key) { +CacheMgr::ItemExists(const std::string& key) { if (cache_ == nullptr) { SERVER_LOG_ERROR << "Cache doesn't exist"; return false; @@ -51,9 +48,9 @@ CacheMgr::ItemExists(const std::string &key) { return cache_->exists(key); } -template +template ItemObj -CacheMgr::GetItem(const std::string &key) { +CacheMgr::GetItem(const std::string& key) { if (cache_ == nullptr) { SERVER_LOG_ERROR << "Cache doesn't exist"; return nullptr; @@ -62,9 +59,9 @@ CacheMgr::GetItem(const std::string &key) { return cache_->get(key); } -template +template void -CacheMgr::InsertItem(const std::string &key, const ItemObj &data) { +CacheMgr::InsertItem(const std::string& key, const ItemObj& data) { if (cache_ == nullptr) { SERVER_LOG_ERROR << "Cache doesn't exist"; return; @@ -74,9 +71,9 @@ CacheMgr::InsertItem(const std::string &key, const ItemObj &data) { server::Metrics::GetInstance().CacheAccessTotalIncrement(); } -template +template void -CacheMgr::EraseItem(const std::string &key) { +CacheMgr::EraseItem(const std::string& key) { if (cache_ == nullptr) { SERVER_LOG_ERROR << "Cache doesn't exist"; return; @@ -86,7 +83,7 @@ CacheMgr::EraseItem(const std::string &key) { server::Metrics::GetInstance().CacheAccessTotalIncrement(); } -template +template void CacheMgr::PrintInfo() { if (cache_ == nullptr) { @@ -97,7 +94,7 @@ CacheMgr::PrintInfo() { cache_->print(); } -template +template void CacheMgr::ClearCache() { if (cache_ == nullptr) { @@ -108,7 +105,7 @@ CacheMgr::ClearCache() { cache_->clear(); } -template +template int64_t CacheMgr::CacheUsage() const { if (cache_ == nullptr) { @@ -119,7 +116,7 @@ CacheMgr::CacheUsage() const { return cache_->usage(); } -template +template int64_t CacheMgr::CacheCapacity() const { if (cache_ == nullptr) { @@ -130,7 +127,7 @@ CacheMgr::CacheCapacity() const { return cache_->capacity(); } -template +template void CacheMgr::SetCapacity(int64_t capacity) { if (cache_ == nullptr) { @@ -140,6 +137,5 @@ CacheMgr::SetCapacity(int64_t capacity) { cache_->set_capacity(capacity); } -} // namespace cache -} // namespace milvus - +} // namespace cache +} // namespace milvus diff --git a/core/src/cache/GpuCacheMgr.cpp b/core/src/cache/GpuCacheMgr.cpp index d862bc0393..72229527fa 100644 --- a/core/src/cache/GpuCacheMgr.cpp +++ b/core/src/cache/GpuCacheMgr.cpp @@ -37,7 +37,7 @@ GpuCacheMgr::GpuCacheMgr() { Status s; int64_t gpu_cache_cap; - s = config.GetCacheConfigGpuCacheCapacity(gpu_cache_cap); + s = config.GetGpuResourceConfigCacheCapacity(gpu_cache_cap); if (!s.ok()) { SERVER_LOG_ERROR << s.message(); } @@ -45,7 +45,7 @@ GpuCacheMgr::GpuCacheMgr() { cache_ = std::make_shared>(cap, 1UL << 32); float gpu_mem_threshold; - s = config.GetCacheConfigGpuCacheThreshold(gpu_mem_threshold); + s = config.GetGpuResourceConfigCacheThreshold(gpu_mem_threshold); if (!s.ok()) { SERVER_LOG_ERROR << s.message(); } diff --git a/core/src/db/DB.h b/core/src/db/DB.h index 07fe30babd..09bbd4af45 100644 --- a/core/src/db/DB.h +++ b/core/src/db/DB.h @@ -47,44 +47,68 @@ class DB { virtual Status CreateTable(meta::TableSchema& table_schema_) = 0; + virtual Status - DeleteTable(const std::string& table_id, const meta::DatesT& dates) = 0; + DropTable(const std::string& table_id, const meta::DatesT& dates) = 0; + virtual Status DescribeTable(meta::TableSchema& table_schema_) = 0; + virtual Status HasTable(const std::string& table_id, bool& has_or_not_) = 0; + virtual Status AllTables(std::vector& table_schema_array) = 0; + virtual Status GetTableRowCount(const std::string& table_id, uint64_t& row_count) = 0; + virtual Status PreloadTable(const std::string& table_id) = 0; + virtual Status UpdateTableFlag(const std::string& table_id, int64_t flag) = 0; virtual Status - InsertVectors(const std::string& table_id_, uint64_t n, const float* vectors, IDNumbers& vector_ids_) = 0; + CreatePartition(const std::string& table_id, const std::string& partition_name, + const std::string& partition_tag) = 0; virtual Status - Query(const std::string& table_id, uint64_t k, uint64_t nq, uint64_t nprobe, const float* vectors, - ResultIds& result_ids, ResultDistances& result_distances) = 0; + DropPartition(const std::string& partition_name) = 0; virtual Status - Query(const std::string& table_id, uint64_t k, uint64_t nq, uint64_t nprobe, const float* vectors, - const meta::DatesT& dates, ResultIds& result_ids, ResultDistances& result_distances) = 0; + DropPartitionByTag(const std::string& table_id, const std::string& partition_tag) = 0; virtual Status - Query(const std::string& table_id, const std::vector& file_ids, uint64_t k, uint64_t nq, + ShowPartitions(const std::string& table_id, std::vector& partiton_schema_array) = 0; + + virtual Status + InsertVectors(const std::string& table_id, const std::string& partition_tag, uint64_t n, const float* vectors, + IDNumbers& vector_ids_) = 0; + + virtual Status + Query(const std::string& table_id, const std::vector& partition_tags, uint64_t k, uint64_t nq, + uint64_t nprobe, const float* vectors, ResultIds& result_ids, ResultDistances& result_distances) = 0; + + virtual Status + Query(const std::string& table_id, const std::vector& partition_tags, uint64_t k, uint64_t nq, uint64_t nprobe, const float* vectors, const meta::DatesT& dates, ResultIds& result_ids, ResultDistances& result_distances) = 0; + virtual Status + QueryByFileID(const std::string& table_id, const std::vector& file_ids, uint64_t k, uint64_t nq, + uint64_t nprobe, const float* vectors, const meta::DatesT& dates, ResultIds& result_ids, + ResultDistances& result_distances) = 0; + virtual Status Size(uint64_t& result) = 0; virtual Status CreateIndex(const std::string& table_id, const TableIndex& index) = 0; + virtual Status DescribeIndex(const std::string& table_id, TableIndex& index) = 0; + virtual Status DropIndex(const std::string& table_id) = 0; diff --git a/core/src/db/DBImpl.cpp b/core/src/db/DBImpl.cpp index 52cf1e668a..dd230ce0d1 100644 --- a/core/src/db/DBImpl.cpp +++ b/core/src/db/DBImpl.cpp @@ -30,6 +30,7 @@ #include "scheduler/job/DeleteJob.h" #include "scheduler/job/SearchJob.h" #include "utils/Log.h" +#include "utils/StringHelpFunctions.h" #include "utils/TimeRecorder.h" #include @@ -38,6 +39,7 @@ #include #include #include +#include #include namespace milvus { @@ -49,6 +51,17 @@ constexpr uint64_t METRIC_ACTION_INTERVAL = 1; constexpr uint64_t COMPACT_ACTION_INTERVAL = 1; constexpr uint64_t INDEX_ACTION_INTERVAL = 1; +static const Status SHUTDOWN_ERROR = Status(DB_ERROR, "Milsvus server is shutdown!"); + +void +TraverseFiles(const meta::DatePartionedTableFilesSchema& date_files, meta::TableFilesSchema& files_array) { + for (auto& day_files : date_files) { + for (auto& file : day_files.second) { + files_array.push_back(file); + } + } +} + } // namespace DBImpl::DBImpl(const DBOptions& options) @@ -113,7 +126,7 @@ DBImpl::DropAll() { Status DBImpl::CreateTable(meta::TableSchema& table_schema) { if (shutting_down_.load(std::memory_order_acquire)) { - return Status(DB_ERROR, "Milsvus server is shutdown!"); + return SHUTDOWN_ERROR; } meta::TableSchema temp_schema = table_schema; @@ -122,34 +135,18 @@ DBImpl::CreateTable(meta::TableSchema& table_schema) { } Status -DBImpl::DeleteTable(const std::string& table_id, const meta::DatesT& dates) { +DBImpl::DropTable(const std::string& table_id, const meta::DatesT& dates) { if (shutting_down_.load(std::memory_order_acquire)) { - return Status(DB_ERROR, "Milsvus server is shutdown!"); + return SHUTDOWN_ERROR; } - // dates partly delete files of the table but currently we don't support - ENGINE_LOG_DEBUG << "Prepare to delete table " << table_id; - - if (dates.empty()) { - mem_mgr_->EraseMemVector(table_id); // not allow insert - meta_ptr_->DeleteTable(table_id); // soft delete table - - // scheduler will determine when to delete table files - auto nres = scheduler::ResMgrInst::GetInstance()->GetNumOfComputeResource(); - scheduler::DeleteJobPtr job = std::make_shared(table_id, meta_ptr_, nres); - scheduler::JobMgrInst::GetInstance()->Put(job); - job->WaitAndDelete(); - } else { - meta_ptr_->DropPartitionsByDates(table_id, dates); - } - - return Status::OK(); + return DropTableRecursively(table_id, dates); } Status DBImpl::DescribeTable(meta::TableSchema& table_schema) { if (shutting_down_.load(std::memory_order_acquire)) { - return Status(DB_ERROR, "Milsvus server is shutdown!"); + return SHUTDOWN_ERROR; } auto stat = meta_ptr_->DescribeTable(table_schema); @@ -160,7 +157,7 @@ DBImpl::DescribeTable(meta::TableSchema& table_schema) { Status DBImpl::HasTable(const std::string& table_id, bool& has_or_not) { if (shutting_down_.load(std::memory_order_acquire)) { - return Status(DB_ERROR, "Milsvus server is shutdown!"); + return SHUTDOWN_ERROR; } return meta_ptr_->HasTable(table_id, has_or_not); @@ -169,7 +166,7 @@ DBImpl::HasTable(const std::string& table_id, bool& has_or_not) { Status DBImpl::AllTables(std::vector& table_schema_array) { if (shutting_down_.load(std::memory_order_acquire)) { - return Status(DB_ERROR, "Milsvus server is shutdown!"); + return SHUTDOWN_ERROR; } return meta_ptr_->AllTables(table_schema_array); @@ -178,55 +175,60 @@ DBImpl::AllTables(std::vector& table_schema_array) { Status DBImpl::PreloadTable(const std::string& table_id) { if (shutting_down_.load(std::memory_order_acquire)) { - return Status(DB_ERROR, "Milsvus server is shutdown!"); + return SHUTDOWN_ERROR; } - meta::DatePartionedTableFilesSchema files; - + // get all table files from parent table meta::DatesT dates; std::vector ids; - auto status = meta_ptr_->FilesToSearch(table_id, ids, dates, files); + meta::TableFilesSchema files_array; + auto status = GetFilesToSearch(table_id, ids, dates, files_array); if (!status.ok()) { return status; } + // get files from partition tables + std::vector partiton_array; + status = meta_ptr_->ShowPartitions(table_id, partiton_array); + for (auto& schema : partiton_array) { + status = GetFilesToSearch(schema.table_id_, ids, dates, files_array); + } + int64_t size = 0; int64_t cache_total = cache::CpuCacheMgr::GetInstance()->CacheCapacity(); int64_t cache_usage = cache::CpuCacheMgr::GetInstance()->CacheUsage(); int64_t available_size = cache_total - cache_usage; - for (auto& day_files : files) { - for (auto& file : day_files.second) { - ExecutionEnginePtr engine = - EngineFactory::Build(file.dimension_, file.location_, (EngineType)file.engine_type_, - (MetricType)file.metric_type_, file.nlist_); - if (engine == nullptr) { - ENGINE_LOG_ERROR << "Invalid engine type"; - return Status(DB_ERROR, "Invalid engine type"); - } + for (auto& file : files_array) { + ExecutionEnginePtr engine = EngineFactory::Build(file.dimension_, file.location_, (EngineType)file.engine_type_, + (MetricType)file.metric_type_, file.nlist_); + if (engine == nullptr) { + ENGINE_LOG_ERROR << "Invalid engine type"; + return Status(DB_ERROR, "Invalid engine type"); + } - size += engine->PhysicalSize(); - if (size > available_size) { - return Status(SERVER_CACHE_FULL, "Cache is full"); - } else { - try { - // step 1: load index - engine->Load(true); - } catch (std::exception& ex) { - std::string msg = "Pre-load table encounter exception: " + std::string(ex.what()); - ENGINE_LOG_ERROR << msg; - return Status(DB_ERROR, msg); - } + size += engine->PhysicalSize(); + if (size > available_size) { + return Status(SERVER_CACHE_FULL, "Cache is full"); + } else { + try { + // step 1: load index + engine->Load(true); + } catch (std::exception& ex) { + std::string msg = "Pre-load table encounter exception: " + std::string(ex.what()); + ENGINE_LOG_ERROR << msg; + return Status(DB_ERROR, msg); } } } + return Status::OK(); } Status DBImpl::UpdateTableFlag(const std::string& table_id, int64_t flag) { if (shutting_down_.load(std::memory_order_acquire)) { - return Status(DB_ERROR, "Milsvus server is shutdown!"); + return SHUTDOWN_ERROR; } return meta_ptr_->UpdateTableFlag(table_id, flag); @@ -235,34 +237,105 @@ DBImpl::UpdateTableFlag(const std::string& table_id, int64_t flag) { Status DBImpl::GetTableRowCount(const std::string& table_id, uint64_t& row_count) { if (shutting_down_.load(std::memory_order_acquire)) { - return Status(DB_ERROR, "Milsvus server is shutdown!"); + return SHUTDOWN_ERROR; } - return meta_ptr_->Count(table_id, row_count); + return GetTableRowCountRecursively(table_id, row_count); } Status -DBImpl::InsertVectors(const std::string& table_id, uint64_t n, const float* vectors, IDNumbers& vector_ids) { - // ENGINE_LOG_DEBUG << "Insert " << n << " vectors to cache"; +DBImpl::CreatePartition(const std::string& table_id, const std::string& partition_name, + const std::string& partition_tag) { if (shutting_down_.load(std::memory_order_acquire)) { - return Status(DB_ERROR, "Milsvus server is shutdown!"); + return SHUTDOWN_ERROR; } + return meta_ptr_->CreatePartition(table_id, partition_name, partition_tag); +} + +Status +DBImpl::DropPartition(const std::string& partition_name) { + if (shutting_down_.load(std::memory_order_acquire)) { + return SHUTDOWN_ERROR; + } + + auto status = mem_mgr_->EraseMemVector(partition_name); // not allow insert + status = meta_ptr_->DropPartition(partition_name); // soft delete table + + // scheduler will determine when to delete table files + auto nres = scheduler::ResMgrInst::GetInstance()->GetNumOfComputeResource(); + scheduler::DeleteJobPtr job = std::make_shared(partition_name, meta_ptr_, nres); + scheduler::JobMgrInst::GetInstance()->Put(job); + job->WaitAndDelete(); + + return Status::OK(); +} + +Status +DBImpl::DropPartitionByTag(const std::string& table_id, const std::string& partition_tag) { + if (shutting_down_.load(std::memory_order_acquire)) { + return SHUTDOWN_ERROR; + } + + std::string partition_name; + auto status = meta_ptr_->GetPartitionName(table_id, partition_tag, partition_name); + if (!status.ok()) { + ENGINE_LOG_ERROR << status.message(); + return status; + } + + return DropPartition(partition_name); +} + +Status +DBImpl::ShowPartitions(const std::string& table_id, std::vector& partiton_schema_array) { + if (shutting_down_.load(std::memory_order_acquire)) { + return SHUTDOWN_ERROR; + } + + return meta_ptr_->ShowPartitions(table_id, partiton_schema_array); +} + +Status +DBImpl::InsertVectors(const std::string& table_id, const std::string& partition_tag, uint64_t n, const float* vectors, + IDNumbers& vector_ids) { + // ENGINE_LOG_DEBUG << "Insert " << n << " vectors to cache"; + if (shutting_down_.load(std::memory_order_acquire)) { + return SHUTDOWN_ERROR; + } + + // if partition is specified, use partition as target table Status status; + std::string target_table_name = table_id; + if (!partition_tag.empty()) { + std::string partition_name; + status = meta_ptr_->GetPartitionName(table_id, partition_tag, target_table_name); + if (!status.ok()) { + ENGINE_LOG_ERROR << status.message(); + return status; + } + } + + // insert vectors into target table milvus::server::CollectInsertMetrics metrics(n, status); - status = mem_mgr_->InsertVectors(table_id, n, vectors, vector_ids); + status = mem_mgr_->InsertVectors(target_table_name, n, vectors, vector_ids); return status; } Status DBImpl::CreateIndex(const std::string& table_id, const TableIndex& index) { + if (shutting_down_.load(std::memory_order_acquire)) { + return SHUTDOWN_ERROR; + } + + Status status; { std::unique_lock lock(build_index_mutex_); // step 1: check index difference TableIndex old_index; - auto status = DescribeIndex(table_id, old_index); + status = DescribeIndex(table_id, old_index); if (!status.ok()) { ENGINE_LOG_ERROR << "Failed to get table index info for table: " << table_id; return status; @@ -272,11 +345,8 @@ DBImpl::CreateIndex(const std::string& table_id, const TableIndex& index) { TableIndex new_index = index; new_index.metric_type_ = old_index.metric_type_; // dont change metric type, it was defined by CreateTable if (!utils::IsSameIndex(old_index, new_index)) { - DropIndex(table_id); - - status = meta_ptr_->UpdateTableIndex(table_id, new_index); + status = UpdateTableIndexRecursively(table_id, new_index); if (!status.ok()) { - ENGINE_LOG_ERROR << "Failed to update table index info for table: " << table_id; return status; } } @@ -287,102 +357,91 @@ DBImpl::CreateIndex(const std::string& table_id, const TableIndex& index) { WaitMergeFileFinish(); // step 4: wait and build index - // for IDMAP type, only wait all NEW file converted to RAW file - // for other type, wait NEW/RAW/NEW_MERGE/NEW_INDEX/TO_INDEX files converted to INDEX files - std::vector file_types; - if (index.engine_type_ == static_cast(EngineType::FAISS_IDMAP)) { - file_types = { - static_cast(meta::TableFileSchema::NEW), - static_cast(meta::TableFileSchema::NEW_MERGE), - }; - } else { - file_types = { - static_cast(meta::TableFileSchema::RAW), - static_cast(meta::TableFileSchema::NEW), - static_cast(meta::TableFileSchema::NEW_MERGE), - static_cast(meta::TableFileSchema::NEW_INDEX), - static_cast(meta::TableFileSchema::TO_INDEX), - }; - } + status = BuildTableIndexRecursively(table_id, index); - std::vector file_ids; - auto status = meta_ptr_->FilesByType(table_id, file_types, file_ids); - int times = 1; - - while (!file_ids.empty()) { - ENGINE_LOG_DEBUG << "Non index files detected! Will build index " << times; - if (index.engine_type_ != (int)EngineType::FAISS_IDMAP) { - status = meta_ptr_->UpdateTableFilesToIndex(table_id); - } - - std::this_thread::sleep_for(std::chrono::milliseconds(std::min(10 * 1000, times * 100))); - status = meta_ptr_->FilesByType(table_id, file_types, file_ids); - times++; - } - - return Status::OK(); + return status; } Status DBImpl::DescribeIndex(const std::string& table_id, TableIndex& index) { + if (shutting_down_.load(std::memory_order_acquire)) { + return SHUTDOWN_ERROR; + } + return meta_ptr_->DescribeTableIndex(table_id, index); } Status DBImpl::DropIndex(const std::string& table_id) { + if (shutting_down_.load(std::memory_order_acquire)) { + return SHUTDOWN_ERROR; + } + ENGINE_LOG_DEBUG << "Drop index for table: " << table_id; - return meta_ptr_->DropTableIndex(table_id); + return DropTableIndexRecursively(table_id); } Status -DBImpl::Query(const std::string& table_id, uint64_t k, uint64_t nq, uint64_t nprobe, const float* vectors, - ResultIds& result_ids, ResultDistances& result_distances) { +DBImpl::Query(const std::string& table_id, const std::vector& partition_tags, uint64_t k, uint64_t nq, + uint64_t nprobe, const float* vectors, ResultIds& result_ids, ResultDistances& result_distances) { if (shutting_down_.load(std::memory_order_acquire)) { - return Status(DB_ERROR, "Milsvus server is shutdown!"); + return SHUTDOWN_ERROR; } meta::DatesT dates = {utils::GetDate()}; - Status result = Query(table_id, k, nq, nprobe, vectors, dates, result_ids, result_distances); - + Status result = Query(table_id, partition_tags, k, nq, nprobe, vectors, dates, result_ids, result_distances); return result; } Status -DBImpl::Query(const std::string& table_id, uint64_t k, uint64_t nq, uint64_t nprobe, const float* vectors, - const meta::DatesT& dates, ResultIds& result_ids, ResultDistances& result_distances) { +DBImpl::Query(const std::string& table_id, const std::vector& partition_tags, uint64_t k, uint64_t nq, + uint64_t nprobe, const float* vectors, const meta::DatesT& dates, ResultIds& result_ids, + ResultDistances& result_distances) { if (shutting_down_.load(std::memory_order_acquire)) { - return Status(DB_ERROR, "Milsvus server is shutdown!"); + return SHUTDOWN_ERROR; } ENGINE_LOG_DEBUG << "Query by dates for table: " << table_id << " date range count: " << dates.size(); - // get all table files from table - meta::DatePartionedTableFilesSchema files; + Status status; std::vector ids; - auto status = meta_ptr_->FilesToSearch(table_id, ids, dates, files); - if (!status.ok()) { - return status; - } + meta::TableFilesSchema files_array; - meta::TableFilesSchema file_id_array; - for (auto& day_files : files) { - for (auto& file : day_files.second) { - file_id_array.push_back(file); + if (partition_tags.empty()) { + // no partition tag specified, means search in whole table + // get all table files from parent table + status = GetFilesToSearch(table_id, ids, dates, files_array); + if (!status.ok()) { + return status; + } + + std::vector partiton_array; + status = meta_ptr_->ShowPartitions(table_id, partiton_array); + for (auto& schema : partiton_array) { + status = GetFilesToSearch(schema.table_id_, ids, dates, files_array); + } + } else { + // get files from specified partitions + std::set partition_name_array; + GetPartitionsByTags(table_id, partition_tags, partition_name_array); + + for (auto& partition_name : partition_name_array) { + status = GetFilesToSearch(partition_name, ids, dates, files_array); } } cache::CpuCacheMgr::GetInstance()->PrintInfo(); // print cache info before query - status = QueryAsync(table_id, file_id_array, k, nq, nprobe, vectors, result_ids, result_distances); + status = QueryAsync(table_id, files_array, k, nq, nprobe, vectors, result_ids, result_distances); cache::CpuCacheMgr::GetInstance()->PrintInfo(); // print cache info after query return status; } Status -DBImpl::Query(const std::string& table_id, const std::vector& file_ids, uint64_t k, uint64_t nq, - uint64_t nprobe, const float* vectors, const meta::DatesT& dates, ResultIds& result_ids, - ResultDistances& result_distances) { +DBImpl::QueryByFileID(const std::string& table_id, const std::vector& file_ids, uint64_t k, uint64_t nq, + uint64_t nprobe, const float* vectors, const meta::DatesT& dates, ResultIds& result_ids, + ResultDistances& result_distances) { if (shutting_down_.load(std::memory_order_acquire)) { - return Status(DB_ERROR, "Milsvus server is shutdown!"); + return SHUTDOWN_ERROR; } ENGINE_LOG_DEBUG << "Query by file ids for table: " << table_id << " date range count: " << dates.size(); @@ -396,25 +455,18 @@ DBImpl::Query(const std::string& table_id, const std::vector& file_ ids.push_back(std::stoul(id, &sz)); } - meta::DatePartionedTableFilesSchema files_array; - auto status = meta_ptr_->FilesToSearch(table_id, ids, dates, files_array); + meta::TableFilesSchema files_array; + auto status = GetFilesToSearch(table_id, ids, dates, files_array); if (!status.ok()) { return status; } - meta::TableFilesSchema file_id_array; - for (auto& day_files : files_array) { - for (auto& file : day_files.second) { - file_id_array.push_back(file); - } - } - - if (file_id_array.empty()) { + if (files_array.empty()) { return Status(DB_ERROR, "Invalid file id"); } cache::CpuCacheMgr::GetInstance()->PrintInfo(); // print cache info before query - status = QueryAsync(table_id, file_id_array, k, nq, nprobe, vectors, result_ids, result_distances); + status = QueryAsync(table_id, files_array, k, nq, nprobe, vectors, result_ids, result_distances); cache::CpuCacheMgr::GetInstance()->PrintInfo(); // print cache info after query return status; } @@ -422,7 +474,7 @@ DBImpl::Query(const std::string& table_id, const std::vector& file_ Status DBImpl::Size(uint64_t& result) { if (shutting_down_.load(std::memory_order_acquire)) { - return Status(DB_ERROR, "Milsvus server is shutdown!"); + return SHUTDOWN_ERROR; } return meta_ptr_->Size(result); @@ -577,6 +629,18 @@ DBImpl::StartCompactionTask() { { std::lock_guard lck(compact_result_mutex_); if (compact_thread_results_.empty()) { + // collect merge files for all tables(if compact_table_ids_ is empty) for two reasons: + // 1. other tables may still has un-merged files + // 2. server may be closed unexpected, these un-merge files need to be merged when server restart + if (compact_table_ids_.empty()) { + std::vector table_schema_array; + meta_ptr_->AllTables(table_schema_array); + for (auto& schema : table_schema_array) { + compact_table_ids_.insert(schema.table_id_); + } + } + + // start merge file thread compact_thread_results_.push_back( compact_thread_pool_.enqueue(&DBImpl::BackgroundCompaction, this, compact_table_ids_)); compact_table_ids_.clear(); @@ -675,7 +739,7 @@ DBImpl::BackgroundMergeFiles(const std::string& table_id) { for (auto& kv : raw_files) { auto files = kv.second; if (files.size() < options_.merge_trigger_number_) { - ENGINE_LOG_DEBUG << "Files number not greater equal than merge trigger number, skip merge action"; + ENGINE_LOG_TRACE << "Files number not greater equal than merge trigger number, skip merge action"; continue; } @@ -774,5 +838,186 @@ DBImpl::BackgroundBuildIndex() { // ENGINE_LOG_TRACE << "Background build index thread exit"; } +Status +DBImpl::GetFilesToSearch(const std::string& table_id, const std::vector& file_ids, const meta::DatesT& dates, + meta::TableFilesSchema& files) { + meta::DatePartionedTableFilesSchema date_files; + auto status = meta_ptr_->FilesToSearch(table_id, file_ids, dates, date_files); + if (!status.ok()) { + return status; + } + + TraverseFiles(date_files, files); + return Status::OK(); +} + +Status +DBImpl::GetPartitionsByTags(const std::string& table_id, const std::vector& partition_tags, + std::set& partition_name_array) { + std::vector partiton_array; + auto status = meta_ptr_->ShowPartitions(table_id, partiton_array); + + for (auto& tag : partition_tags) { + // trim side-blank of tag, only compare valid characters + // for example: " ab cd " is treated as "ab cd" + std::string valid_tag = tag; + server::StringHelpFunctions::TrimStringBlank(valid_tag); + for (auto& schema : partiton_array) { + if (server::StringHelpFunctions::IsRegexMatch(schema.partition_tag_, valid_tag)) { + partition_name_array.insert(schema.table_id_); + } + } + } + + return Status::OK(); +} + +Status +DBImpl::DropTableRecursively(const std::string& table_id, const meta::DatesT& dates) { + // dates partly delete files of the table but currently we don't support + ENGINE_LOG_DEBUG << "Prepare to delete table " << table_id; + + Status status; + if (dates.empty()) { + status = mem_mgr_->EraseMemVector(table_id); // not allow insert + status = meta_ptr_->DropTable(table_id); // soft delete table + + // scheduler will determine when to delete table files + auto nres = scheduler::ResMgrInst::GetInstance()->GetNumOfComputeResource(); + scheduler::DeleteJobPtr job = std::make_shared(table_id, meta_ptr_, nres); + scheduler::JobMgrInst::GetInstance()->Put(job); + job->WaitAndDelete(); + } else { + status = meta_ptr_->DropDataByDate(table_id, dates); + } + + std::vector partiton_array; + status = meta_ptr_->ShowPartitions(table_id, partiton_array); + for (auto& schema : partiton_array) { + status = DropTableRecursively(schema.table_id_, dates); + if (!status.ok()) { + return status; + } + } + + return Status::OK(); +} + +Status +DBImpl::UpdateTableIndexRecursively(const std::string& table_id, const TableIndex& index) { + DropIndex(table_id); + + auto status = meta_ptr_->UpdateTableIndex(table_id, index); + if (!status.ok()) { + ENGINE_LOG_ERROR << "Failed to update table index info for table: " << table_id; + return status; + } + + std::vector partiton_array; + status = meta_ptr_->ShowPartitions(table_id, partiton_array); + for (auto& schema : partiton_array) { + status = UpdateTableIndexRecursively(schema.table_id_, index); + if (!status.ok()) { + return status; + } + } + + return Status::OK(); +} + +Status +DBImpl::BuildTableIndexRecursively(const std::string& table_id, const TableIndex& index) { + // for IDMAP type, only wait all NEW file converted to RAW file + // for other type, wait NEW/RAW/NEW_MERGE/NEW_INDEX/TO_INDEX files converted to INDEX files + std::vector file_types; + if (index.engine_type_ == static_cast(EngineType::FAISS_IDMAP)) { + file_types = { + static_cast(meta::TableFileSchema::NEW), + static_cast(meta::TableFileSchema::NEW_MERGE), + }; + } else { + file_types = { + static_cast(meta::TableFileSchema::RAW), + static_cast(meta::TableFileSchema::NEW), + static_cast(meta::TableFileSchema::NEW_MERGE), + static_cast(meta::TableFileSchema::NEW_INDEX), + static_cast(meta::TableFileSchema::TO_INDEX), + }; + } + + // get files to build index + std::vector file_ids; + auto status = meta_ptr_->FilesByType(table_id, file_types, file_ids); + int times = 1; + + while (!file_ids.empty()) { + ENGINE_LOG_DEBUG << "Non index files detected! Will build index " << times; + if (index.engine_type_ != (int)EngineType::FAISS_IDMAP) { + status = meta_ptr_->UpdateTableFilesToIndex(table_id); + } + + std::this_thread::sleep_for(std::chrono::milliseconds(std::min(10 * 1000, times * 100))); + status = meta_ptr_->FilesByType(table_id, file_types, file_ids); + times++; + } + + // build index for partition + std::vector partiton_array; + status = meta_ptr_->ShowPartitions(table_id, partiton_array); + for (auto& schema : partiton_array) { + status = BuildTableIndexRecursively(schema.table_id_, index); + if (!status.ok()) { + return status; + } + } + + return Status::OK(); +} + +Status +DBImpl::DropTableIndexRecursively(const std::string& table_id) { + ENGINE_LOG_DEBUG << "Drop index for table: " << table_id; + auto status = meta_ptr_->DropTableIndex(table_id); + if (!status.ok()) { + return status; + } + + // drop partition index + std::vector partiton_array; + status = meta_ptr_->ShowPartitions(table_id, partiton_array); + for (auto& schema : partiton_array) { + status = DropTableIndexRecursively(schema.table_id_); + if (!status.ok()) { + return status; + } + } + + return Status::OK(); +} + +Status +DBImpl::GetTableRowCountRecursively(const std::string& table_id, uint64_t& row_count) { + row_count = 0; + auto status = meta_ptr_->Count(table_id, row_count); + if (!status.ok()) { + return status; + } + + // get partition row count + std::vector partiton_array; + status = meta_ptr_->ShowPartitions(table_id, partiton_array); + for (auto& schema : partiton_array) { + uint64_t partition_row_count = 0; + status = GetTableRowCountRecursively(schema.table_id_, partition_row_count); + if (!status.ok()) { + return status; + } + + row_count += partition_row_count; + } + + return Status::OK(); +} + } // namespace engine } // namespace milvus diff --git a/core/src/db/DBImpl.h b/core/src/db/DBImpl.h index ad9c574bb1..a0c5cc356d 100644 --- a/core/src/db/DBImpl.h +++ b/core/src/db/DBImpl.h @@ -57,7 +57,7 @@ class DBImpl : public DB { CreateTable(meta::TableSchema& table_schema) override; Status - DeleteTable(const std::string& table_id, const meta::DatesT& dates) override; + DropTable(const std::string& table_id, const meta::DatesT& dates) override; Status DescribeTable(meta::TableSchema& table_schema) override; @@ -78,7 +78,21 @@ class DBImpl : public DB { GetTableRowCount(const std::string& table_id, uint64_t& row_count) override; Status - InsertVectors(const std::string& table_id, uint64_t n, const float* vectors, IDNumbers& vector_ids) override; + CreatePartition(const std::string& table_id, const std::string& partition_name, + const std::string& partition_tag) override; + + Status + DropPartition(const std::string& partition_name) override; + + Status + DropPartitionByTag(const std::string& table_id, const std::string& partition_tag) override; + + Status + ShowPartitions(const std::string& table_id, std::vector& partiton_schema_array) override; + + Status + InsertVectors(const std::string& table_id, const std::string& partition_tag, uint64_t n, const float* vectors, + IDNumbers& vector_ids) override; Status CreateIndex(const std::string& table_id, const TableIndex& index) override; @@ -90,18 +104,19 @@ class DBImpl : public DB { DropIndex(const std::string& table_id) override; Status - Query(const std::string& table_id, uint64_t k, uint64_t nq, uint64_t nprobe, const float* vectors, - ResultIds& result_ids, ResultDistances& result_distances) override; + Query(const std::string& table_id, const std::vector& partition_tags, uint64_t k, uint64_t nq, + uint64_t nprobe, const float* vectors, ResultIds& result_ids, ResultDistances& result_distances) override; Status - Query(const std::string& table_id, uint64_t k, uint64_t nq, uint64_t nprobe, const float* vectors, - const meta::DatesT& dates, ResultIds& result_ids, ResultDistances& result_distances) override; - - Status - Query(const std::string& table_id, const std::vector& file_ids, uint64_t k, uint64_t nq, + Query(const std::string& table_id, const std::vector& partition_tags, uint64_t k, uint64_t nq, uint64_t nprobe, const float* vectors, const meta::DatesT& dates, ResultIds& result_ids, ResultDistances& result_distances) override; + Status + QueryByFileID(const std::string& table_id, const std::vector& file_ids, uint64_t k, uint64_t nq, + uint64_t nprobe, const float* vectors, const meta::DatesT& dates, ResultIds& result_ids, + ResultDistances& result_distances) override; + Status Size(uint64_t& result) override; @@ -137,6 +152,29 @@ class DBImpl : public DB { Status MemSerialize(); + Status + GetFilesToSearch(const std::string& table_id, const std::vector& file_ids, const meta::DatesT& dates, + meta::TableFilesSchema& files); + + Status + GetPartitionsByTags(const std::string& table_id, const std::vector& partition_tags, + std::set& partition_name_array); + + Status + DropTableRecursively(const std::string& table_id, const meta::DatesT& dates); + + Status + UpdateTableIndexRecursively(const std::string& table_id, const TableIndex& index); + + Status + BuildTableIndexRecursively(const std::string& table_id, const TableIndex& index); + + Status + DropTableIndexRecursively(const std::string& table_id); + + Status + GetTableRowCountRecursively(const std::string& table_id, uint64_t& row_count); + private: const DBOptions options_; diff --git a/core/src/db/Options.h b/core/src/db/Options.h index ebecb4de5a..91459a8d5f 100644 --- a/core/src/db/Options.h +++ b/core/src/db/Options.h @@ -33,7 +33,7 @@ static const char* ARCHIVE_CONF_DISK = "disk"; static const char* ARCHIVE_CONF_DAYS = "days"; struct ArchiveConf { - using CriteriaT = std::map; + using CriteriaT = std::map; explicit ArchiveConf(const std::string& type, const std::string& criterias = std::string()); diff --git a/core/src/db/Types.h b/core/src/db/Types.h index cc2eab0383..76c06126f8 100644 --- a/core/src/db/Types.h +++ b/core/src/db/Types.h @@ -27,8 +27,7 @@ namespace milvus { namespace engine { -using IDNumber = faiss::Index::idx_t; - +typedef int64_t IDNumber; typedef IDNumber* IDNumberPtr; typedef std::vector IDNumbers; diff --git a/core/src/db/engine/ExecutionEngine.h b/core/src/db/engine/ExecutionEngine.h index 51c77eb78e..c8784e8a90 100644 --- a/core/src/db/engine/ExecutionEngine.h +++ b/core/src/db/engine/ExecutionEngine.h @@ -26,6 +26,7 @@ namespace milvus { namespace engine { +// TODO(linxj): replace with VecIndex::IndexType enum class EngineType { INVALID = 0, FAISS_IDMAP = 1, @@ -33,7 +34,10 @@ enum class EngineType { FAISS_IVFSQ8, NSG_MIX, FAISS_IVFSQ8H, - MAX_VALUE = FAISS_IVFSQ8H, + FAISS_PQ, + SPTAG_KDT, + SPTAG_BKT, + MAX_VALUE = SPTAG_BKT, }; enum class MetricType { diff --git a/core/src/db/engine/ExecutionEngineImpl.cpp b/core/src/db/engine/ExecutionEngineImpl.cpp index ee04191fef..ca307b90fc 100644 --- a/core/src/db/engine/ExecutionEngineImpl.cpp +++ b/core/src/db/engine/ExecutionEngineImpl.cpp @@ -25,6 +25,7 @@ #include "utils/CommonUtil.h" #include "utils/Exception.h" #include "utils/Log.h" + #include "wrapper/ConfAdapter.h" #include "wrapper/ConfAdapterMgr.h" #include "wrapper/VecImpl.h" @@ -92,11 +93,19 @@ ExecutionEngineImpl::CreatetVecIndex(EngineType type) { break; } case EngineType::FAISS_IVFFLAT: { +#ifdef MILVUS_CPU_VERSION + index = GetVecIndexFactory(IndexType::FAISS_IVFFLAT_CPU); +#else index = GetVecIndexFactory(IndexType::FAISS_IVFFLAT_MIX); +#endif break; } case EngineType::FAISS_IVFSQ8: { +#ifdef MILVUS_CPU_VERSION + index = GetVecIndexFactory(IndexType::FAISS_IVFSQ8_CPU); +#else index = GetVecIndexFactory(IndexType::FAISS_IVFSQ8_MIX); +#endif break; } case EngineType::NSG_MIX: { @@ -107,6 +116,22 @@ ExecutionEngineImpl::CreatetVecIndex(EngineType type) { index = GetVecIndexFactory(IndexType::FAISS_IVFSQ8_HYBRID); break; } + case EngineType::FAISS_PQ: { +#ifdef MILVUS_CPU_VERSION + index = GetVecIndexFactory(IndexType::FAISS_IVFPQ_CPU); +#else + index = GetVecIndexFactory(IndexType::FAISS_IVFPQ_MIX); +#endif + break; + } + case EngineType::SPTAG_KDT: { + index = GetVecIndexFactory(IndexType::SPTAG_KDT_RNT_CPU); + break; + } + case EngineType::SPTAG_BKT: { + index = GetVecIndexFactory(IndexType::SPTAG_BKT_RNT_CPU); + break; + } default: { ENGINE_LOG_ERROR << "Unsupported index type"; return nullptr; @@ -127,7 +152,14 @@ ExecutionEngineImpl::HybridLoad() const { } const std::string key = location_ + ".quantizer"; - std::vector gpus = scheduler::get_gpu_pool(); + + server::Config& config = server::Config::GetInstance(); + std::vector gpus; + Status s = config.GetGpuResourceConfigSearchResources(gpus); + if (!s.ok()) { + ENGINE_LOG_ERROR << s.message(); + return; + } // cache hit { @@ -309,18 +341,36 @@ ExecutionEngineImpl::CopyToGpu(uint64_t device_id, bool hybrid) { return Status::OK(); } #endif - try { - index_ = index_->CopyToGpu(device_id); - ENGINE_LOG_DEBUG << "CPU to GPU" << device_id; - } catch (std::exception& e) { - ENGINE_LOG_ERROR << e.what(); - return Status(DB_ERROR, e.what()); + + auto index = std::static_pointer_cast(cache::GpuCacheMgr::GetInstance(device_id)->GetIndex(location_)); + bool already_in_cache = (index != nullptr); + if (already_in_cache) { + index_ = index; + } else { + if (index_ == nullptr) { + ENGINE_LOG_ERROR << "ExecutionEngineImpl: index is null, failed to copy to gpu"; + return Status(DB_ERROR, "index is null"); + } + + try { + index_ = index_->CopyToGpu(device_id); + ENGINE_LOG_DEBUG << "CPU to GPU" << device_id; + } catch (std::exception& e) { + ENGINE_LOG_ERROR << e.what(); + return Status(DB_ERROR, e.what()); + } } + + if (!already_in_cache) { + GpuCache(device_id); + } + return Status::OK(); } Status ExecutionEngineImpl::CopyToIndexFileToGpu(uint64_t device_id) { + gpu_num_ = device_id; auto to_index_data = std::make_shared(PhysicalSize()); cache::DataObjPtr obj = std::static_pointer_cast(to_index_data); milvus::cache::GpuCacheMgr::GetInstance(device_id)->InsertItem(location_, obj); @@ -544,12 +594,16 @@ ExecutionEngineImpl::GpuCache(uint64_t gpu_id) { Status ExecutionEngineImpl::Init() { server::Config& config = server::Config::GetInstance(); - Status s = config.GetResourceConfigIndexBuildDevice(gpu_num_); - if (!s.ok()) { - return s; + std::vector gpu_ids; + Status s = config.GetGpuResourceConfigBuildIndexResources(gpu_ids); + for (auto id : gpu_ids) { + if (gpu_num_ == id) { + return Status::OK(); + } } - return Status::OK(); + std::string msg = "Invalid gpu_num"; + return Status(SERVER_INVALID_ARGUMENT, msg); } } // namespace engine diff --git a/core/src/db/engine/ExecutionEngineImpl.h b/core/src/db/engine/ExecutionEngineImpl.h index 7eb304426e..da0e7cfb64 100644 --- a/core/src/db/engine/ExecutionEngineImpl.h +++ b/core/src/db/engine/ExecutionEngineImpl.h @@ -122,8 +122,8 @@ class ExecutionEngineImpl : public ExecutionEngine { int64_t dim_; std::string location_; - int32_t nlist_ = 0; - int32_t gpu_num_ = 0; + int64_t nlist_ = 0; + int64_t gpu_num_ = 0; }; } // namespace engine diff --git a/core/src/db/meta/Meta.h b/core/src/db/meta/Meta.h index ec4b66916d..f538bebce6 100644 --- a/core/src/db/meta/Meta.h +++ b/core/src/db/meta/Meta.h @@ -50,14 +50,11 @@ class Meta { virtual Status AllTables(std::vector& table_schema_array) = 0; - virtual Status - UpdateTableIndex(const std::string& table_id, const TableIndex& index) = 0; - virtual Status UpdateTableFlag(const std::string& table_id, int64_t flag) = 0; virtual Status - DeleteTable(const std::string& table_id) = 0; + DropTable(const std::string& table_id) = 0; virtual Status DeleteTableFiles(const std::string& table_id) = 0; @@ -66,20 +63,41 @@ class Meta { CreateTableFile(TableFileSchema& file_schema) = 0; virtual Status - DropPartitionsByDates(const std::string& table_id, const DatesT& dates) = 0; + DropDataByDate(const std::string& table_id, const DatesT& dates) = 0; virtual Status GetTableFiles(const std::string& table_id, const std::vector& ids, TableFilesSchema& table_files) = 0; - virtual Status - UpdateTableFilesToIndex(const std::string& table_id) = 0; - virtual Status UpdateTableFile(TableFileSchema& file_schema) = 0; virtual Status UpdateTableFiles(TableFilesSchema& files) = 0; + virtual Status + UpdateTableIndex(const std::string& table_id, const TableIndex& index) = 0; + + virtual Status + UpdateTableFilesToIndex(const std::string& table_id) = 0; + + virtual Status + DescribeTableIndex(const std::string& table_id, TableIndex& index) = 0; + + virtual Status + DropTableIndex(const std::string& table_id) = 0; + + virtual Status + CreatePartition(const std::string& table_name, const std::string& partition_name, const std::string& tag) = 0; + + virtual Status + DropPartition(const std::string& partition_name) = 0; + + virtual Status + ShowPartitions(const std::string& table_name, std::vector& partiton_schema_array) = 0; + + virtual Status + GetPartitionName(const std::string& table_name, const std::string& tag, std::string& partition_name) = 0; + virtual Status FilesToSearch(const std::string& table_id, const std::vector& ids, const DatesT& dates, DatePartionedTableFilesSchema& files) = 0; @@ -87,12 +105,6 @@ class Meta { virtual Status FilesToMerge(const std::string& table_id, DatePartionedTableFilesSchema& files) = 0; - virtual Status - Size(uint64_t& result) = 0; - - virtual Status - Archive() = 0; - virtual Status FilesToIndex(TableFilesSchema&) = 0; @@ -101,10 +113,10 @@ class Meta { std::vector& file_ids) = 0; virtual Status - DescribeTableIndex(const std::string& table_id, TableIndex& index) = 0; + Size(uint64_t& result) = 0; virtual Status - DropTableIndex(const std::string& table_id) = 0; + Archive() = 0; virtual Status CleanUp() = 0; diff --git a/core/src/db/meta/MetaTypes.h b/core/src/db/meta/MetaTypes.h index c973f3fdea..d98b74be7d 100644 --- a/core/src/db/meta/MetaTypes.h +++ b/core/src/db/meta/MetaTypes.h @@ -19,6 +19,7 @@ #include "db/Constants.h" #include "db/engine/ExecutionEngine.h" +#include "src/version.h" #include #include @@ -33,6 +34,7 @@ constexpr int32_t DEFAULT_ENGINE_TYPE = (int)EngineType::FAISS_IDMAP; constexpr int32_t DEFAULT_NLIST = 16384; constexpr int32_t DEFAULT_METRIC_TYPE = (int)MetricType::L2; constexpr int32_t DEFAULT_INDEX_FILE_SIZE = ONE_GB; +constexpr char CURRENT_VERSION[] = MILVUS_VERSION; constexpr int64_t FLAG_MASK_NO_USERID = 0x1; constexpr int64_t FLAG_MASK_HAS_USERID = 0x1 << 1; @@ -57,6 +59,9 @@ struct TableSchema { int32_t engine_type_ = DEFAULT_ENGINE_TYPE; int32_t nlist_ = DEFAULT_NLIST; int32_t metric_type_ = DEFAULT_METRIC_TYPE; + std::string owner_table_; + std::string partition_tag_; + std::string version_ = CURRENT_VERSION; }; // TableSchema struct TableFileSchema { diff --git a/core/src/db/meta/MySQLMetaImpl.cpp b/core/src/db/meta/MySQLMetaImpl.cpp index c7a054524c..4406b87f7e 100644 --- a/core/src/db/meta/MySQLMetaImpl.cpp +++ b/core/src/db/meta/MySQLMetaImpl.cpp @@ -22,6 +22,7 @@ #include "metrics/Metrics.h" #include "utils/Exception.h" #include "utils/Log.h" +#include "utils/StringHelpFunctions.h" #include #include @@ -145,6 +146,10 @@ static const MetaSchema TABLES_SCHEMA(META_TABLES, { MetaField("engine_type", "INT", "DEFAULT 1 NOT NULL"), MetaField("nlist", "INT", "DEFAULT 16384 NOT NULL"), MetaField("metric_type", "INT", "DEFAULT 1 NOT NULL"), + MetaField("owner_table", "VARCHAR(255)", "NOT NULL"), + MetaField("partition_tag", "VARCHAR(255)", "NOT NULL"), + MetaField("version", "VARCHAR(64)", + std::string("DEFAULT '") + CURRENT_VERSION + "'"), }); // TableFiles schema @@ -294,7 +299,7 @@ MySQLMetaImpl::Initialize() { mysqlpp::ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); if (connectionPtr == nullptr) { - return Status(DB_ERROR, "Failed to connect to database server"); + return Status(DB_ERROR, "Failed to connect to meta server(mysql)"); } if (!connectionPtr->thread_aware()) { @@ -328,9 +333,350 @@ MySQLMetaImpl::Initialize() { return Status::OK(); } +Status +MySQLMetaImpl::CreateTable(TableSchema& table_schema) { + try { + server::MetricCollector metric; + { + mysqlpp::ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); + + if (connectionPtr == nullptr) { + return Status(DB_ERROR, "Failed to connect to meta server(mysql)"); + } + + mysqlpp::Query createTableQuery = connectionPtr->query(); + + if (table_schema.table_id_.empty()) { + NextTableId(table_schema.table_id_); + } else { + createTableQuery << "SELECT state FROM " << META_TABLES << " WHERE table_id = " << mysqlpp::quote + << table_schema.table_id_ << ";"; + + ENGINE_LOG_DEBUG << "MySQLMetaImpl::CreateTable: " << createTableQuery.str(); + + mysqlpp::StoreQueryResult res = createTableQuery.store(); + + if (res.num_rows() == 1) { + int state = res[0]["state"]; + if (TableSchema::TO_DELETE == state) { + return Status(DB_ERROR, "Table already exists and it is in delete state, please wait a second"); + } else { + return Status(DB_ALREADY_EXIST, "Table already exists"); + } + } + } + + table_schema.id_ = -1; + table_schema.created_on_ = utils::GetMicroSecTimeStamp(); + + std::string id = "NULL"; // auto-increment + std::string& table_id = table_schema.table_id_; + std::string state = std::to_string(table_schema.state_); + std::string dimension = std::to_string(table_schema.dimension_); + std::string created_on = std::to_string(table_schema.created_on_); + std::string flag = std::to_string(table_schema.flag_); + std::string index_file_size = std::to_string(table_schema.index_file_size_); + std::string engine_type = std::to_string(table_schema.engine_type_); + std::string nlist = std::to_string(table_schema.nlist_); + std::string metric_type = std::to_string(table_schema.metric_type_); + std::string& owner_table = table_schema.owner_table_; + std::string& partition_tag = table_schema.partition_tag_; + std::string& version = table_schema.version_; + + createTableQuery << "INSERT INTO " << META_TABLES << " VALUES(" << id << ", " << mysqlpp::quote << table_id + << ", " << state << ", " << dimension << ", " << created_on << ", " << flag << ", " + << index_file_size << ", " << engine_type << ", " << nlist << ", " << metric_type << ", " + << mysqlpp::quote << owner_table << ", " << mysqlpp::quote << partition_tag << ", " + << mysqlpp::quote << version << ");"; + + ENGINE_LOG_DEBUG << "MySQLMetaImpl::CreateTable: " << createTableQuery.str(); + + if (mysqlpp::SimpleResult res = createTableQuery.execute()) { + table_schema.id_ = res.insert_id(); // Might need to use SELECT LAST_INSERT_ID()? + + // Consume all results to avoid "Commands out of sync" error + } else { + return HandleException("Add Table Error", createTableQuery.error()); + } + } // Scoped Connection + + ENGINE_LOG_DEBUG << "Successfully create table: " << table_schema.table_id_; + return utils::CreateTablePath(options_, table_schema.table_id_); + } catch (std::exception& e) { + return HandleException("GENERAL ERROR WHEN CREATING TABLE", e.what()); + } +} + +Status +MySQLMetaImpl::DescribeTable(TableSchema& table_schema) { + try { + server::MetricCollector metric; + mysqlpp::StoreQueryResult res; + { + mysqlpp::ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); + + if (connectionPtr == nullptr) { + return Status(DB_ERROR, "Failed to connect to meta server(mysql)"); + } + + mysqlpp::Query describeTableQuery = connectionPtr->query(); + describeTableQuery + << "SELECT id, state, dimension, created_on, flag, index_file_size, engine_type, nlist, metric_type" + << " ,owner_table, partition_tag, version" + << " FROM " << META_TABLES << " WHERE table_id = " << mysqlpp::quote << table_schema.table_id_ + << " AND state <> " << std::to_string(TableSchema::TO_DELETE) << ";"; + + ENGINE_LOG_DEBUG << "MySQLMetaImpl::DescribeTable: " << describeTableQuery.str(); + + res = describeTableQuery.store(); + } // Scoped Connection + + if (res.num_rows() == 1) { + const mysqlpp::Row& resRow = res[0]; + table_schema.id_ = resRow["id"]; // implicit conversion + table_schema.state_ = resRow["state"]; + table_schema.dimension_ = resRow["dimension"]; + table_schema.created_on_ = resRow["created_on"]; + table_schema.flag_ = resRow["flag"]; + table_schema.index_file_size_ = resRow["index_file_size"]; + table_schema.engine_type_ = resRow["engine_type"]; + table_schema.nlist_ = resRow["nlist"]; + table_schema.metric_type_ = resRow["metric_type"]; + resRow["owner_table"].to_string(table_schema.owner_table_); + resRow["partition_tag"].to_string(table_schema.partition_tag_); + resRow["version"].to_string(table_schema.version_); + } else { + return Status(DB_NOT_FOUND, "Table " + table_schema.table_id_ + " not found"); + } + } catch (std::exception& e) { + return HandleException("GENERAL ERROR WHEN DESCRIBING TABLE", e.what()); + } + + return Status::OK(); +} + +Status +MySQLMetaImpl::HasTable(const std::string& table_id, bool& has_or_not) { + try { + server::MetricCollector metric; + mysqlpp::StoreQueryResult res; + { + mysqlpp::ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); + + if (connectionPtr == nullptr) { + return Status(DB_ERROR, "Failed to connect to meta server(mysql)"); + } + + mysqlpp::Query hasTableQuery = connectionPtr->query(); + // since table_id is a unique column we just need to check whether it exists or not + hasTableQuery << "SELECT EXISTS" + << " (SELECT 1 FROM " << META_TABLES << " WHERE table_id = " << mysqlpp::quote << table_id + << " AND state <> " << std::to_string(TableSchema::TO_DELETE) << ")" + << " AS " << mysqlpp::quote << "check" + << ";"; + + ENGINE_LOG_DEBUG << "MySQLMetaImpl::HasTable: " << hasTableQuery.str(); + + res = hasTableQuery.store(); + } // Scoped Connection + + int check = res[0]["check"]; + has_or_not = (check == 1); + } catch (std::exception& e) { + return HandleException("GENERAL ERROR WHEN CHECKING IF TABLE EXISTS", e.what()); + } + + return Status::OK(); +} + +Status +MySQLMetaImpl::AllTables(std::vector& table_schema_array) { + try { + server::MetricCollector metric; + mysqlpp::StoreQueryResult res; + { + mysqlpp::ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); + + if (connectionPtr == nullptr) { + return Status(DB_ERROR, "Failed to connect to meta server(mysql)"); + } + + mysqlpp::Query allTablesQuery = connectionPtr->query(); + allTablesQuery << "SELECT id, table_id, dimension, engine_type, nlist, index_file_size, metric_type" + << " ,owner_table, partition_tag, version" + << " FROM " << META_TABLES << " WHERE state <> " << std::to_string(TableSchema::TO_DELETE) + << ";"; + + ENGINE_LOG_DEBUG << "MySQLMetaImpl::AllTables: " << allTablesQuery.str(); + + res = allTablesQuery.store(); + } // Scoped Connection + + for (auto& resRow : res) { + TableSchema table_schema; + table_schema.id_ = resRow["id"]; // implicit conversion + resRow["table_id"].to_string(table_schema.table_id_); + table_schema.dimension_ = resRow["dimension"]; + table_schema.index_file_size_ = resRow["index_file_size"]; + table_schema.engine_type_ = resRow["engine_type"]; + table_schema.nlist_ = resRow["nlist"]; + table_schema.metric_type_ = resRow["metric_type"]; + resRow["owner_table"].to_string(table_schema.owner_table_); + resRow["partition_tag"].to_string(table_schema.partition_tag_); + resRow["version"].to_string(table_schema.version_); + + table_schema_array.emplace_back(table_schema); + } + } catch (std::exception& e) { + return HandleException("GENERAL ERROR WHEN DESCRIBING ALL TABLES", e.what()); + } + + return Status::OK(); +} + +Status +MySQLMetaImpl::DropTable(const std::string& table_id) { + try { + server::MetricCollector metric; + { + mysqlpp::ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); + + if (connectionPtr == nullptr) { + return Status(DB_ERROR, "Failed to connect to meta server(mysql)"); + } + + // soft delete table + mysqlpp::Query deleteTableQuery = connectionPtr->query(); + // + deleteTableQuery << "UPDATE " << META_TABLES << " SET state = " << std::to_string(TableSchema::TO_DELETE) + << " WHERE table_id = " << mysqlpp::quote << table_id << ";"; + + ENGINE_LOG_DEBUG << "MySQLMetaImpl::DeleteTable: " << deleteTableQuery.str(); + + if (!deleteTableQuery.exec()) { + return HandleException("QUERY ERROR WHEN DELETING TABLE", deleteTableQuery.error()); + } + } // Scoped Connection + + if (mode_ == DBOptions::MODE::CLUSTER_WRITABLE) { + DeleteTableFiles(table_id); + } + + ENGINE_LOG_DEBUG << "Successfully delete table, table id = " << table_id; + } catch (std::exception& e) { + return HandleException("GENERAL ERROR WHEN DELETING TABLE", e.what()); + } + + return Status::OK(); +} + +Status +MySQLMetaImpl::DeleteTableFiles(const std::string& table_id) { + try { + server::MetricCollector metric; + { + mysqlpp::ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); + + if (connectionPtr == nullptr) { + return Status(DB_ERROR, "Failed to connect to meta server(mysql)"); + } + + // soft delete table files + mysqlpp::Query deleteTableFilesQuery = connectionPtr->query(); + // + deleteTableFilesQuery << "UPDATE " << META_TABLEFILES + << " SET file_type = " << std::to_string(TableFileSchema::TO_DELETE) + << " ,updated_time = " << std::to_string(utils::GetMicroSecTimeStamp()) + << " WHERE table_id = " << mysqlpp::quote << table_id << " AND file_type <> " + << std::to_string(TableFileSchema::TO_DELETE) << ";"; + + ENGINE_LOG_DEBUG << "MySQLMetaImpl::DeleteTableFiles: " << deleteTableFilesQuery.str(); + + if (!deleteTableFilesQuery.exec()) { + return HandleException("QUERY ERROR WHEN DELETING TABLE FILES", deleteTableFilesQuery.error()); + } + } // Scoped Connection + + ENGINE_LOG_DEBUG << "Successfully delete table files, table id = " << table_id; + } catch (std::exception& e) { + return HandleException("GENERAL ERROR WHEN DELETING TABLE FILES", e.what()); + } + + return Status::OK(); +} + +Status +MySQLMetaImpl::CreateTableFile(TableFileSchema& file_schema) { + if (file_schema.date_ == EmptyDate) { + file_schema.date_ = utils::GetDate(); + } + TableSchema table_schema; + table_schema.table_id_ = file_schema.table_id_; + auto status = DescribeTable(table_schema); + if (!status.ok()) { + return status; + } + + try { + server::MetricCollector metric; + + NextFileId(file_schema.file_id_); + file_schema.dimension_ = table_schema.dimension_; + file_schema.file_size_ = 0; + file_schema.row_count_ = 0; + file_schema.created_on_ = utils::GetMicroSecTimeStamp(); + file_schema.updated_time_ = file_schema.created_on_; + file_schema.index_file_size_ = table_schema.index_file_size_; + file_schema.engine_type_ = table_schema.engine_type_; + file_schema.nlist_ = table_schema.nlist_; + file_schema.metric_type_ = table_schema.metric_type_; + + std::string id = "NULL"; // auto-increment + std::string table_id = file_schema.table_id_; + std::string engine_type = std::to_string(file_schema.engine_type_); + std::string file_id = file_schema.file_id_; + std::string file_type = std::to_string(file_schema.file_type_); + std::string file_size = std::to_string(file_schema.file_size_); + std::string row_count = std::to_string(file_schema.row_count_); + std::string updated_time = std::to_string(file_schema.updated_time_); + std::string created_on = std::to_string(file_schema.created_on_); + std::string date = std::to_string(file_schema.date_); + + { + mysqlpp::ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); + + if (connectionPtr == nullptr) { + return Status(DB_ERROR, "Failed to connect to meta server(mysql)"); + } + + mysqlpp::Query createTableFileQuery = connectionPtr->query(); + + createTableFileQuery << "INSERT INTO " << META_TABLEFILES << " VALUES(" << id << ", " << mysqlpp::quote + << table_id << ", " << engine_type << ", " << mysqlpp::quote << file_id << ", " + << file_type << ", " << file_size << ", " << row_count << ", " << updated_time << ", " + << created_on << ", " << date << ");"; + + ENGINE_LOG_DEBUG << "MySQLMetaImpl::CreateTableFile: " << createTableFileQuery.str(); + + if (mysqlpp::SimpleResult res = createTableFileQuery.execute()) { + file_schema.id_ = res.insert_id(); // Might need to use SELECT LAST_INSERT_ID()? + + // Consume all results to avoid "Commands out of sync" error + } else { + return HandleException("QUERY ERROR WHEN CREATING TABLE FILE", createTableFileQuery.error()); + } + } // Scoped Connection + + ENGINE_LOG_DEBUG << "Successfully create table file, file id = " << file_schema.file_id_; + return utils::CreateTableFilePath(options_, file_schema); + } catch (std::exception& e) { + return HandleException("GENERAL ERROR WHEN CREATING TABLE FILE", e.what()); + } +} + // TODO(myh): Delete single vecotor by id Status -MySQLMetaImpl::DropPartitionsByDates(const std::string& table_id, const DatesT& dates) { +MySQLMetaImpl::DropDataByDate(const std::string& table_id, const DatesT& dates) { if (dates.empty()) { return Status::OK(); } @@ -354,18 +700,18 @@ MySQLMetaImpl::DropPartitionsByDates(const std::string& table_id, const DatesT& mysqlpp::ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); if (connectionPtr == nullptr) { - return Status(DB_ERROR, "Failed to connect to database server"); + return Status(DB_ERROR, "Failed to connect to meta server(mysql)"); } mysqlpp::Query dropPartitionsByDatesQuery = connectionPtr->query(); - dropPartitionsByDatesQuery << "UPDATE " << META_TABLEFILES << " " - << "SET file_type = " << std::to_string(TableFileSchema::TO_DELETE) << "," - << "updated_time = " << utils::GetMicroSecTimeStamp() << " " - << "WHERE table_id = " << mysqlpp::quote << table_id << " AND " - << "date in (" << dateListStr << ");"; + dropPartitionsByDatesQuery << "UPDATE " << META_TABLEFILES + << " SET file_type = " << std::to_string(TableFileSchema::TO_DELETE) + << " ,updated_time = " << utils::GetMicroSecTimeStamp() + << " WHERE table_id = " << mysqlpp::quote << table_id << " AND date in (" + << dateListStr << ");"; - ENGINE_LOG_DEBUG << "MySQLMetaImpl::DropPartitionsByDates: " << dropPartitionsByDatesQuery.str(); + ENGINE_LOG_DEBUG << "MySQLMetaImpl::DropDataByDate: " << dropPartitionsByDatesQuery.str(); if (!dropPartitionsByDatesQuery.exec()) { return HandleException("QUERY ERROR WHEN DROPPING PARTITIONS BY DATES", @@ -373,7 +719,7 @@ MySQLMetaImpl::DropPartitionsByDates(const std::string& table_id, const DatesT& } } // Scoped Connection - ENGINE_LOG_DEBUG << "Successfully drop partitions, table id = " << table_schema.table_id_; + ENGINE_LOG_DEBUG << "Successfully drop data by date, table id = " << table_schema.table_id_; } catch (std::exception& e) { return HandleException("GENERAL ERROR WHEN DROPPING PARTITIONS BY DATES", e.what()); } @@ -381,72 +727,801 @@ MySQLMetaImpl::DropPartitionsByDates(const std::string& table_id, const DatesT& } Status -MySQLMetaImpl::CreateTable(TableSchema& table_schema) { +MySQLMetaImpl::GetTableFiles(const std::string& table_id, const std::vector& ids, + TableFilesSchema& table_files) { + if (ids.empty()) { + return Status::OK(); + } + + std::stringstream idSS; + for (auto& id : ids) { + idSS << "id = " << std::to_string(id) << " OR "; + } + std::string idStr = idSS.str(); + idStr = idStr.substr(0, idStr.size() - 4); // remove the last " OR " + + try { + mysqlpp::StoreQueryResult res; + { + mysqlpp::ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); + + if (connectionPtr == nullptr) { + return Status(DB_ERROR, "Failed to connect to meta server(mysql)"); + } + + mysqlpp::Query getTableFileQuery = connectionPtr->query(); + getTableFileQuery << "SELECT id, engine_type, file_id, file_type, file_size, row_count, date, created_on" + << " FROM " << META_TABLEFILES << " WHERE table_id = " << mysqlpp::quote << table_id + << " AND (" << idStr << ")" + << " AND file_type <> " << std::to_string(TableFileSchema::TO_DELETE) << ";"; + + ENGINE_LOG_DEBUG << "MySQLMetaImpl::GetTableFiles: " << getTableFileQuery.str(); + + res = getTableFileQuery.store(); + } // Scoped Connection + + TableSchema table_schema; + table_schema.table_id_ = table_id; + DescribeTable(table_schema); + + Status ret; + for (auto& resRow : res) { + TableFileSchema file_schema; + file_schema.id_ = resRow["id"]; + file_schema.table_id_ = table_id; + file_schema.index_file_size_ = table_schema.index_file_size_; + file_schema.engine_type_ = resRow["engine_type"]; + file_schema.nlist_ = table_schema.nlist_; + file_schema.metric_type_ = table_schema.metric_type_; + resRow["file_id"].to_string(file_schema.file_id_); + file_schema.file_type_ = resRow["file_type"]; + file_schema.file_size_ = resRow["file_size"]; + file_schema.row_count_ = resRow["row_count"]; + file_schema.date_ = resRow["date"]; + file_schema.created_on_ = resRow["created_on"]; + file_schema.dimension_ = table_schema.dimension_; + + utils::GetTableFilePath(options_, file_schema); + table_files.emplace_back(file_schema); + } + + ENGINE_LOG_DEBUG << "Get table files by id"; + return ret; + } catch (std::exception& e) { + return HandleException("GENERAL ERROR WHEN RETRIEVING TABLE FILES", e.what()); + } +} + +Status +MySQLMetaImpl::UpdateTableIndex(const std::string& table_id, const TableIndex& index) { + try { + server::MetricCollector metric; + + { + mysqlpp::ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); + + if (connectionPtr == nullptr) { + return Status(DB_ERROR, "Failed to connect to meta server(mysql)"); + } + + mysqlpp::Query updateTableIndexParamQuery = connectionPtr->query(); + updateTableIndexParamQuery << "SELECT id, state, dimension, created_on" + << " FROM " << META_TABLES << " WHERE table_id = " << mysqlpp::quote << table_id + << " AND state <> " << std::to_string(TableSchema::TO_DELETE) << ";"; + + ENGINE_LOG_DEBUG << "MySQLMetaImpl::UpdateTableIndex: " << updateTableIndexParamQuery.str(); + + mysqlpp::StoreQueryResult res = updateTableIndexParamQuery.store(); + + if (res.num_rows() == 1) { + const mysqlpp::Row& resRow = res[0]; + + size_t id = resRow["id"]; + int32_t state = resRow["state"]; + uint16_t dimension = resRow["dimension"]; + int64_t created_on = resRow["created_on"]; + + updateTableIndexParamQuery << "UPDATE " << META_TABLES << " SET id = " << id << " ,state = " << state + << " ,dimension = " << dimension << " ,created_on = " << created_on + << " ,engine_type = " << index.engine_type_ << " ,nlist = " << index.nlist_ + << " ,metric_type = " << index.metric_type_ + << " WHERE table_id = " << mysqlpp::quote << table_id << ";"; + + ENGINE_LOG_DEBUG << "MySQLMetaImpl::UpdateTableIndex: " << updateTableIndexParamQuery.str(); + + if (!updateTableIndexParamQuery.exec()) { + return HandleException("QUERY ERROR WHEN UPDATING TABLE INDEX PARAM", + updateTableIndexParamQuery.error()); + } + } else { + return Status(DB_NOT_FOUND, "Table " + table_id + " not found"); + } + } // Scoped Connection + + ENGINE_LOG_DEBUG << "Successfully update table index, table id = " << table_id; + } catch (std::exception& e) { + return HandleException("GENERAL ERROR WHEN UPDATING TABLE INDEX PARAM", e.what()); + } + + return Status::OK(); +} + +Status +MySQLMetaImpl::UpdateTableFlag(const std::string& table_id, int64_t flag) { + try { + server::MetricCollector metric; + + { + mysqlpp::ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); + + if (connectionPtr == nullptr) { + return Status(DB_ERROR, "Failed to connect to meta server(mysql)"); + } + + mysqlpp::Query updateTableFlagQuery = connectionPtr->query(); + updateTableFlagQuery << "UPDATE " << META_TABLES << " SET flag = " << flag + << " WHERE table_id = " << mysqlpp::quote << table_id << ";"; + + ENGINE_LOG_DEBUG << "MySQLMetaImpl::UpdateTableFlag: " << updateTableFlagQuery.str(); + + if (!updateTableFlagQuery.exec()) { + return HandleException("QUERY ERROR WHEN UPDATING TABLE FLAG", updateTableFlagQuery.error()); + } + } // Scoped Connection + + ENGINE_LOG_DEBUG << "Successfully update table flag, table id = " << table_id; + } catch (std::exception& e) { + return HandleException("GENERAL ERROR WHEN UPDATING TABLE FLAG", e.what()); + } + + return Status::OK(); +} + +// ZR: this function assumes all fields in file_schema have value +Status +MySQLMetaImpl::UpdateTableFile(TableFileSchema& file_schema) { + file_schema.updated_time_ = utils::GetMicroSecTimeStamp(); + try { server::MetricCollector metric; { mysqlpp::ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); if (connectionPtr == nullptr) { - return Status(DB_ERROR, "Failed to connect to database server"); + return Status(DB_ERROR, "Failed to connect to meta server(mysql)"); } - mysqlpp::Query createTableQuery = connectionPtr->query(); + mysqlpp::Query updateTableFileQuery = connectionPtr->query(); - if (table_schema.table_id_.empty()) { - NextTableId(table_schema.table_id_); - } else { - createTableQuery << "SELECT state FROM " << META_TABLES << " " - << "WHERE table_id = " << mysqlpp::quote << table_schema.table_id_ << ";"; + // if the table has been deleted, just mark the table file as TO_DELETE + // clean thread will delete the file later + updateTableFileQuery << "SELECT state FROM " << META_TABLES << " WHERE table_id = " << mysqlpp::quote + << file_schema.table_id_ << ";"; - ENGINE_LOG_DEBUG << "MySQLMetaImpl::CreateTable: " << createTableQuery.str(); + ENGINE_LOG_DEBUG << "MySQLMetaImpl::UpdateTableFile: " << updateTableFileQuery.str(); - mysqlpp::StoreQueryResult res = createTableQuery.store(); + mysqlpp::StoreQueryResult res = updateTableFileQuery.store(); - if (res.num_rows() == 1) { - int state = res[0]["state"]; - if (TableSchema::TO_DELETE == state) { - return Status(DB_ERROR, "Table already exists and it is in delete state, please wait a second"); - } else { - return Status(DB_ALREADY_EXIST, "Table already exists"); - } + if (res.num_rows() == 1) { + int state = res[0]["state"]; + if (state == TableSchema::TO_DELETE) { + file_schema.file_type_ = TableFileSchema::TO_DELETE; } + } else { + file_schema.file_type_ = TableFileSchema::TO_DELETE; } - table_schema.id_ = -1; - table_schema.created_on_ = utils::GetMicroSecTimeStamp(); + std::string id = std::to_string(file_schema.id_); + std::string table_id = file_schema.table_id_; + std::string engine_type = std::to_string(file_schema.engine_type_); + std::string file_id = file_schema.file_id_; + std::string file_type = std::to_string(file_schema.file_type_); + std::string file_size = std::to_string(file_schema.file_size_); + std::string row_count = std::to_string(file_schema.row_count_); + std::string updated_time = std::to_string(file_schema.updated_time_); + std::string created_on = std::to_string(file_schema.created_on_); + std::string date = std::to_string(file_schema.date_); - std::string id = "NULL"; // auto-increment - std::string table_id = table_schema.table_id_; - std::string state = std::to_string(table_schema.state_); - std::string dimension = std::to_string(table_schema.dimension_); - std::string created_on = std::to_string(table_schema.created_on_); - std::string flag = std::to_string(table_schema.flag_); - std::string index_file_size = std::to_string(table_schema.index_file_size_); - std::string engine_type = std::to_string(table_schema.engine_type_); - std::string nlist = std::to_string(table_schema.nlist_); - std::string metric_type = std::to_string(table_schema.metric_type_); + updateTableFileQuery << "UPDATE " << META_TABLEFILES << " SET table_id = " << mysqlpp::quote << table_id + << " ,engine_type = " << engine_type << " ,file_id = " << mysqlpp::quote << file_id + << " ,file_type = " << file_type << " ,file_size = " << file_size + << " ,row_count = " << row_count << " ,updated_time = " << updated_time + << " ,created_on = " << created_on << " ,date = " << date << " WHERE id = " << id + << ";"; - createTableQuery << "INSERT INTO " << META_TABLES << " " - << "VALUES(" << id << ", " << mysqlpp::quote << table_id << ", " << state << ", " - << dimension << ", " << created_on << ", " << flag << ", " << index_file_size << ", " - << engine_type << ", " << nlist << ", " << metric_type << ");"; + ENGINE_LOG_DEBUG << "MySQLMetaImpl::UpdateTableFile: " << updateTableFileQuery.str(); - ENGINE_LOG_DEBUG << "MySQLMetaImpl::CreateTable: " << createTableQuery.str(); - - if (mysqlpp::SimpleResult res = createTableQuery.execute()) { - table_schema.id_ = res.insert_id(); // Might need to use SELECT LAST_INSERT_ID()? - - // Consume all results to avoid "Commands out of sync" error - } else { - return HandleException("Add Table Error", createTableQuery.error()); + if (!updateTableFileQuery.exec()) { + ENGINE_LOG_DEBUG << "table_id= " << file_schema.table_id_ << " file_id=" << file_schema.file_id_; + return HandleException("QUERY ERROR WHEN UPDATING TABLE FILE", updateTableFileQuery.error()); } } // Scoped Connection - ENGINE_LOG_DEBUG << "Successfully create table: " << table_schema.table_id_; - return utils::CreateTablePath(options_, table_schema.table_id_); + ENGINE_LOG_DEBUG << "Update single table file, file id = " << file_schema.file_id_; } catch (std::exception& e) { - return HandleException("GENERAL ERROR WHEN CREATING TABLE", e.what()); + return HandleException("GENERAL ERROR WHEN UPDATING TABLE FILE", e.what()); + } + + return Status::OK(); +} + +Status +MySQLMetaImpl::UpdateTableFilesToIndex(const std::string& table_id) { + try { + mysqlpp::ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); + + if (connectionPtr == nullptr) { + return Status(DB_ERROR, "Failed to connect to meta server(mysql)"); + } + + mysqlpp::Query updateTableFilesToIndexQuery = connectionPtr->query(); + + updateTableFilesToIndexQuery << "UPDATE " << META_TABLEFILES + << " SET file_type = " << std::to_string(TableFileSchema::TO_INDEX) + << " WHERE table_id = " << mysqlpp::quote << table_id + << " AND file_type = " << std::to_string(TableFileSchema::RAW) << ";"; + + ENGINE_LOG_DEBUG << "MySQLMetaImpl::UpdateTableFilesToIndex: " << updateTableFilesToIndexQuery.str(); + + if (!updateTableFilesToIndexQuery.exec()) { + return HandleException("QUERY ERROR WHEN UPDATING TABLE FILE TO INDEX", + updateTableFilesToIndexQuery.error()); + } + + ENGINE_LOG_DEBUG << "Update files to to_index, table id = " << table_id; + } catch (std::exception& e) { + return HandleException("GENERAL ERROR WHEN UPDATING TABLE FILES TO INDEX", e.what()); + } + + return Status::OK(); +} + +Status +MySQLMetaImpl::UpdateTableFiles(TableFilesSchema& files) { + try { + server::MetricCollector metric; + { + mysqlpp::ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); + + if (connectionPtr == nullptr) { + return Status(DB_ERROR, "Failed to connect to meta server(mysql)"); + } + + mysqlpp::Query updateTableFilesQuery = connectionPtr->query(); + + std::map has_tables; + for (auto& file_schema : files) { + if (has_tables.find(file_schema.table_id_) != has_tables.end()) { + continue; + } + + updateTableFilesQuery << "SELECT EXISTS" + << " (SELECT 1 FROM " << META_TABLES << " WHERE table_id = " << mysqlpp::quote + << file_schema.table_id_ << " AND state <> " + << std::to_string(TableSchema::TO_DELETE) << ")" + << " AS " << mysqlpp::quote << "check" + << ";"; + + ENGINE_LOG_DEBUG << "MySQLMetaImpl::UpdateTableFiles: " << updateTableFilesQuery.str(); + + mysqlpp::StoreQueryResult res = updateTableFilesQuery.store(); + + int check = res[0]["check"]; + has_tables[file_schema.table_id_] = (check == 1); + } + + for (auto& file_schema : files) { + if (!has_tables[file_schema.table_id_]) { + file_schema.file_type_ = TableFileSchema::TO_DELETE; + } + file_schema.updated_time_ = utils::GetMicroSecTimeStamp(); + + std::string id = std::to_string(file_schema.id_); + std::string& table_id = file_schema.table_id_; + std::string engine_type = std::to_string(file_schema.engine_type_); + std::string& file_id = file_schema.file_id_; + std::string file_type = std::to_string(file_schema.file_type_); + std::string file_size = std::to_string(file_schema.file_size_); + std::string row_count = std::to_string(file_schema.row_count_); + std::string updated_time = std::to_string(file_schema.updated_time_); + std::string created_on = std::to_string(file_schema.created_on_); + std::string date = std::to_string(file_schema.date_); + + updateTableFilesQuery << "UPDATE " << META_TABLEFILES << " SET table_id = " << mysqlpp::quote + << table_id << " ,engine_type = " << engine_type + << " ,file_id = " << mysqlpp::quote << file_id << " ,file_type = " << file_type + << " ,file_size = " << file_size << " ,row_count = " << row_count + << " ,updated_time = " << updated_time << " ,created_on = " << created_on + << " ,date = " << date << " WHERE id = " << id << ";"; + + ENGINE_LOG_DEBUG << "MySQLMetaImpl::UpdateTableFiles: " << updateTableFilesQuery.str(); + + if (!updateTableFilesQuery.exec()) { + return HandleException("QUERY ERROR WHEN UPDATING TABLE FILES", updateTableFilesQuery.error()); + } + } + } // Scoped Connection + + ENGINE_LOG_DEBUG << "Update " << files.size() << " table files"; + } catch (std::exception& e) { + return HandleException("GENERAL ERROR WHEN UPDATING TABLE FILES", e.what()); + } + + return Status::OK(); +} + +Status +MySQLMetaImpl::DescribeTableIndex(const std::string& table_id, TableIndex& index) { + try { + server::MetricCollector metric; + + { + mysqlpp::ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); + + if (connectionPtr == nullptr) { + return Status(DB_ERROR, "Failed to connect to meta server(mysql)"); + } + + mysqlpp::Query describeTableIndexQuery = connectionPtr->query(); + describeTableIndexQuery << "SELECT engine_type, nlist, index_file_size, metric_type" + << " FROM " << META_TABLES << " WHERE table_id = " << mysqlpp::quote << table_id + << " AND state <> " << std::to_string(TableSchema::TO_DELETE) << ";"; + + ENGINE_LOG_DEBUG << "MySQLMetaImpl::DescribeTableIndex: " << describeTableIndexQuery.str(); + + mysqlpp::StoreQueryResult res = describeTableIndexQuery.store(); + + if (res.num_rows() == 1) { + const mysqlpp::Row& resRow = res[0]; + + index.engine_type_ = resRow["engine_type"]; + index.nlist_ = resRow["nlist"]; + index.metric_type_ = resRow["metric_type"]; + } else { + return Status(DB_NOT_FOUND, "Table " + table_id + " not found"); + } + } // Scoped Connection + } catch (std::exception& e) { + return HandleException("GENERAL ERROR WHEN UPDATING TABLE FLAG", e.what()); + } + + return Status::OK(); +} + +Status +MySQLMetaImpl::DropTableIndex(const std::string& table_id) { + try { + server::MetricCollector metric; + + { + mysqlpp::ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); + + if (connectionPtr == nullptr) { + return Status(DB_ERROR, "Failed to connect to meta server(mysql)"); + } + + mysqlpp::Query dropTableIndexQuery = connectionPtr->query(); + + // soft delete index files + dropTableIndexQuery << "UPDATE " << META_TABLEFILES + << " SET file_type = " << std::to_string(TableFileSchema::TO_DELETE) + << " ,updated_time = " << utils::GetMicroSecTimeStamp() + << " WHERE table_id = " << mysqlpp::quote << table_id + << " AND file_type = " << std::to_string(TableFileSchema::INDEX) << ";"; + + ENGINE_LOG_DEBUG << "MySQLMetaImpl::DropTableIndex: " << dropTableIndexQuery.str(); + + if (!dropTableIndexQuery.exec()) { + return HandleException("QUERY ERROR WHEN DROPPING TABLE INDEX", dropTableIndexQuery.error()); + } + + // set all backup file to raw + dropTableIndexQuery << "UPDATE " << META_TABLEFILES + << " SET file_type = " << std::to_string(TableFileSchema::RAW) + << " ,updated_time = " << utils::GetMicroSecTimeStamp() + << " WHERE table_id = " << mysqlpp::quote << table_id + << " AND file_type = " << std::to_string(TableFileSchema::BACKUP) << ";"; + + ENGINE_LOG_DEBUG << "MySQLMetaImpl::DropTableIndex: " << dropTableIndexQuery.str(); + + if (!dropTableIndexQuery.exec()) { + return HandleException("QUERY ERROR WHEN DROPPING TABLE INDEX", dropTableIndexQuery.error()); + } + + // set table index type to raw + dropTableIndexQuery << "UPDATE " << META_TABLES + << " SET engine_type = " << std::to_string(DEFAULT_ENGINE_TYPE) + << " ,nlist = " << std::to_string(DEFAULT_NLIST) + << " ,metric_type = " << std::to_string(DEFAULT_METRIC_TYPE) + << " WHERE table_id = " << mysqlpp::quote << table_id << ";"; + + ENGINE_LOG_DEBUG << "MySQLMetaImpl::DropTableIndex: " << dropTableIndexQuery.str(); + + if (!dropTableIndexQuery.exec()) { + return HandleException("QUERY ERROR WHEN DROPPING TABLE INDEX", dropTableIndexQuery.error()); + } + } // Scoped Connection + + ENGINE_LOG_DEBUG << "Successfully drop table index, table id = " << table_id; + } catch (std::exception& e) { + return HandleException("GENERAL ERROR WHEN DROPPING TABLE INDEX", e.what()); + } + + return Status::OK(); +} + +Status +MySQLMetaImpl::CreatePartition(const std::string& table_id, const std::string& partition_name, const std::string& tag) { + server::MetricCollector metric; + + TableSchema table_schema; + table_schema.table_id_ = table_id; + auto status = DescribeTable(table_schema); + if (!status.ok()) { + return status; + } + + // not allow create partition under partition + if (!table_schema.owner_table_.empty()) { + return Status(DB_ERROR, "Nested partition is not allowed"); + } + + // trim side-blank of tag, only compare valid characters + // for example: " ab cd " is treated as "ab cd" + std::string valid_tag = tag; + server::StringHelpFunctions::TrimStringBlank(valid_tag); + + // not allow duplicated partition + std::string exist_partition; + GetPartitionName(table_id, valid_tag, exist_partition); + if (!exist_partition.empty()) { + return Status(DB_ERROR, "Duplicate partition is not allowed"); + } + + if (partition_name == "") { + // generate unique partition name + NextTableId(table_schema.table_id_); + } else { + table_schema.table_id_ = partition_name; + } + + table_schema.id_ = -1; + table_schema.flag_ = 0; + table_schema.created_on_ = utils::GetMicroSecTimeStamp(); + table_schema.owner_table_ = table_id; + table_schema.partition_tag_ = valid_tag; + + status = CreateTable(table_schema); + if (status.code() == DB_ALREADY_EXIST) { + return Status(DB_ALREADY_EXIST, "Partition already exists"); + } + + return status; +} + +Status +MySQLMetaImpl::DropPartition(const std::string& partition_name) { + return DropTable(partition_name); +} + +Status +MySQLMetaImpl::ShowPartitions(const std::string& table_id, std::vector& partiton_schema_array) { + try { + server::MetricCollector metric; + mysqlpp::StoreQueryResult res; + { + mysqlpp::ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); + + if (connectionPtr == nullptr) { + return Status(DB_ERROR, "Failed to connect to meta server(mysql)"); + } + + mysqlpp::Query allPartitionsQuery = connectionPtr->query(); + allPartitionsQuery << "SELECT table_id FROM " << META_TABLES << " WHERE owner_table = " << mysqlpp::quote + << table_id << " AND state <> " << std::to_string(TableSchema::TO_DELETE) << ";"; + + ENGINE_LOG_DEBUG << "MySQLMetaImpl::AllTables: " << allPartitionsQuery.str(); + + res = allPartitionsQuery.store(); + } // Scoped Connection + + for (auto& resRow : res) { + meta::TableSchema partition_schema; + resRow["table_id"].to_string(partition_schema.table_id_); + DescribeTable(partition_schema); + partiton_schema_array.emplace_back(partition_schema); + } + } catch (std::exception& e) { + return HandleException("GENERAL ERROR WHEN SHOW PARTITIONS", e.what()); + } + + return Status::OK(); +} + +Status +MySQLMetaImpl::GetPartitionName(const std::string& table_id, const std::string& tag, std::string& partition_name) { + try { + server::MetricCollector metric; + mysqlpp::StoreQueryResult res; + + // trim side-blank of tag, only compare valid characters + // for example: " ab cd " is treated as "ab cd" + std::string valid_tag = tag; + server::StringHelpFunctions::TrimStringBlank(valid_tag); + + { + mysqlpp::ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); + + if (connectionPtr == nullptr) { + return Status(DB_ERROR, "Failed to connect to meta server(mysql)"); + } + + mysqlpp::Query allPartitionsQuery = connectionPtr->query(); + allPartitionsQuery << "SELECT table_id FROM " << META_TABLES << " WHERE owner_table = " << mysqlpp::quote + << table_id << " AND partition_tag = " << mysqlpp::quote << valid_tag << " AND state <> " + << std::to_string(TableSchema::TO_DELETE) << ";"; + + ENGINE_LOG_DEBUG << "MySQLMetaImpl::AllTables: " << allPartitionsQuery.str(); + + res = allPartitionsQuery.store(); + } // Scoped Connection + + if (res.num_rows() > 0) { + const mysqlpp::Row& resRow = res[0]; + resRow["table_id"].to_string(partition_name); + } else { + return Status(DB_NOT_FOUND, "Partition " + valid_tag + " of table " + table_id + " not found"); + } + } catch (std::exception& e) { + return HandleException("GENERAL ERROR WHEN GET PARTITION NAME", e.what()); + } + + return Status::OK(); +} + +Status +MySQLMetaImpl::FilesToSearch(const std::string& table_id, const std::vector& ids, const DatesT& dates, + DatePartionedTableFilesSchema& files) { + files.clear(); + + try { + server::MetricCollector metric; + mysqlpp::StoreQueryResult res; + { + mysqlpp::ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); + + if (connectionPtr == nullptr) { + return Status(DB_ERROR, "Failed to connect to meta server(mysql)"); + } + + mysqlpp::Query filesToSearchQuery = connectionPtr->query(); + filesToSearchQuery << "SELECT id, table_id, engine_type, file_id, file_type, file_size, row_count, date" + << " FROM " << META_TABLEFILES << " WHERE table_id = " << mysqlpp::quote << table_id; + + if (!dates.empty()) { + std::stringstream partitionListSS; + for (auto& date : dates) { + partitionListSS << std::to_string(date) << ", "; + } + std::string partitionListStr = partitionListSS.str(); + + partitionListStr = partitionListStr.substr(0, partitionListStr.size() - 2); // remove the last ", " + filesToSearchQuery << " AND date IN (" << partitionListStr << ")"; + } + + if (!ids.empty()) { + std::stringstream idSS; + for (auto& id : ids) { + idSS << "id = " << std::to_string(id) << " OR "; + } + std::string idStr = idSS.str(); + idStr = idStr.substr(0, idStr.size() - 4); // remove the last " OR " + + filesToSearchQuery << " AND (" << idStr << ")"; + } + // End + filesToSearchQuery << " AND" + << " (file_type = " << std::to_string(TableFileSchema::RAW) + << " OR file_type = " << std::to_string(TableFileSchema::TO_INDEX) + << " OR file_type = " << std::to_string(TableFileSchema::INDEX) << ");"; + + ENGINE_LOG_DEBUG << "MySQLMetaImpl::FilesToSearch: " << filesToSearchQuery.str(); + + res = filesToSearchQuery.store(); + } // Scoped Connection + + TableSchema table_schema; + table_schema.table_id_ = table_id; + auto status = DescribeTable(table_schema); + if (!status.ok()) { + return status; + } + + Status ret; + TableFileSchema table_file; + for (auto& resRow : res) { + table_file.id_ = resRow["id"]; // implicit conversion + resRow["table_id"].to_string(table_file.table_id_); + table_file.index_file_size_ = table_schema.index_file_size_; + table_file.engine_type_ = resRow["engine_type"]; + table_file.nlist_ = table_schema.nlist_; + table_file.metric_type_ = table_schema.metric_type_; + resRow["file_id"].to_string(table_file.file_id_); + table_file.file_type_ = resRow["file_type"]; + table_file.file_size_ = resRow["file_size"]; + table_file.row_count_ = resRow["row_count"]; + table_file.date_ = resRow["date"]; + table_file.dimension_ = table_schema.dimension_; + + auto status = utils::GetTableFilePath(options_, table_file); + if (!status.ok()) { + ret = status; + } + + auto dateItr = files.find(table_file.date_); + if (dateItr == files.end()) { + files[table_file.date_] = TableFilesSchema(); + } + + files[table_file.date_].push_back(table_file); + } + + if (res.size() > 0) { + ENGINE_LOG_DEBUG << "Collect " << res.size() << " to-search files"; + } + return ret; + } catch (std::exception& e) { + return HandleException("GENERAL ERROR WHEN FINDING TABLE FILES TO SEARCH", e.what()); + } +} + +Status +MySQLMetaImpl::FilesToMerge(const std::string& table_id, DatePartionedTableFilesSchema& files) { + files.clear(); + + try { + server::MetricCollector metric; + + // check table existence + TableSchema table_schema; + table_schema.table_id_ = table_id; + auto status = DescribeTable(table_schema); + if (!status.ok()) { + return status; + } + + mysqlpp::StoreQueryResult res; + { + mysqlpp::ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); + + if (connectionPtr == nullptr) { + return Status(DB_ERROR, "Failed to connect to meta server(mysql)"); + } + + mysqlpp::Query filesToMergeQuery = connectionPtr->query(); + filesToMergeQuery + << "SELECT id, table_id, file_id, file_type, file_size, row_count, date, engine_type, created_on" + << " FROM " << META_TABLEFILES << " WHERE table_id = " << mysqlpp::quote << table_id + << " AND file_type = " << std::to_string(TableFileSchema::RAW) << " ORDER BY row_count DESC;"; + + ENGINE_LOG_DEBUG << "MySQLMetaImpl::FilesToMerge: " << filesToMergeQuery.str(); + + res = filesToMergeQuery.store(); + } // Scoped Connection + + Status ret; + int64_t to_merge_files = 0; + for (auto& resRow : res) { + TableFileSchema table_file; + table_file.file_size_ = resRow["file_size"]; + if (table_file.file_size_ >= table_schema.index_file_size_) { + continue; // skip large file + } + + table_file.id_ = resRow["id"]; // implicit conversion + resRow["table_id"].to_string(table_file.table_id_); + resRow["file_id"].to_string(table_file.file_id_); + table_file.file_type_ = resRow["file_type"]; + table_file.row_count_ = resRow["row_count"]; + table_file.date_ = resRow["date"]; + table_file.index_file_size_ = table_schema.index_file_size_; + table_file.engine_type_ = resRow["engine_type"]; + table_file.nlist_ = table_schema.nlist_; + table_file.metric_type_ = table_schema.metric_type_; + table_file.created_on_ = resRow["created_on"]; + table_file.dimension_ = table_schema.dimension_; + + auto status = utils::GetTableFilePath(options_, table_file); + if (!status.ok()) { + ret = status; + } + + auto dateItr = files.find(table_file.date_); + if (dateItr == files.end()) { + files[table_file.date_] = TableFilesSchema(); + to_merge_files++; + } + + files[table_file.date_].push_back(table_file); + } + + if (to_merge_files > 0) { + ENGINE_LOG_TRACE << "Collect " << to_merge_files << " to-merge files"; + } + return ret; + } catch (std::exception& e) { + return HandleException("GENERAL ERROR WHEN FINDING TABLE FILES TO MERGE", e.what()); + } +} + +Status +MySQLMetaImpl::FilesToIndex(TableFilesSchema& files) { + files.clear(); + + try { + server::MetricCollector metric; + mysqlpp::StoreQueryResult res; + { + mysqlpp::ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); + + if (connectionPtr == nullptr) { + return Status(DB_ERROR, "Failed to connect to meta server(mysql)"); + } + + mysqlpp::Query filesToIndexQuery = connectionPtr->query(); + filesToIndexQuery + << "SELECT id, table_id, engine_type, file_id, file_type, file_size, row_count, date, created_on" + << " FROM " << META_TABLEFILES << " WHERE file_type = " << std::to_string(TableFileSchema::TO_INDEX) + << ";"; + + ENGINE_LOG_DEBUG << "MySQLMetaImpl::FilesToIndex: " << filesToIndexQuery.str(); + + res = filesToIndexQuery.store(); + } // Scoped Connection + + Status ret; + std::map groups; + TableFileSchema table_file; + for (auto& resRow : res) { + table_file.id_ = resRow["id"]; // implicit conversion + resRow["table_id"].to_string(table_file.table_id_); + table_file.engine_type_ = resRow["engine_type"]; + resRow["file_id"].to_string(table_file.file_id_); + table_file.file_type_ = resRow["file_type"]; + table_file.file_size_ = resRow["file_size"]; + table_file.row_count_ = resRow["row_count"]; + table_file.date_ = resRow["date"]; + table_file.created_on_ = resRow["created_on"]; + + auto groupItr = groups.find(table_file.table_id_); + if (groupItr == groups.end()) { + TableSchema table_schema; + table_schema.table_id_ = table_file.table_id_; + auto status = DescribeTable(table_schema); + if (!status.ok()) { + return status; + } + groups[table_file.table_id_] = table_schema; + } + table_file.dimension_ = groups[table_file.table_id_].dimension_; + table_file.index_file_size_ = groups[table_file.table_id_].index_file_size_; + table_file.nlist_ = groups[table_file.table_id_].nlist_; + table_file.metric_type_ = groups[table_file.table_id_].metric_type_; + + auto status = utils::GetTableFilePath(options_, table_file); + if (!status.ok()) { + ret = status; + } + + files.push_back(table_file); + } + + if (res.size() > 0) { + ENGINE_LOG_DEBUG << "Collect " << res.size() << " to-index files"; + } + return ret; + } catch (std::exception& e) { + return HandleException("GENERAL ERROR WHEN FINDING TABLE FILES TO INDEX", e.what()); } } @@ -465,7 +1540,7 @@ MySQLMetaImpl::FilesByType(const std::string& table_id, const std::vector& mysqlpp::ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); if (connectionPtr == nullptr) { - return Status(DB_ERROR, "Failed to connect to database server"); + return Status(DB_ERROR, "Failed to connect to meta server(mysql)"); } std::string types; @@ -478,9 +1553,9 @@ MySQLMetaImpl::FilesByType(const std::string& table_id, const std::vector& mysqlpp::Query hasNonIndexFilesQuery = connectionPtr->query(); // since table_id is a unique column we just need to check whether it exists or not - hasNonIndexFilesQuery << "SELECT file_id, file_type FROM " << META_TABLEFILES << " " - << "WHERE table_id = " << mysqlpp::quote << table_id << " AND " - << "file_type in (" << types << ");"; + hasNonIndexFilesQuery << "SELECT file_id, file_type" + << " FROM " << META_TABLEFILES << " WHERE table_id = " << mysqlpp::quote << table_id + << " AND file_type in (" << types << ");"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::FilesByType: " << hasNonIndexFilesQuery.str(); @@ -535,854 +1610,6 @@ MySQLMetaImpl::FilesByType(const std::string& table_id, const std::vector& return Status::OK(); } -Status -MySQLMetaImpl::UpdateTableIndex(const std::string& table_id, const TableIndex& index) { - try { - server::MetricCollector metric; - - { - mysqlpp::ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); - - if (connectionPtr == nullptr) { - return Status(DB_ERROR, "Failed to connect to database server"); - } - - mysqlpp::Query updateTableIndexParamQuery = connectionPtr->query(); - updateTableIndexParamQuery << "SELECT id, state, dimension, created_on FROM " << META_TABLES << " " - << "WHERE table_id = " << mysqlpp::quote << table_id << " AND " - << "state <> " << std::to_string(TableSchema::TO_DELETE) << ";"; - - ENGINE_LOG_DEBUG << "MySQLMetaImpl::UpdateTableIndex: " << updateTableIndexParamQuery.str(); - - mysqlpp::StoreQueryResult res = updateTableIndexParamQuery.store(); - - if (res.num_rows() == 1) { - const mysqlpp::Row& resRow = res[0]; - - size_t id = resRow["id"]; - int32_t state = resRow["state"]; - uint16_t dimension = resRow["dimension"]; - int64_t created_on = resRow["created_on"]; - - updateTableIndexParamQuery << "UPDATE " << META_TABLES << " " - << "SET id = " << id << ", " - << "state = " << state << ", " - << "dimension = " << dimension << ", " - << "created_on = " << created_on << ", " - << "engine_type = " << index.engine_type_ << ", " - << "nlist = " << index.nlist_ << ", " - << "metric_type = " << index.metric_type_ << " " - << "WHERE table_id = " << mysqlpp::quote << table_id << ";"; - - ENGINE_LOG_DEBUG << "MySQLMetaImpl::UpdateTableIndex: " << updateTableIndexParamQuery.str(); - - if (!updateTableIndexParamQuery.exec()) { - return HandleException("QUERY ERROR WHEN UPDATING TABLE INDEX PARAM", - updateTableIndexParamQuery.error()); - } - } else { - return Status(DB_NOT_FOUND, "Table " + table_id + " not found"); - } - } // Scoped Connection - - ENGINE_LOG_DEBUG << "Successfully update table index, table id = " << table_id; - } catch (std::exception& e) { - return HandleException("GENERAL ERROR WHEN UPDATING TABLE INDEX PARAM", e.what()); - } - - return Status::OK(); -} - -Status -MySQLMetaImpl::UpdateTableFlag(const std::string& table_id, int64_t flag) { - try { - server::MetricCollector metric; - - { - mysqlpp::ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); - - if (connectionPtr == nullptr) { - return Status(DB_ERROR, "Failed to connect to database server"); - } - - mysqlpp::Query updateTableFlagQuery = connectionPtr->query(); - updateTableFlagQuery << "UPDATE " << META_TABLES << " " - << "SET flag = " << flag << " " - << "WHERE table_id = " << mysqlpp::quote << table_id << ";"; - - ENGINE_LOG_DEBUG << "MySQLMetaImpl::UpdateTableFlag: " << updateTableFlagQuery.str(); - - if (!updateTableFlagQuery.exec()) { - return HandleException("QUERY ERROR WHEN UPDATING TABLE FLAG", updateTableFlagQuery.error()); - } - } // Scoped Connection - - ENGINE_LOG_DEBUG << "Successfully update table flag, table id = " << table_id; - } catch (std::exception& e) { - return HandleException("GENERAL ERROR WHEN UPDATING TABLE FLAG", e.what()); - } - - return Status::OK(); -} - -Status -MySQLMetaImpl::DescribeTableIndex(const std::string& table_id, TableIndex& index) { - try { - server::MetricCollector metric; - - { - mysqlpp::ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); - - if (connectionPtr == nullptr) { - return Status(DB_ERROR, "Failed to connect to database server"); - } - - mysqlpp::Query describeTableIndexQuery = connectionPtr->query(); - describeTableIndexQuery << "SELECT engine_type, nlist, index_file_size, metric_type FROM " << META_TABLES - << " " - << "WHERE table_id = " << mysqlpp::quote << table_id << " AND " - << "state <> " << std::to_string(TableSchema::TO_DELETE) << ";"; - - ENGINE_LOG_DEBUG << "MySQLMetaImpl::DescribeTableIndex: " << describeTableIndexQuery.str(); - - mysqlpp::StoreQueryResult res = describeTableIndexQuery.store(); - - if (res.num_rows() == 1) { - const mysqlpp::Row& resRow = res[0]; - - index.engine_type_ = resRow["engine_type"]; - index.nlist_ = resRow["nlist"]; - index.metric_type_ = resRow["metric_type"]; - } else { - return Status(DB_NOT_FOUND, "Table " + table_id + " not found"); - } - } // Scoped Connection - } catch (std::exception& e) { - return HandleException("GENERAL ERROR WHEN UPDATING TABLE FLAG", e.what()); - } - - return Status::OK(); -} - -Status -MySQLMetaImpl::DropTableIndex(const std::string& table_id) { - try { - server::MetricCollector metric; - - { - mysqlpp::ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); - - if (connectionPtr == nullptr) { - return Status(DB_ERROR, "Failed to connect to database server"); - } - - mysqlpp::Query dropTableIndexQuery = connectionPtr->query(); - - // soft delete index files - dropTableIndexQuery << "UPDATE " << META_TABLEFILES << " " - << "SET file_type = " << std::to_string(TableFileSchema::TO_DELETE) << "," - << "updated_time = " << utils::GetMicroSecTimeStamp() << " " - << "WHERE table_id = " << mysqlpp::quote << table_id << " AND " - << "file_type = " << std::to_string(TableFileSchema::INDEX) << ";"; - - ENGINE_LOG_DEBUG << "MySQLMetaImpl::DropTableIndex: " << dropTableIndexQuery.str(); - - if (!dropTableIndexQuery.exec()) { - return HandleException("QUERY ERROR WHEN DROPPING TABLE INDEX", dropTableIndexQuery.error()); - } - - // set all backup file to raw - dropTableIndexQuery << "UPDATE " << META_TABLEFILES << " " - << "SET file_type = " << std::to_string(TableFileSchema::RAW) << "," - << "updated_time = " << utils::GetMicroSecTimeStamp() << " " - << "WHERE table_id = " << mysqlpp::quote << table_id << " AND " - << "file_type = " << std::to_string(TableFileSchema::BACKUP) << ";"; - - ENGINE_LOG_DEBUG << "MySQLMetaImpl::DropTableIndex: " << dropTableIndexQuery.str(); - - if (!dropTableIndexQuery.exec()) { - return HandleException("QUERY ERROR WHEN DROPPING TABLE INDEX", dropTableIndexQuery.error()); - } - - // set table index type to raw - dropTableIndexQuery << "UPDATE " << META_TABLES << " " - << "SET engine_type = " << std::to_string(DEFAULT_ENGINE_TYPE) << "," - << "nlist = " << std::to_string(DEFAULT_NLIST) << ", " - << "metric_type = " << std::to_string(DEFAULT_METRIC_TYPE) << " " - << "WHERE table_id = " << mysqlpp::quote << table_id << ";"; - - ENGINE_LOG_DEBUG << "MySQLMetaImpl::DropTableIndex: " << dropTableIndexQuery.str(); - - if (!dropTableIndexQuery.exec()) { - return HandleException("QUERY ERROR WHEN DROPPING TABLE INDEX", dropTableIndexQuery.error()); - } - } // Scoped Connection - - ENGINE_LOG_DEBUG << "Successfully drop table index, table id = " << table_id; - } catch (std::exception& e) { - return HandleException("GENERAL ERROR WHEN DROPPING TABLE INDEX", e.what()); - } - - return Status::OK(); -} - -Status -MySQLMetaImpl::DeleteTable(const std::string& table_id) { - try { - server::MetricCollector metric; - { - mysqlpp::ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); - - if (connectionPtr == nullptr) { - return Status(DB_ERROR, "Failed to connect to database server"); - } - - // soft delete table - mysqlpp::Query deleteTableQuery = connectionPtr->query(); - // - deleteTableQuery << "UPDATE " << META_TABLES << " " - << "SET state = " << std::to_string(TableSchema::TO_DELETE) << " " - << "WHERE table_id = " << mysqlpp::quote << table_id << ";"; - - ENGINE_LOG_DEBUG << "MySQLMetaImpl::DeleteTable: " << deleteTableQuery.str(); - - if (!deleteTableQuery.exec()) { - return HandleException("QUERY ERROR WHEN DELETING TABLE", deleteTableQuery.error()); - } - } // Scoped Connection - - if (mode_ == DBOptions::MODE::CLUSTER_WRITABLE) { - DeleteTableFiles(table_id); - } - - ENGINE_LOG_DEBUG << "Successfully delete table, table id = " << table_id; - } catch (std::exception& e) { - return HandleException("GENERAL ERROR WHEN DELETING TABLE", e.what()); - } - - return Status::OK(); -} - -Status -MySQLMetaImpl::DeleteTableFiles(const std::string& table_id) { - try { - server::MetricCollector metric; - { - mysqlpp::ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); - - if (connectionPtr == nullptr) { - return Status(DB_ERROR, "Failed to connect to database server"); - } - - // soft delete table files - mysqlpp::Query deleteTableFilesQuery = connectionPtr->query(); - // - deleteTableFilesQuery << "UPDATE " << META_TABLEFILES << " " - << "SET file_type = " << std::to_string(TableFileSchema::TO_DELETE) << ", " - << "updated_time = " << std::to_string(utils::GetMicroSecTimeStamp()) << " " - << "WHERE table_id = " << mysqlpp::quote << table_id << " AND " - << "file_type <> " << std::to_string(TableFileSchema::TO_DELETE) << ";"; - - ENGINE_LOG_DEBUG << "MySQLMetaImpl::DeleteTableFiles: " << deleteTableFilesQuery.str(); - - if (!deleteTableFilesQuery.exec()) { - return HandleException("QUERY ERROR WHEN DELETING TABLE FILES", deleteTableFilesQuery.error()); - } - } // Scoped Connection - - ENGINE_LOG_DEBUG << "Successfully delete table files, table id = " << table_id; - } catch (std::exception& e) { - return HandleException("GENERAL ERROR WHEN DELETING TABLE FILES", e.what()); - } - - return Status::OK(); -} - -Status -MySQLMetaImpl::DescribeTable(TableSchema& table_schema) { - try { - server::MetricCollector metric; - mysqlpp::StoreQueryResult res; - { - mysqlpp::ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); - - if (connectionPtr == nullptr) { - return Status(DB_ERROR, "Failed to connect to database server"); - } - - mysqlpp::Query describeTableQuery = connectionPtr->query(); - describeTableQuery - << "SELECT id, state, dimension, created_on, flag, index_file_size, engine_type, nlist, metric_type " - << " FROM " << META_TABLES << " " - << "WHERE table_id = " << mysqlpp::quote << table_schema.table_id_ << " " - << "AND state <> " << std::to_string(TableSchema::TO_DELETE) << ";"; - - ENGINE_LOG_DEBUG << "MySQLMetaImpl::DescribeTable: " << describeTableQuery.str(); - - res = describeTableQuery.store(); - } // Scoped Connection - - if (res.num_rows() == 1) { - const mysqlpp::Row& resRow = res[0]; - - table_schema.id_ = resRow["id"]; // implicit conversion - - table_schema.state_ = resRow["state"]; - - table_schema.dimension_ = resRow["dimension"]; - - table_schema.created_on_ = resRow["created_on"]; - - table_schema.flag_ = resRow["flag"]; - - table_schema.index_file_size_ = resRow["index_file_size"]; - - table_schema.engine_type_ = resRow["engine_type"]; - - table_schema.nlist_ = resRow["nlist"]; - - table_schema.metric_type_ = resRow["metric_type"]; - } else { - return Status(DB_NOT_FOUND, "Table " + table_schema.table_id_ + " not found"); - } - } catch (std::exception& e) { - return HandleException("GENERAL ERROR WHEN DESCRIBING TABLE", e.what()); - } - - return Status::OK(); -} - -Status -MySQLMetaImpl::HasTable(const std::string& table_id, bool& has_or_not) { - try { - server::MetricCollector metric; - mysqlpp::StoreQueryResult res; - { - mysqlpp::ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); - - if (connectionPtr == nullptr) { - return Status(DB_ERROR, "Failed to connect to database server"); - } - - mysqlpp::Query hasTableQuery = connectionPtr->query(); - // since table_id is a unique column we just need to check whether it exists or not - hasTableQuery << "SELECT EXISTS " - << "(SELECT 1 FROM " << META_TABLES << " " - << "WHERE table_id = " << mysqlpp::quote << table_id << " " - << "AND state <> " << std::to_string(TableSchema::TO_DELETE) << ") " - << "AS " << mysqlpp::quote << "check" - << ";"; - - ENGINE_LOG_DEBUG << "MySQLMetaImpl::HasTable: " << hasTableQuery.str(); - - res = hasTableQuery.store(); - } // Scoped Connection - - int check = res[0]["check"]; - has_or_not = (check == 1); - } catch (std::exception& e) { - return HandleException("GENERAL ERROR WHEN CHECKING IF TABLE EXISTS", e.what()); - } - - return Status::OK(); -} - -Status -MySQLMetaImpl::AllTables(std::vector& table_schema_array) { - try { - server::MetricCollector metric; - mysqlpp::StoreQueryResult res; - { - mysqlpp::ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); - - if (connectionPtr == nullptr) { - return Status(DB_ERROR, "Failed to connect to database server"); - } - - mysqlpp::Query allTablesQuery = connectionPtr->query(); - allTablesQuery << "SELECT id, table_id, dimension, engine_type, nlist, index_file_size, metric_type FROM " - << META_TABLES << " " - << "WHERE state <> " << std::to_string(TableSchema::TO_DELETE) << ";"; - - ENGINE_LOG_DEBUG << "MySQLMetaImpl::AllTables: " << allTablesQuery.str(); - - res = allTablesQuery.store(); - } // Scoped Connection - - for (auto& resRow : res) { - TableSchema table_schema; - - table_schema.id_ = resRow["id"]; // implicit conversion - - std::string table_id; - resRow["table_id"].to_string(table_id); - table_schema.table_id_ = table_id; - - table_schema.dimension_ = resRow["dimension"]; - - table_schema.index_file_size_ = resRow["index_file_size"]; - - table_schema.engine_type_ = resRow["engine_type"]; - - table_schema.nlist_ = resRow["nlist"]; - - table_schema.metric_type_ = resRow["metric_type"]; - - table_schema_array.emplace_back(table_schema); - } - } catch (std::exception& e) { - return HandleException("GENERAL ERROR WHEN DESCRIBING ALL TABLES", e.what()); - } - - return Status::OK(); -} - -Status -MySQLMetaImpl::CreateTableFile(TableFileSchema& file_schema) { - if (file_schema.date_ == EmptyDate) { - file_schema.date_ = utils::GetDate(); - } - TableSchema table_schema; - table_schema.table_id_ = file_schema.table_id_; - auto status = DescribeTable(table_schema); - if (!status.ok()) { - return status; - } - - try { - server::MetricCollector metric; - - NextFileId(file_schema.file_id_); - file_schema.dimension_ = table_schema.dimension_; - file_schema.file_size_ = 0; - file_schema.row_count_ = 0; - file_schema.created_on_ = utils::GetMicroSecTimeStamp(); - file_schema.updated_time_ = file_schema.created_on_; - file_schema.index_file_size_ = table_schema.index_file_size_; - file_schema.engine_type_ = table_schema.engine_type_; - file_schema.nlist_ = table_schema.nlist_; - file_schema.metric_type_ = table_schema.metric_type_; - - std::string id = "NULL"; // auto-increment - std::string table_id = file_schema.table_id_; - std::string engine_type = std::to_string(file_schema.engine_type_); - std::string file_id = file_schema.file_id_; - std::string file_type = std::to_string(file_schema.file_type_); - std::string file_size = std::to_string(file_schema.file_size_); - std::string row_count = std::to_string(file_schema.row_count_); - std::string updated_time = std::to_string(file_schema.updated_time_); - std::string created_on = std::to_string(file_schema.created_on_); - std::string date = std::to_string(file_schema.date_); - - { - mysqlpp::ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); - - if (connectionPtr == nullptr) { - return Status(DB_ERROR, "Failed to connect to database server"); - } - - mysqlpp::Query createTableFileQuery = connectionPtr->query(); - - createTableFileQuery << "INSERT INTO " << META_TABLEFILES << " " - << "VALUES(" << id << ", " << mysqlpp::quote << table_id << ", " << engine_type << ", " - << mysqlpp::quote << file_id << ", " << file_type << ", " << file_size << ", " - << row_count << ", " << updated_time << ", " << created_on << ", " << date << ");"; - - ENGINE_LOG_DEBUG << "MySQLMetaImpl::CreateTableFile: " << createTableFileQuery.str(); - - if (mysqlpp::SimpleResult res = createTableFileQuery.execute()) { - file_schema.id_ = res.insert_id(); // Might need to use SELECT LAST_INSERT_ID()? - - // Consume all results to avoid "Commands out of sync" error - } else { - return HandleException("QUERY ERROR WHEN CREATING TABLE FILE", createTableFileQuery.error()); - } - } // Scoped Connection - - ENGINE_LOG_DEBUG << "Successfully create table file, file id = " << file_schema.file_id_; - return utils::CreateTableFilePath(options_, file_schema); - } catch (std::exception& e) { - return HandleException("GENERAL ERROR WHEN CREATING TABLE FILE", e.what()); - } -} - -Status -MySQLMetaImpl::FilesToIndex(TableFilesSchema& files) { - files.clear(); - - try { - server::MetricCollector metric; - mysqlpp::StoreQueryResult res; - { - mysqlpp::ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); - - if (connectionPtr == nullptr) { - return Status(DB_ERROR, "Failed to connect to database server"); - } - - mysqlpp::Query filesToIndexQuery = connectionPtr->query(); - filesToIndexQuery - << "SELECT id, table_id, engine_type, file_id, file_type, file_size, row_count, date, created_on FROM " - << META_TABLEFILES << " " - << "WHERE file_type = " << std::to_string(TableFileSchema::TO_INDEX) << ";"; - - ENGINE_LOG_DEBUG << "MySQLMetaImpl::FilesToIndex: " << filesToIndexQuery.str(); - - res = filesToIndexQuery.store(); - } // Scoped Connection - - Status ret; - std::map groups; - TableFileSchema table_file; - for (auto& resRow : res) { - table_file.id_ = resRow["id"]; // implicit conversion - - std::string table_id; - resRow["table_id"].to_string(table_id); - table_file.table_id_ = table_id; - - table_file.engine_type_ = resRow["engine_type"]; - - std::string file_id; - resRow["file_id"].to_string(file_id); - table_file.file_id_ = file_id; - - table_file.file_type_ = resRow["file_type"]; - - table_file.file_size_ = resRow["file_size"]; - - table_file.row_count_ = resRow["row_count"]; - - table_file.date_ = resRow["date"]; - - table_file.created_on_ = resRow["created_on"]; - - auto groupItr = groups.find(table_file.table_id_); - if (groupItr == groups.end()) { - TableSchema table_schema; - table_schema.table_id_ = table_file.table_id_; - auto status = DescribeTable(table_schema); - if (!status.ok()) { - return status; - } - groups[table_file.table_id_] = table_schema; - } - table_file.dimension_ = groups[table_file.table_id_].dimension_; - table_file.index_file_size_ = groups[table_file.table_id_].index_file_size_; - table_file.nlist_ = groups[table_file.table_id_].nlist_; - table_file.metric_type_ = groups[table_file.table_id_].metric_type_; - - auto status = utils::GetTableFilePath(options_, table_file); - if (!status.ok()) { - ret = status; - } - - files.push_back(table_file); - } - - if (res.size() > 0) { - ENGINE_LOG_DEBUG << "Collect " << res.size() << " to-index files"; - } - return ret; - } catch (std::exception& e) { - return HandleException("GENERAL ERROR WHEN FINDING TABLE FILES TO INDEX", e.what()); - } -} - -Status -MySQLMetaImpl::FilesToSearch(const std::string& table_id, const std::vector& ids, const DatesT& dates, - DatePartionedTableFilesSchema& files) { - files.clear(); - - try { - server::MetricCollector metric; - mysqlpp::StoreQueryResult res; - { - mysqlpp::ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); - - if (connectionPtr == nullptr) { - return Status(DB_ERROR, "Failed to connect to database server"); - } - - mysqlpp::Query filesToSearchQuery = connectionPtr->query(); - filesToSearchQuery - << "SELECT id, table_id, engine_type, file_id, file_type, file_size, row_count, date FROM " - << META_TABLEFILES << " " - << "WHERE table_id = " << mysqlpp::quote << table_id; - - if (!dates.empty()) { - std::stringstream partitionListSS; - for (auto& date : dates) { - partitionListSS << std::to_string(date) << ", "; - } - std::string partitionListStr = partitionListSS.str(); - - partitionListStr = partitionListStr.substr(0, partitionListStr.size() - 2); // remove the last ", " - filesToSearchQuery << " AND " - << "date IN (" << partitionListStr << ")"; - } - - if (!ids.empty()) { - std::stringstream idSS; - for (auto& id : ids) { - idSS << "id = " << std::to_string(id) << " OR "; - } - std::string idStr = idSS.str(); - idStr = idStr.substr(0, idStr.size() - 4); // remove the last " OR " - - filesToSearchQuery << " AND " - << "(" << idStr << ")"; - } - // End - filesToSearchQuery << " AND " - << "(file_type = " << std::to_string(TableFileSchema::RAW) << " OR " - << "file_type = " << std::to_string(TableFileSchema::TO_INDEX) << " OR " - << "file_type = " << std::to_string(TableFileSchema::INDEX) << ");"; - - ENGINE_LOG_DEBUG << "MySQLMetaImpl::FilesToSearch: " << filesToSearchQuery.str(); - - res = filesToSearchQuery.store(); - } // Scoped Connection - - TableSchema table_schema; - table_schema.table_id_ = table_id; - auto status = DescribeTable(table_schema); - if (!status.ok()) { - return status; - } - - Status ret; - TableFileSchema table_file; - for (auto& resRow : res) { - table_file.id_ = resRow["id"]; // implicit conversion - - std::string table_id_str; - resRow["table_id"].to_string(table_id_str); - table_file.table_id_ = table_id_str; - - table_file.index_file_size_ = table_schema.index_file_size_; - - table_file.engine_type_ = resRow["engine_type"]; - - table_file.nlist_ = table_schema.nlist_; - - table_file.metric_type_ = table_schema.metric_type_; - - std::string file_id; - resRow["file_id"].to_string(file_id); - table_file.file_id_ = file_id; - - table_file.file_type_ = resRow["file_type"]; - - table_file.file_size_ = resRow["file_size"]; - - table_file.row_count_ = resRow["row_count"]; - - table_file.date_ = resRow["date"]; - - table_file.dimension_ = table_schema.dimension_; - - auto status = utils::GetTableFilePath(options_, table_file); - if (!status.ok()) { - ret = status; - } - - auto dateItr = files.find(table_file.date_); - if (dateItr == files.end()) { - files[table_file.date_] = TableFilesSchema(); - } - - files[table_file.date_].push_back(table_file); - } - - if (res.size() > 0) { - ENGINE_LOG_DEBUG << "Collect " << res.size() << " to-search files"; - } - return ret; - } catch (std::exception& e) { - return HandleException("GENERAL ERROR WHEN FINDING TABLE FILES TO SEARCH", e.what()); - } -} - -Status -MySQLMetaImpl::FilesToMerge(const std::string& table_id, DatePartionedTableFilesSchema& files) { - files.clear(); - - try { - server::MetricCollector metric; - - // check table existence - TableSchema table_schema; - table_schema.table_id_ = table_id; - auto status = DescribeTable(table_schema); - if (!status.ok()) { - return status; - } - - mysqlpp::StoreQueryResult res; - { - mysqlpp::ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); - - if (connectionPtr == nullptr) { - return Status(DB_ERROR, "Failed to connect to database server"); - } - - mysqlpp::Query filesToMergeQuery = connectionPtr->query(); - filesToMergeQuery - << "SELECT id, table_id, file_id, file_type, file_size, row_count, date, engine_type, created_on FROM " - << META_TABLEFILES << " " - << "WHERE table_id = " << mysqlpp::quote << table_id << " AND " - << "file_type = " << std::to_string(TableFileSchema::RAW) << " " - << "ORDER BY row_count DESC" - << ";"; - - ENGINE_LOG_DEBUG << "MySQLMetaImpl::FilesToMerge: " << filesToMergeQuery.str(); - - res = filesToMergeQuery.store(); - } // Scoped Connection - - Status ret; - for (auto& resRow : res) { - TableFileSchema table_file; - table_file.file_size_ = resRow["file_size"]; - if (table_file.file_size_ >= table_schema.index_file_size_) { - continue; // skip large file - } - - table_file.id_ = resRow["id"]; // implicit conversion - - std::string table_id_str; - resRow["table_id"].to_string(table_id_str); - table_file.table_id_ = table_id_str; - - std::string file_id; - resRow["file_id"].to_string(file_id); - table_file.file_id_ = file_id; - - table_file.file_type_ = resRow["file_type"]; - - table_file.row_count_ = resRow["row_count"]; - - table_file.date_ = resRow["date"]; - - table_file.index_file_size_ = table_schema.index_file_size_; - - table_file.engine_type_ = resRow["engine_type"]; - - table_file.nlist_ = table_schema.nlist_; - - table_file.metric_type_ = table_schema.metric_type_; - - table_file.created_on_ = resRow["created_on"]; - - table_file.dimension_ = table_schema.dimension_; - - auto status = utils::GetTableFilePath(options_, table_file); - if (!status.ok()) { - ret = status; - } - - auto dateItr = files.find(table_file.date_); - if (dateItr == files.end()) { - files[table_file.date_] = TableFilesSchema(); - } - - files[table_file.date_].push_back(table_file); - } - - if (res.size() > 0) { - ENGINE_LOG_DEBUG << "Collect " << res.size() << " to-merge files"; - } - return ret; - } catch (std::exception& e) { - return HandleException("GENERAL ERROR WHEN FINDING TABLE FILES TO MERGE", e.what()); - } -} - -Status -MySQLMetaImpl::GetTableFiles(const std::string& table_id, const std::vector& ids, - TableFilesSchema& table_files) { - if (ids.empty()) { - return Status::OK(); - } - - std::stringstream idSS; - for (auto& id : ids) { - idSS << "id = " << std::to_string(id) << " OR "; - } - std::string idStr = idSS.str(); - idStr = idStr.substr(0, idStr.size() - 4); // remove the last " OR " - - try { - mysqlpp::StoreQueryResult res; - { - mysqlpp::ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); - - if (connectionPtr == nullptr) { - return Status(DB_ERROR, "Failed to connect to database server"); - } - - mysqlpp::Query getTableFileQuery = connectionPtr->query(); - getTableFileQuery - << "SELECT id, engine_type, file_id, file_type, file_size, row_count, date, created_on FROM " - << META_TABLEFILES << " " - << "WHERE table_id = " << mysqlpp::quote << table_id << " AND " - << "(" << idStr << ") AND " - << "file_type <> " << std::to_string(TableFileSchema::TO_DELETE) << ";"; - - ENGINE_LOG_DEBUG << "MySQLMetaImpl::GetTableFiles: " << getTableFileQuery.str(); - - res = getTableFileQuery.store(); - } // Scoped Connection - - TableSchema table_schema; - table_schema.table_id_ = table_id; - DescribeTable(table_schema); - - Status ret; - for (auto& resRow : res) { - TableFileSchema file_schema; - - file_schema.id_ = resRow["id"]; - - file_schema.table_id_ = table_id; - - file_schema.index_file_size_ = table_schema.index_file_size_; - - file_schema.engine_type_ = resRow["engine_type"]; - - file_schema.nlist_ = table_schema.nlist_; - - file_schema.metric_type_ = table_schema.metric_type_; - - std::string file_id; - resRow["file_id"].to_string(file_id); - file_schema.file_id_ = file_id; - - file_schema.file_type_ = resRow["file_type"]; - - file_schema.file_size_ = resRow["file_size"]; - - file_schema.row_count_ = resRow["row_count"]; - - file_schema.date_ = resRow["date"]; - - file_schema.created_on_ = resRow["created_on"]; - - file_schema.dimension_ = table_schema.dimension_; - - utils::GetTableFilePath(options_, file_schema); - - table_files.emplace_back(file_schema); - } - - ENGINE_LOG_DEBUG << "Get table files by id"; - return ret; - } catch (std::exception& e) { - return HandleException("GENERAL ERROR WHEN RETRIEVING TABLE FILES", e.what()); - } -} - // TODO(myh): Support swap to cloud storage Status MySQLMetaImpl::Archive() { @@ -1402,14 +1629,14 @@ MySQLMetaImpl::Archive() { mysqlpp::ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); if (connectionPtr == nullptr) { - return Status(DB_ERROR, "Failed to connect to database server"); + return Status(DB_ERROR, "Failed to connect to meta server(mysql)"); } mysqlpp::Query archiveQuery = connectionPtr->query(); - archiveQuery << "UPDATE " << META_TABLEFILES << " " - << "SET file_type = " << std::to_string(TableFileSchema::TO_DELETE) << " " - << "WHERE created_on < " << std::to_string(now - usecs) << " AND " - << "file_type <> " << std::to_string(TableFileSchema::TO_DELETE) << ";"; + archiveQuery << "UPDATE " << META_TABLEFILES + << " SET file_type = " << std::to_string(TableFileSchema::TO_DELETE) + << " WHERE created_on < " << std::to_string(now - usecs) << " AND file_type <> " + << std::to_string(TableFileSchema::TO_DELETE) << ";"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::Archive: " << archiveQuery.str(); @@ -1446,12 +1673,13 @@ MySQLMetaImpl::Size(uint64_t& result) { mysqlpp::ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); if (connectionPtr == nullptr) { - return Status(DB_ERROR, "Failed to connect to database server"); + return Status(DB_ERROR, "Failed to connect to meta server(mysql)"); } mysqlpp::Query getSizeQuery = connectionPtr->query(); - getSizeQuery << "SELECT IFNULL(SUM(file_size),0) AS sum FROM " << META_TABLEFILES << " " - << "WHERE file_type <> " << std::to_string(TableFileSchema::TO_DELETE) << ";"; + getSizeQuery << "SELECT IFNULL(SUM(file_size),0) AS sum" + << " FROM " << META_TABLEFILES << " WHERE file_type <> " + << std::to_string(TableFileSchema::TO_DELETE) << ";"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::Size: " << getSizeQuery.str(); @@ -1470,434 +1698,20 @@ MySQLMetaImpl::Size(uint64_t& result) { return Status::OK(); } -Status -MySQLMetaImpl::DiscardFiles(int64_t to_discard_size) { - if (to_discard_size <= 0) { - return Status::OK(); - } - ENGINE_LOG_DEBUG << "About to discard size=" << to_discard_size; - - try { - server::MetricCollector metric; - bool status; - { - mysqlpp::ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); - - if (connectionPtr == nullptr) { - return Status(DB_ERROR, "Failed to connect to database server"); - } - - mysqlpp::Query discardFilesQuery = connectionPtr->query(); - discardFilesQuery << "SELECT id, file_size FROM " << META_TABLEFILES << " " - << "WHERE file_type <> " << std::to_string(TableFileSchema::TO_DELETE) << " " - << "ORDER BY id ASC " - << "LIMIT 10;"; - - ENGINE_LOG_DEBUG << "MySQLMetaImpl::DiscardFiles: " << discardFilesQuery.str(); - - mysqlpp::StoreQueryResult res = discardFilesQuery.store(); - if (res.num_rows() == 0) { - return Status::OK(); - } - - TableFileSchema table_file; - std::stringstream idsToDiscardSS; - for (auto& resRow : res) { - if (to_discard_size <= 0) { - break; - } - table_file.id_ = resRow["id"]; - table_file.file_size_ = resRow["file_size"]; - idsToDiscardSS << "id = " << std::to_string(table_file.id_) << " OR "; - ENGINE_LOG_DEBUG << "Discard table_file.id=" << table_file.file_id_ - << " table_file.size=" << table_file.file_size_; - to_discard_size -= table_file.file_size_; - } - - std::string idsToDiscardStr = idsToDiscardSS.str(); - idsToDiscardStr = idsToDiscardStr.substr(0, idsToDiscardStr.size() - 4); // remove the last " OR " - - discardFilesQuery << "UPDATE " << META_TABLEFILES << " " - << "SET file_type = " << std::to_string(TableFileSchema::TO_DELETE) << ", " - << "updated_time = " << std::to_string(utils::GetMicroSecTimeStamp()) << " " - << "WHERE " << idsToDiscardStr << ";"; - - ENGINE_LOG_DEBUG << "MySQLMetaImpl::DiscardFiles: " << discardFilesQuery.str(); - - status = discardFilesQuery.exec(); - if (!status) { - return HandleException("QUERY ERROR WHEN DISCARDING FILES", discardFilesQuery.error()); - } - } // Scoped Connection - - return DiscardFiles(to_discard_size); - } catch (std::exception& e) { - return HandleException("GENERAL ERROR WHEN DISCARDING FILES", e.what()); - } -} - -// ZR: this function assumes all fields in file_schema have value -Status -MySQLMetaImpl::UpdateTableFile(TableFileSchema& file_schema) { - file_schema.updated_time_ = utils::GetMicroSecTimeStamp(); - - try { - server::MetricCollector metric; - { - mysqlpp::ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); - - if (connectionPtr == nullptr) { - return Status(DB_ERROR, "Failed to connect to database server"); - } - - mysqlpp::Query updateTableFileQuery = connectionPtr->query(); - - // if the table has been deleted, just mark the table file as TO_DELETE - // clean thread will delete the file later - updateTableFileQuery << "SELECT state FROM " << META_TABLES << " " - << "WHERE table_id = " << mysqlpp::quote << file_schema.table_id_ << ";"; - - ENGINE_LOG_DEBUG << "MySQLMetaImpl::UpdateTableFile: " << updateTableFileQuery.str(); - - mysqlpp::StoreQueryResult res = updateTableFileQuery.store(); - - if (res.num_rows() == 1) { - int state = res[0]["state"]; - if (state == TableSchema::TO_DELETE) { - file_schema.file_type_ = TableFileSchema::TO_DELETE; - } - } else { - file_schema.file_type_ = TableFileSchema::TO_DELETE; - } - - std::string id = std::to_string(file_schema.id_); - std::string table_id = file_schema.table_id_; - std::string engine_type = std::to_string(file_schema.engine_type_); - std::string file_id = file_schema.file_id_; - std::string file_type = std::to_string(file_schema.file_type_); - std::string file_size = std::to_string(file_schema.file_size_); - std::string row_count = std::to_string(file_schema.row_count_); - std::string updated_time = std::to_string(file_schema.updated_time_); - std::string created_on = std::to_string(file_schema.created_on_); - std::string date = std::to_string(file_schema.date_); - - updateTableFileQuery << "UPDATE " << META_TABLEFILES << " " - << "SET table_id = " << mysqlpp::quote << table_id << ", " - << "engine_type = " << engine_type << ", " - << "file_id = " << mysqlpp::quote << file_id << ", " - << "file_type = " << file_type << ", " - << "file_size = " << file_size << ", " - << "row_count = " << row_count << ", " - << "updated_time = " << updated_time << ", " - << "created_on = " << created_on << ", " - << "date = " << date << " " - << "WHERE id = " << id << ";"; - - ENGINE_LOG_DEBUG << "MySQLMetaImpl::UpdateTableFile: " << updateTableFileQuery.str(); - - if (!updateTableFileQuery.exec()) { - ENGINE_LOG_DEBUG << "table_id= " << file_schema.table_id_ << " file_id=" << file_schema.file_id_; - return HandleException("QUERY ERROR WHEN UPDATING TABLE FILE", updateTableFileQuery.error()); - } - } // Scoped Connection - - ENGINE_LOG_DEBUG << "Update single table file, file id = " << file_schema.file_id_; - } catch (std::exception& e) { - return HandleException("GENERAL ERROR WHEN UPDATING TABLE FILE", e.what()); - } - - return Status::OK(); -} - -Status -MySQLMetaImpl::UpdateTableFilesToIndex(const std::string& table_id) { - try { - mysqlpp::ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); - - if (connectionPtr == nullptr) { - return Status(DB_ERROR, "Failed to connect to database server"); - } - - mysqlpp::Query updateTableFilesToIndexQuery = connectionPtr->query(); - - updateTableFilesToIndexQuery << "UPDATE " << META_TABLEFILES << " " - << "SET file_type = " << std::to_string(TableFileSchema::TO_INDEX) << " " - << "WHERE table_id = " << mysqlpp::quote << table_id << " AND " - << "file_type = " << std::to_string(TableFileSchema::RAW) << ";"; - - ENGINE_LOG_DEBUG << "MySQLMetaImpl::UpdateTableFilesToIndex: " << updateTableFilesToIndexQuery.str(); - - if (!updateTableFilesToIndexQuery.exec()) { - return HandleException("QUERY ERROR WHEN UPDATING TABLE FILE TO INDEX", - updateTableFilesToIndexQuery.error()); - } - - ENGINE_LOG_DEBUG << "Update files to to_index, table id = " << table_id; - } catch (std::exception& e) { - return HandleException("GENERAL ERROR WHEN UPDATING TABLE FILES TO INDEX", e.what()); - } - - return Status::OK(); -} - -Status -MySQLMetaImpl::UpdateTableFiles(TableFilesSchema& files) { - try { - server::MetricCollector metric; - { - mysqlpp::ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); - - if (connectionPtr == nullptr) { - return Status(DB_ERROR, "Failed to connect to database server"); - } - - mysqlpp::Query updateTableFilesQuery = connectionPtr->query(); - - std::map has_tables; - for (auto& file_schema : files) { - if (has_tables.find(file_schema.table_id_) != has_tables.end()) { - continue; - } - - updateTableFilesQuery << "SELECT EXISTS " - << "(SELECT 1 FROM " << META_TABLES << " " - << "WHERE table_id = " << mysqlpp::quote << file_schema.table_id_ << " " - << "AND state <> " << std::to_string(TableSchema::TO_DELETE) << ") " - << "AS " << mysqlpp::quote << "check" - << ";"; - - ENGINE_LOG_DEBUG << "MySQLMetaImpl::UpdateTableFiles: " << updateTableFilesQuery.str(); - - mysqlpp::StoreQueryResult res = updateTableFilesQuery.store(); - - int check = res[0]["check"]; - has_tables[file_schema.table_id_] = (check == 1); - } - - for (auto& file_schema : files) { - if (!has_tables[file_schema.table_id_]) { - file_schema.file_type_ = TableFileSchema::TO_DELETE; - } - file_schema.updated_time_ = utils::GetMicroSecTimeStamp(); - - std::string id = std::to_string(file_schema.id_); - std::string table_id = file_schema.table_id_; - std::string engine_type = std::to_string(file_schema.engine_type_); - std::string file_id = file_schema.file_id_; - std::string file_type = std::to_string(file_schema.file_type_); - std::string file_size = std::to_string(file_schema.file_size_); - std::string row_count = std::to_string(file_schema.row_count_); - std::string updated_time = std::to_string(file_schema.updated_time_); - std::string created_on = std::to_string(file_schema.created_on_); - std::string date = std::to_string(file_schema.date_); - - updateTableFilesQuery << "UPDATE " << META_TABLEFILES << " " - << "SET table_id = " << mysqlpp::quote << table_id << ", " - << "engine_type = " << engine_type << ", " - << "file_id = " << mysqlpp::quote << file_id << ", " - << "file_type = " << file_type << ", " - << "file_size = " << file_size << ", " - << "row_count = " << row_count << ", " - << "updated_time = " << updated_time << ", " - << "created_on = " << created_on << ", " - << "date = " << date << " " - << "WHERE id = " << id << ";"; - - ENGINE_LOG_DEBUG << "MySQLMetaImpl::UpdateTableFiles: " << updateTableFilesQuery.str(); - - if (!updateTableFilesQuery.exec()) { - return HandleException("QUERY ERROR WHEN UPDATING TABLE FILES", updateTableFilesQuery.error()); - } - } - } // Scoped Connection - - ENGINE_LOG_DEBUG << "Update " << files.size() << " table files"; - } catch (std::exception& e) { - return HandleException("GENERAL ERROR WHEN UPDATING TABLE FILES", e.what()); - } - - return Status::OK(); -} - -Status -MySQLMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) { - auto now = utils::GetMicroSecTimeStamp(); - std::set table_ids; - - // remove to_delete files - try { - server::MetricCollector metric; - - { - mysqlpp::ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); - - if (connectionPtr == nullptr) { - return Status(DB_ERROR, "Failed to connect to database server"); - } - - mysqlpp::Query cleanUpFilesWithTTLQuery = connectionPtr->query(); - cleanUpFilesWithTTLQuery << "SELECT id, table_id, file_id, date FROM " << META_TABLEFILES << " " - << "WHERE file_type = " << std::to_string(TableFileSchema::TO_DELETE) << " AND " - << "updated_time < " << std::to_string(now - seconds * US_PS) << ";"; - - ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUpFilesWithTTL: " << cleanUpFilesWithTTLQuery.str(); - - mysqlpp::StoreQueryResult res = cleanUpFilesWithTTLQuery.store(); - - TableFileSchema table_file; - std::vector idsToDelete; - - for (auto& resRow : res) { - table_file.id_ = resRow["id"]; // implicit conversion - - std::string table_id; - resRow["table_id"].to_string(table_id); - table_file.table_id_ = table_id; - - std::string file_id; - resRow["file_id"].to_string(file_id); - table_file.file_id_ = file_id; - - table_file.date_ = resRow["date"]; - - utils::DeleteTableFilePath(options_, table_file); - - ENGINE_LOG_DEBUG << "Removing file id:" << table_file.id_ << " location:" << table_file.location_; - - idsToDelete.emplace_back(std::to_string(table_file.id_)); - - table_ids.insert(table_file.table_id_); - } - - if (!idsToDelete.empty()) { - std::stringstream idsToDeleteSS; - for (auto& id : idsToDelete) { - idsToDeleteSS << "id = " << id << " OR "; - } - - std::string idsToDeleteStr = idsToDeleteSS.str(); - idsToDeleteStr = idsToDeleteStr.substr(0, idsToDeleteStr.size() - 4); // remove the last " OR " - cleanUpFilesWithTTLQuery << "DELETE FROM " << META_TABLEFILES << " " - << "WHERE " << idsToDeleteStr << ";"; - - ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUpFilesWithTTL: " << cleanUpFilesWithTTLQuery.str(); - - if (!cleanUpFilesWithTTLQuery.exec()) { - return HandleException("QUERY ERROR WHEN CLEANING UP FILES WITH TTL", - cleanUpFilesWithTTLQuery.error()); - } - } - - if (res.size() > 0) { - ENGINE_LOG_DEBUG << "Clean " << res.size() << " files deleted in " << seconds << " seconds"; - } - } // Scoped Connection - } catch (std::exception& e) { - return HandleException("GENERAL ERROR WHEN CLEANING UP FILES WITH TTL", e.what()); - } - - // remove to_delete tables - try { - server::MetricCollector metric; - - { - mysqlpp::ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); - - if (connectionPtr == nullptr) { - return Status(DB_ERROR, "Failed to connect to database server"); - } - - mysqlpp::Query cleanUpFilesWithTTLQuery = connectionPtr->query(); - cleanUpFilesWithTTLQuery << "SELECT id, table_id FROM " << META_TABLES << " " - << "WHERE state = " << std::to_string(TableSchema::TO_DELETE) << ";"; - - ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUpFilesWithTTL: " << cleanUpFilesWithTTLQuery.str(); - - mysqlpp::StoreQueryResult res = cleanUpFilesWithTTLQuery.store(); - - if (!res.empty()) { - std::stringstream idsToDeleteSS; - for (auto& resRow : res) { - size_t id = resRow["id"]; - std::string table_id; - resRow["table_id"].to_string(table_id); - - utils::DeleteTablePath(options_, table_id, false); // only delete empty folder - - idsToDeleteSS << "id = " << std::to_string(id) << " OR "; - } - std::string idsToDeleteStr = idsToDeleteSS.str(); - idsToDeleteStr = idsToDeleteStr.substr(0, idsToDeleteStr.size() - 4); // remove the last " OR " - cleanUpFilesWithTTLQuery << "DELETE FROM " << META_TABLES << " " - << "WHERE " << idsToDeleteStr << ";"; - - ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUpFilesWithTTL: " << cleanUpFilesWithTTLQuery.str(); - - if (!cleanUpFilesWithTTLQuery.exec()) { - return HandleException("QUERY ERROR WHEN CLEANING UP TABLES WITH TTL", - cleanUpFilesWithTTLQuery.error()); - } - } - - if (res.size() > 0) { - ENGINE_LOG_DEBUG << "Remove " << res.size() << " tables from meta"; - } - } // Scoped Connection - } catch (std::exception& e) { - return HandleException("GENERAL ERROR WHEN CLEANING UP TABLES WITH TTL", e.what()); - } - - // remove deleted table folder - // don't remove table folder until all its files has been deleted - try { - server::MetricCollector metric; - - { - mysqlpp::ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); - - if (connectionPtr == nullptr) { - return Status(DB_ERROR, "Failed to connect to database server"); - } - - for (auto& table_id : table_ids) { - mysqlpp::Query cleanUpFilesWithTTLQuery = connectionPtr->query(); - cleanUpFilesWithTTLQuery << "SELECT file_id FROM " << META_TABLEFILES << " " - << "WHERE table_id = " << mysqlpp::quote << table_id << ";"; - - ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUpFilesWithTTL: " << cleanUpFilesWithTTLQuery.str(); - - mysqlpp::StoreQueryResult res = cleanUpFilesWithTTLQuery.store(); - - if (res.empty()) { - utils::DeleteTablePath(options_, table_id); - } - } - - if (table_ids.size() > 0) { - ENGINE_LOG_DEBUG << "Remove " << table_ids.size() << " tables folder"; - } - } - } catch (std::exception& e) { - return HandleException("GENERAL ERROR WHEN CLEANING UP TABLES WITH TTL", e.what()); - } - - return Status::OK(); -} - Status MySQLMetaImpl::CleanUp() { try { mysqlpp::ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); if (connectionPtr == nullptr) { - return Status(DB_ERROR, "Failed to connect to database server"); + return Status(DB_ERROR, "Failed to connect to meta server(mysql)"); } mysqlpp::Query cleanUpQuery = connectionPtr->query(); - cleanUpQuery << "SELECT table_name " - << "FROM information_schema.tables " - << "WHERE table_schema = " << mysqlpp::quote << mysql_connection_pool_->getDB() << " " - << "AND table_name = " << mysqlpp::quote << META_TABLEFILES << ";"; + cleanUpQuery << "SELECT table_name" + << " FROM information_schema.tables" + << " WHERE table_schema = " << mysqlpp::quote << mysql_connection_pool_->getDB() + << " AND table_name = " << mysqlpp::quote << META_TABLEFILES << ";"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUp: " << cleanUpQuery.str(); @@ -1926,6 +1740,165 @@ MySQLMetaImpl::CleanUp() { return Status::OK(); } +Status +MySQLMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) { + auto now = utils::GetMicroSecTimeStamp(); + std::set table_ids; + + // remove to_delete files + try { + server::MetricCollector metric; + + { + mysqlpp::ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); + + if (connectionPtr == nullptr) { + return Status(DB_ERROR, "Failed to connect to meta server(mysql)"); + } + + mysqlpp::Query cleanUpFilesWithTTLQuery = connectionPtr->query(); + cleanUpFilesWithTTLQuery << "SELECT id, table_id, file_id, date" + << " FROM " << META_TABLEFILES + << " WHERE file_type = " << std::to_string(TableFileSchema::TO_DELETE) + << " AND updated_time < " << std::to_string(now - seconds * US_PS) << ";"; + + ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUpFilesWithTTL: " << cleanUpFilesWithTTLQuery.str(); + + mysqlpp::StoreQueryResult res = cleanUpFilesWithTTLQuery.store(); + + TableFileSchema table_file; + std::vector idsToDelete; + + for (auto& resRow : res) { + table_file.id_ = resRow["id"]; // implicit conversion + resRow["table_id"].to_string(table_file.table_id_); + resRow["file_id"].to_string(table_file.file_id_); + table_file.date_ = resRow["date"]; + + utils::DeleteTableFilePath(options_, table_file); + + ENGINE_LOG_DEBUG << "Removing file id:" << table_file.id_ << " location:" << table_file.location_; + + idsToDelete.emplace_back(std::to_string(table_file.id_)); + table_ids.insert(table_file.table_id_); + } + + if (!idsToDelete.empty()) { + std::stringstream idsToDeleteSS; + for (auto& id : idsToDelete) { + idsToDeleteSS << "id = " << id << " OR "; + } + + std::string idsToDeleteStr = idsToDeleteSS.str(); + idsToDeleteStr = idsToDeleteStr.substr(0, idsToDeleteStr.size() - 4); // remove the last " OR " + cleanUpFilesWithTTLQuery << "DELETE FROM " << META_TABLEFILES << " WHERE " << idsToDeleteStr << ";"; + + ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUpFilesWithTTL: " << cleanUpFilesWithTTLQuery.str(); + + if (!cleanUpFilesWithTTLQuery.exec()) { + return HandleException("QUERY ERROR WHEN CLEANING UP FILES WITH TTL", + cleanUpFilesWithTTLQuery.error()); + } + } + + if (res.size() > 0) { + ENGINE_LOG_DEBUG << "Clean " << res.size() << " files deleted in " << seconds << " seconds"; + } + } // Scoped Connection + } catch (std::exception& e) { + return HandleException("GENERAL ERROR WHEN CLEANING UP FILES WITH TTL", e.what()); + } + + // remove to_delete tables + try { + server::MetricCollector metric; + + { + mysqlpp::ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); + + if (connectionPtr == nullptr) { + return Status(DB_ERROR, "Failed to connect to meta server(mysql)"); + } + + mysqlpp::Query cleanUpFilesWithTTLQuery = connectionPtr->query(); + cleanUpFilesWithTTLQuery << "SELECT id, table_id" + << " FROM " << META_TABLES + << " WHERE state = " << std::to_string(TableSchema::TO_DELETE) << ";"; + + ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUpFilesWithTTL: " << cleanUpFilesWithTTLQuery.str(); + + mysqlpp::StoreQueryResult res = cleanUpFilesWithTTLQuery.store(); + + int64_t remove_tables = 0; + if (!res.empty()) { + std::stringstream idsToDeleteSS; + for (auto& resRow : res) { + size_t id = resRow["id"]; + std::string table_id; + resRow["table_id"].to_string(table_id); + + utils::DeleteTablePath(options_, table_id, false); // only delete empty folder + remove_tables++; + idsToDeleteSS << "id = " << std::to_string(id) << " OR "; + } + std::string idsToDeleteStr = idsToDeleteSS.str(); + idsToDeleteStr = idsToDeleteStr.substr(0, idsToDeleteStr.size() - 4); // remove the last " OR " + cleanUpFilesWithTTLQuery << "DELETE FROM " << META_TABLES << " WHERE " << idsToDeleteStr << ";"; + + ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUpFilesWithTTL: " << cleanUpFilesWithTTLQuery.str(); + + if (!cleanUpFilesWithTTLQuery.exec()) { + return HandleException("QUERY ERROR WHEN CLEANING UP TABLES WITH TTL", + cleanUpFilesWithTTLQuery.error()); + } + } + + if (remove_tables > 0) { + ENGINE_LOG_DEBUG << "Remove " << remove_tables << " tables from meta"; + } + } // Scoped Connection + } catch (std::exception& e) { + return HandleException("GENERAL ERROR WHEN CLEANING UP TABLES WITH TTL", e.what()); + } + + // remove deleted table folder + // don't remove table folder until all its files has been deleted + try { + server::MetricCollector metric; + + { + mysqlpp::ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); + + if (connectionPtr == nullptr) { + return Status(DB_ERROR, "Failed to connect to meta server(mysql)"); + } + + for (auto& table_id : table_ids) { + mysqlpp::Query cleanUpFilesWithTTLQuery = connectionPtr->query(); + cleanUpFilesWithTTLQuery << "SELECT file_id" + << " FROM " << META_TABLEFILES << " WHERE table_id = " << mysqlpp::quote + << table_id << ";"; + + ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUpFilesWithTTL: " << cleanUpFilesWithTTLQuery.str(); + + mysqlpp::StoreQueryResult res = cleanUpFilesWithTTLQuery.store(); + + if (res.empty()) { + utils::DeleteTablePath(options_, table_id); + } + } + + if (table_ids.size() > 0) { + ENGINE_LOG_DEBUG << "Remove " << table_ids.size() << " tables folder"; + } + } + } catch (std::exception& e) { + return HandleException("GENERAL ERROR WHEN CLEANING UP TABLES WITH TTL", e.what()); + } + + return Status::OK(); +} + Status MySQLMetaImpl::Count(const std::string& table_id, uint64_t& result) { try { @@ -1944,15 +1917,15 @@ MySQLMetaImpl::Count(const std::string& table_id, uint64_t& result) { mysqlpp::ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); if (connectionPtr == nullptr) { - return Status(DB_ERROR, "Failed to connect to database server"); + return Status(DB_ERROR, "Failed to connect to meta server(mysql)"); } mysqlpp::Query countQuery = connectionPtr->query(); - countQuery << "SELECT row_count FROM " << META_TABLEFILES << " " - << "WHERE table_id = " << mysqlpp::quote << table_id << " AND " - << "(file_type = " << std::to_string(TableFileSchema::RAW) << " OR " - << "file_type = " << std::to_string(TableFileSchema::TO_INDEX) << " OR " - << "file_type = " << std::to_string(TableFileSchema::INDEX) << ");"; + countQuery << "SELECT row_count" + << " FROM " << META_TABLEFILES << " WHERE table_id = " << mysqlpp::quote << table_id + << " AND (file_type = " << std::to_string(TableFileSchema::RAW) + << " OR file_type = " << std::to_string(TableFileSchema::TO_INDEX) + << " OR file_type = " << std::to_string(TableFileSchema::INDEX) << ");"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::Count: " << countQuery.str(); @@ -1978,7 +1951,7 @@ MySQLMetaImpl::DropAll() { mysqlpp::ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); if (connectionPtr == nullptr) { - return Status(DB_ERROR, "Failed to connect to database server"); + return Status(DB_ERROR, "Failed to connect to meta server(mysql)"); } mysqlpp::Query dropTableQuery = connectionPtr->query(); @@ -1995,6 +1968,72 @@ MySQLMetaImpl::DropAll() { } } +Status +MySQLMetaImpl::DiscardFiles(int64_t to_discard_size) { + if (to_discard_size <= 0) { + return Status::OK(); + } + ENGINE_LOG_DEBUG << "About to discard size=" << to_discard_size; + + try { + server::MetricCollector metric; + bool status; + { + mysqlpp::ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_); + + if (connectionPtr == nullptr) { + return Status(DB_ERROR, "Failed to connect to meta server(mysql)"); + } + + mysqlpp::Query discardFilesQuery = connectionPtr->query(); + discardFilesQuery << "SELECT id, file_size" + << " FROM " << META_TABLEFILES << " WHERE file_type <> " + << std::to_string(TableFileSchema::TO_DELETE) << " ORDER BY id ASC " + << " LIMIT 10;"; + + ENGINE_LOG_DEBUG << "MySQLMetaImpl::DiscardFiles: " << discardFilesQuery.str(); + + mysqlpp::StoreQueryResult res = discardFilesQuery.store(); + if (res.num_rows() == 0) { + return Status::OK(); + } + + TableFileSchema table_file; + std::stringstream idsToDiscardSS; + for (auto& resRow : res) { + if (to_discard_size <= 0) { + break; + } + table_file.id_ = resRow["id"]; + table_file.file_size_ = resRow["file_size"]; + idsToDiscardSS << "id = " << std::to_string(table_file.id_) << " OR "; + ENGINE_LOG_DEBUG << "Discard table_file.id=" << table_file.file_id_ + << " table_file.size=" << table_file.file_size_; + to_discard_size -= table_file.file_size_; + } + + std::string idsToDiscardStr = idsToDiscardSS.str(); + idsToDiscardStr = idsToDiscardStr.substr(0, idsToDiscardStr.size() - 4); // remove the last " OR " + + discardFilesQuery << "UPDATE " << META_TABLEFILES + << " SET file_type = " << std::to_string(TableFileSchema::TO_DELETE) + << " ,updated_time = " << std::to_string(utils::GetMicroSecTimeStamp()) << " WHERE " + << idsToDiscardStr << ";"; + + ENGINE_LOG_DEBUG << "MySQLMetaImpl::DiscardFiles: " << discardFilesQuery.str(); + + status = discardFilesQuery.exec(); + if (!status) { + return HandleException("QUERY ERROR WHEN DISCARDING FILES", discardFilesQuery.error()); + } + } // Scoped Connection + + return DiscardFiles(to_discard_size); + } catch (std::exception& e) { + return HandleException("GENERAL ERROR WHEN DISCARDING FILES", e.what()); + } +} + } // namespace meta } // namespace engine } // namespace milvus diff --git a/core/src/db/meta/MySQLMetaImpl.h b/core/src/db/meta/MySQLMetaImpl.h index bb7fb5b59f..00b7627548 100644 --- a/core/src/db/meta/MySQLMetaImpl.h +++ b/core/src/db/meta/MySQLMetaImpl.h @@ -49,7 +49,7 @@ class MySQLMetaImpl : public Meta { AllTables(std::vector& table_schema_array) override; Status - DeleteTable(const std::string& table_id) override; + DropTable(const std::string& table_id) override; Status DeleteTableFiles(const std::string& table_id) override; @@ -58,27 +58,17 @@ class MySQLMetaImpl : public Meta { CreateTableFile(TableFileSchema& file_schema) override; Status - DropPartitionsByDates(const std::string& table_id, const DatesT& dates) override; + DropDataByDate(const std::string& table_id, const DatesT& dates) override; Status GetTableFiles(const std::string& table_id, const std::vector& ids, TableFilesSchema& table_files) override; - Status - FilesByType(const std::string& table_id, const std::vector& file_types, - std::vector& file_ids) override; - Status UpdateTableIndex(const std::string& table_id, const TableIndex& index) override; Status UpdateTableFlag(const std::string& table_id, int64_t flag) override; - Status - DescribeTableIndex(const std::string& table_id, TableIndex& index) override; - - Status - DropTableIndex(const std::string& table_id) override; - Status UpdateTableFile(TableFileSchema& file_schema) override; @@ -88,6 +78,24 @@ class MySQLMetaImpl : public Meta { Status UpdateTableFiles(TableFilesSchema& files) override; + Status + DescribeTableIndex(const std::string& table_id, TableIndex& index) override; + + Status + DropTableIndex(const std::string& table_id) override; + + Status + CreatePartition(const std::string& table_id, const std::string& partition_name, const std::string& tag) override; + + Status + DropPartition(const std::string& partition_name) override; + + Status + ShowPartitions(const std::string& table_id, std::vector& partiton_schema_array) override; + + Status + GetPartitionName(const std::string& table_id, const std::string& tag, std::string& partition_name) override; + Status FilesToSearch(const std::string& table_id, const std::vector& ids, const DatesT& dates, DatePartionedTableFilesSchema& files) override; @@ -98,6 +106,10 @@ class MySQLMetaImpl : public Meta { Status FilesToIndex(TableFilesSchema&) override; + Status + FilesByType(const std::string& table_id, const std::vector& file_types, + std::vector& file_ids) override; + Status Archive() override; diff --git a/core/src/db/meta/SqliteMetaImpl.cpp b/core/src/db/meta/SqliteMetaImpl.cpp index 4123858b44..12128c074d 100644 --- a/core/src/db/meta/SqliteMetaImpl.cpp +++ b/core/src/db/meta/SqliteMetaImpl.cpp @@ -16,24 +16,24 @@ // under the License. #include "db/meta/SqliteMetaImpl.h" +#include "MetaConsts.h" #include "db/IDGenerator.h" #include "db/Utils.h" -#include "utils/Log.h" -#include "utils/Exception.h" -#include "MetaConsts.h" #include "metrics/Metrics.h" +#include "utils/Exception.h" +#include "utils/Log.h" +#include "utils/StringHelpFunctions.h" +#include #include -#include -#include #include #include #include -#include +#include #include +#include #include -#include - +#include namespace milvus { namespace engine { @@ -44,7 +44,7 @@ using namespace sqlite_orm; namespace { Status -HandleException(const std::string &desc, const char *what = nullptr) { +HandleException(const std::string& desc, const char* what = nullptr) { if (what == nullptr) { ENGINE_LOG_ERROR << desc; return Status(DB_META_TRANSACTION_FAILED, desc); @@ -55,7 +55,7 @@ HandleException(const std::string &desc, const char *what = nullptr) { } } -} // namespace +} // namespace inline auto StoragePrototype(const std::string &path) { @@ -70,7 +70,10 @@ StoragePrototype(const std::string &path) { make_column("index_file_size", &TableSchema::index_file_size_), make_column("engine_type", &TableSchema::engine_type_), make_column("nlist", &TableSchema::nlist_), - make_column("metric_type", &TableSchema::metric_type_)), + make_column("metric_type", &TableSchema::metric_type_), + make_column("owner_table", &TableSchema::owner_table_, default_value("")), + make_column("partition_tag", &TableSchema::partition_tag_, default_value("")), + make_column("version", &TableSchema::version_, default_value(CURRENT_VERSION))), make_table(META_TABLEFILES, make_column("id", &TableFileSchema::id_, primary_key()), make_column("table_id", &TableFileSchema::table_id_), @@ -87,8 +90,7 @@ StoragePrototype(const std::string &path) { using ConnectorT = decltype(StoragePrototype("")); static std::unique_ptr ConnectorPtr; -SqliteMetaImpl::SqliteMetaImpl(const DBMetaOptions &options) - : options_(options) { +SqliteMetaImpl::SqliteMetaImpl(const DBMetaOptions& options) : options_(options) { Initialize(); } @@ -96,7 +98,7 @@ SqliteMetaImpl::~SqliteMetaImpl() { } Status -SqliteMetaImpl::NextTableId(std::string &table_id) { +SqliteMetaImpl::NextTableId(std::string& table_id) { std::lock_guard lock(genid_mutex_); // avoid duplicated id std::stringstream ss; SimpleIDGenerator g; @@ -106,7 +108,7 @@ SqliteMetaImpl::NextTableId(std::string &table_id) { } Status -SqliteMetaImpl::NextFileId(std::string &file_id) { +SqliteMetaImpl::NextFileId(std::string& file_id) { std::lock_guard lock(genid_mutex_); // avoid duplicated id std::stringstream ss; SimpleIDGenerator g; @@ -121,14 +123,14 @@ SqliteMetaImpl::ValidateMetaSchema() { return; } - //old meta could be recreated since schema changed, throw exception if meta schema is not compatible + // old meta could be recreated since schema changed, throw exception if meta schema is not compatible auto ret = ConnectorPtr->sync_schema_simulate(); - if (ret.find(META_TABLES) != ret.end() - && sqlite_orm::sync_schema_result::dropped_and_recreated == ret[META_TABLES]) { + if (ret.find(META_TABLES) != ret.end() && + sqlite_orm::sync_schema_result::dropped_and_recreated == ret[META_TABLES]) { throw Exception(DB_INCOMPATIB_META, "Meta Tables schema is created by Milvus old version"); } - if (ret.find(META_TABLEFILES) != ret.end() - && sqlite_orm::sync_schema_result::dropped_and_recreated == ret[META_TABLEFILES]) { + if (ret.find(META_TABLEFILES) != ret.end() && + sqlite_orm::sync_schema_result::dropped_and_recreated == ret[META_TABLEFILES]) { throw Exception(DB_INCOMPATIB_META, "Meta TableFiles schema is created by Milvus old version"); } } @@ -149,68 +151,14 @@ SqliteMetaImpl::Initialize() { ValidateMetaSchema(); ConnectorPtr->sync_schema(); - ConnectorPtr->open_forever(); // thread safe option - ConnectorPtr->pragma.journal_mode(journal_mode::WAL); // WAL => write ahead log + ConnectorPtr->open_forever(); // thread safe option + ConnectorPtr->pragma.journal_mode(journal_mode::WAL); // WAL => write ahead log CleanUp(); return Status::OK(); } -// TODO(myh): Delete single vecotor by id -Status -SqliteMetaImpl::DropPartitionsByDates(const std::string &table_id, - const DatesT &dates) { - if (dates.empty()) { - return Status::OK(); - } - - TableSchema table_schema; - table_schema.table_id_ = table_id; - auto status = DescribeTable(table_schema); - if (!status.ok()) { - return status; - } - - try { - //sqlite_orm has a bug, 'in' statement cannot handle too many elements - //so we split one query into multi-queries, this is a work-around!! - std::vector split_dates; - split_dates.push_back(DatesT()); - const size_t batch_size = 30; - for(DateT date : dates) { - DatesT& last_batch = *split_dates.rbegin(); - last_batch.push_back(date); - if(last_batch.size() > batch_size) { - split_dates.push_back(DatesT()); - } - } - - //multi-threads call sqlite update may get exception('bad logic', etc), so we add a lock here - std::lock_guard meta_lock(meta_mutex_); - - for(auto& batch_dates : split_dates) { - if(batch_dates.empty()) { - continue; - } - - ConnectorPtr->update_all( - set( - c(&TableFileSchema::file_type_) = (int)TableFileSchema::TO_DELETE, - c(&TableFileSchema::updated_time_) = utils::GetMicroSecTimeStamp()), - where( - c(&TableFileSchema::table_id_) == table_id and - in(&TableFileSchema::date_, batch_dates))); - } - - ENGINE_LOG_DEBUG << "Successfully drop partitions, table id = " << table_schema.table_id_; - } catch (std::exception &e) { - return HandleException("Encounter exception when drop partition", e.what()); - } - - return Status::OK(); -} - Status SqliteMetaImpl::CreateTable(TableSchema &table_schema) { try { @@ -253,7 +201,113 @@ SqliteMetaImpl::CreateTable(TableSchema &table_schema) { } Status -SqliteMetaImpl::DeleteTable(const std::string &table_id) { +SqliteMetaImpl::DescribeTable(TableSchema &table_schema) { + try { + server::MetricCollector metric; + + auto groups = ConnectorPtr->select(columns(&TableSchema::id_, + &TableSchema::state_, + &TableSchema::dimension_, + &TableSchema::created_on_, + &TableSchema::flag_, + &TableSchema::index_file_size_, + &TableSchema::engine_type_, + &TableSchema::nlist_, + &TableSchema::metric_type_, + &TableSchema::owner_table_, + &TableSchema::partition_tag_, + &TableSchema::version_), + where(c(&TableSchema::table_id_) == table_schema.table_id_ + and c(&TableSchema::state_) != (int) TableSchema::TO_DELETE)); + + if (groups.size() == 1) { + table_schema.id_ = std::get<0>(groups[0]); + table_schema.state_ = std::get<1>(groups[0]); + table_schema.dimension_ = std::get<2>(groups[0]); + table_schema.created_on_ = std::get<3>(groups[0]); + table_schema.flag_ = std::get<4>(groups[0]); + table_schema.index_file_size_ = std::get<5>(groups[0]); + table_schema.engine_type_ = std::get<6>(groups[0]); + table_schema.nlist_ = std::get<7>(groups[0]); + table_schema.metric_type_ = std::get<8>(groups[0]); + table_schema.owner_table_ = std::get<9>(groups[0]); + table_schema.partition_tag_ = std::get<10>(groups[0]); + table_schema.version_ = std::get<11>(groups[0]); + } else { + return Status(DB_NOT_FOUND, "Table " + table_schema.table_id_ + " not found"); + } + } catch (std::exception &e) { + return HandleException("Encounter exception when describe table", e.what()); + } + + return Status::OK(); +} + +Status +SqliteMetaImpl::HasTable(const std::string &table_id, bool &has_or_not) { + has_or_not = false; + + try { + server::MetricCollector metric; + auto tables = ConnectorPtr->select(columns(&TableSchema::id_), + where(c(&TableSchema::table_id_) == table_id + and c(&TableSchema::state_) != (int) TableSchema::TO_DELETE)); + if (tables.size() == 1) { + has_or_not = true; + } else { + has_or_not = false; + } + } catch (std::exception &e) { + return HandleException("Encounter exception when lookup table", e.what()); + } + + return Status::OK(); +} + +Status +SqliteMetaImpl::AllTables(std::vector &table_schema_array) { + try { + server::MetricCollector metric; + + auto selected = ConnectorPtr->select(columns(&TableSchema::id_, + &TableSchema::table_id_, + &TableSchema::dimension_, + &TableSchema::created_on_, + &TableSchema::flag_, + &TableSchema::index_file_size_, + &TableSchema::engine_type_, + &TableSchema::nlist_, + &TableSchema::metric_type_, + &TableSchema::owner_table_, + &TableSchema::partition_tag_, + &TableSchema::version_), + where(c(&TableSchema::state_) != (int) TableSchema::TO_DELETE)); + for (auto &table : selected) { + TableSchema schema; + schema.id_ = std::get<0>(table); + schema.table_id_ = std::get<1>(table); + schema.dimension_ = std::get<2>(table); + schema.created_on_ = std::get<3>(table); + schema.flag_ = std::get<4>(table); + schema.index_file_size_ = std::get<5>(table); + schema.engine_type_ = std::get<6>(table); + schema.nlist_ = std::get<7>(table); + schema.metric_type_ = std::get<8>(table); + schema.owner_table_ = std::get<9>(table); + schema.partition_tag_ = std::get<10>(table); + schema.version_ = std::get<11>(table); + + table_schema_array.emplace_back(schema); + } + } catch (std::exception &e) { + return HandleException("Encounter exception when lookup all tables", e.what()); + } + + return Status::OK(); +} + +Status +SqliteMetaImpl::DropTable(const std::string &table_id) { try { server::MetricCollector metric; @@ -301,291 +355,6 @@ SqliteMetaImpl::DeleteTableFiles(const std::string &table_id) { return Status::OK(); } -Status -SqliteMetaImpl::DescribeTable(TableSchema &table_schema) { - try { - server::MetricCollector metric; - - auto groups = ConnectorPtr->select(columns(&TableSchema::id_, - &TableSchema::state_, - &TableSchema::dimension_, - &TableSchema::created_on_, - &TableSchema::flag_, - &TableSchema::index_file_size_, - &TableSchema::engine_type_, - &TableSchema::nlist_, - &TableSchema::metric_type_), - where(c(&TableSchema::table_id_) == table_schema.table_id_ - and c(&TableSchema::state_) != (int) TableSchema::TO_DELETE)); - - if (groups.size() == 1) { - table_schema.id_ = std::get<0>(groups[0]); - table_schema.state_ = std::get<1>(groups[0]); - table_schema.dimension_ = std::get<2>(groups[0]); - table_schema.created_on_ = std::get<3>(groups[0]); - table_schema.flag_ = std::get<4>(groups[0]); - table_schema.index_file_size_ = std::get<5>(groups[0]); - table_schema.engine_type_ = std::get<6>(groups[0]); - table_schema.nlist_ = std::get<7>(groups[0]); - table_schema.metric_type_ = std::get<8>(groups[0]); - } else { - return Status(DB_NOT_FOUND, "Table " + table_schema.table_id_ + " not found"); - } - } catch (std::exception &e) { - return HandleException("Encounter exception when describe table", e.what()); - } - - return Status::OK(); -} - -Status -SqliteMetaImpl::FilesByType(const std::string &table_id, - const std::vector &file_types, - std::vector &file_ids) { - if (file_types.empty()) { - return Status(DB_ERROR, "file types array is empty"); - } - - try { - file_ids.clear(); - auto selected = ConnectorPtr->select(columns(&TableFileSchema::file_id_, - &TableFileSchema::file_type_), - where(in(&TableFileSchema::file_type_, file_types) - and c(&TableFileSchema::table_id_) == table_id)); - - if (selected.size() >= 1) { - int raw_count = 0, new_count = 0, new_merge_count = 0, new_index_count = 0; - int to_index_count = 0, index_count = 0, backup_count = 0; - for (auto &file : selected) { - file_ids.push_back(std::get<0>(file)); - switch (std::get<1>(file)) { - case (int) TableFileSchema::RAW:raw_count++; - break; - case (int) TableFileSchema::NEW:new_count++; - break; - case (int) TableFileSchema::NEW_MERGE:new_merge_count++; - break; - case (int) TableFileSchema::NEW_INDEX:new_index_count++; - break; - case (int) TableFileSchema::TO_INDEX:to_index_count++; - break; - case (int) TableFileSchema::INDEX:index_count++; - break; - case (int) TableFileSchema::BACKUP:backup_count++; - break; - default:break; - } - } - - ENGINE_LOG_DEBUG << "Table " << table_id << " currently has raw files:" << raw_count - << " new files:" << new_count << " new_merge files:" << new_merge_count - << " new_index files:" << new_index_count << " to_index files:" << to_index_count - << " index files:" << index_count << " backup files:" << backup_count; - } - } catch (std::exception &e) { - return HandleException("Encounter exception when check non index files", e.what()); - } - return Status::OK(); -} - -Status -SqliteMetaImpl::UpdateTableIndex(const std::string &table_id, const TableIndex &index) { - try { - server::MetricCollector metric; - - //multi-threads call sqlite update may get exception('bad logic', etc), so we add a lock here - std::lock_guard meta_lock(meta_mutex_); - - auto tables = ConnectorPtr->select(columns(&TableSchema::id_, - &TableSchema::state_, - &TableSchema::dimension_, - &TableSchema::created_on_, - &TableSchema::flag_, - &TableSchema::index_file_size_), - where(c(&TableSchema::table_id_) == table_id - and c(&TableSchema::state_) != (int) TableSchema::TO_DELETE)); - - if (tables.size() > 0) { - meta::TableSchema table_schema; - table_schema.id_ = std::get<0>(tables[0]); - table_schema.table_id_ = table_id; - table_schema.state_ = std::get<1>(tables[0]); - table_schema.dimension_ = std::get<2>(tables[0]); - table_schema.created_on_ = std::get<3>(tables[0]); - table_schema.flag_ = std::get<4>(tables[0]); - table_schema.index_file_size_ = std::get<5>(tables[0]); - table_schema.engine_type_ = index.engine_type_; - table_schema.nlist_ = index.nlist_; - table_schema.metric_type_ = index.metric_type_; - - ConnectorPtr->update(table_schema); - } else { - return Status(DB_NOT_FOUND, "Table " + table_id + " not found"); - } - - //set all backup file to raw - ConnectorPtr->update_all( - set( - c(&TableFileSchema::file_type_) = (int) TableFileSchema::RAW, - c(&TableFileSchema::updated_time_) = utils::GetMicroSecTimeStamp()), - where( - c(&TableFileSchema::table_id_) == table_id and - c(&TableFileSchema::file_type_) == (int) TableFileSchema::BACKUP)); - - ENGINE_LOG_DEBUG << "Successfully update table index, table id = " << table_id; - } catch (std::exception &e) { - std::string msg = "Encounter exception when update table index: table_id = " + table_id; - return HandleException(msg, e.what()); - } - - return Status::OK(); -} - -Status -SqliteMetaImpl::UpdateTableFlag(const std::string &table_id, int64_t flag) { - try { - server::MetricCollector metric; - - //set all backup file to raw - ConnectorPtr->update_all( - set( - c(&TableSchema::flag_) = flag), - where( - c(&TableSchema::table_id_) == table_id)); - ENGINE_LOG_DEBUG << "Successfully update table flag, table id = " << table_id; - } catch (std::exception &e) { - std::string msg = "Encounter exception when update table flag: table_id = " + table_id; - return HandleException(msg, e.what()); - } - - return Status::OK(); -} - -Status -SqliteMetaImpl::DescribeTableIndex(const std::string &table_id, TableIndex &index) { - try { - server::MetricCollector metric; - - auto groups = ConnectorPtr->select(columns(&TableSchema::engine_type_, - &TableSchema::nlist_, - &TableSchema::metric_type_), - where(c(&TableSchema::table_id_) == table_id - and c(&TableSchema::state_) != (int) TableSchema::TO_DELETE)); - - if (groups.size() == 1) { - index.engine_type_ = std::get<0>(groups[0]); - index.nlist_ = std::get<1>(groups[0]); - index.metric_type_ = std::get<2>(groups[0]); - } else { - return Status(DB_NOT_FOUND, "Table " + table_id + " not found"); - } - } catch (std::exception &e) { - return HandleException("Encounter exception when describe index", e.what()); - } - - return Status::OK(); -} - -Status -SqliteMetaImpl::DropTableIndex(const std::string &table_id) { - try { - server::MetricCollector metric; - - //multi-threads call sqlite update may get exception('bad logic', etc), so we add a lock here - std::lock_guard meta_lock(meta_mutex_); - - //soft delete index files - ConnectorPtr->update_all( - set( - c(&TableFileSchema::file_type_) = (int) TableFileSchema::TO_DELETE, - c(&TableFileSchema::updated_time_) = utils::GetMicroSecTimeStamp()), - where( - c(&TableFileSchema::table_id_) == table_id and - c(&TableFileSchema::file_type_) == (int) TableFileSchema::INDEX)); - - //set all backup file to raw - ConnectorPtr->update_all( - set( - c(&TableFileSchema::file_type_) = (int) TableFileSchema::RAW, - c(&TableFileSchema::updated_time_) = utils::GetMicroSecTimeStamp()), - where( - c(&TableFileSchema::table_id_) == table_id and - c(&TableFileSchema::file_type_) == (int) TableFileSchema::BACKUP)); - - //set table index type to raw - ConnectorPtr->update_all( - set( - c(&TableSchema::engine_type_) = DEFAULT_ENGINE_TYPE, - c(&TableSchema::nlist_) = DEFAULT_NLIST, - c(&TableSchema::metric_type_) = DEFAULT_METRIC_TYPE), - where( - c(&TableSchema::table_id_) == table_id)); - - ENGINE_LOG_DEBUG << "Successfully drop table index, table id = " << table_id; - } catch (std::exception &e) { - return HandleException("Encounter exception when delete table index files", e.what()); - } - - return Status::OK(); -} - -Status -SqliteMetaImpl::HasTable(const std::string &table_id, bool &has_or_not) { - has_or_not = false; - - try { - server::MetricCollector metric; - auto tables = ConnectorPtr->select(columns(&TableSchema::id_), - where(c(&TableSchema::table_id_) == table_id - and c(&TableSchema::state_) != (int) TableSchema::TO_DELETE)); - if (tables.size() == 1) { - has_or_not = true; - } else { - has_or_not = false; - } - } catch (std::exception &e) { - return HandleException("Encounter exception when lookup table", e.what()); - } - - return Status::OK(); -} - -Status -SqliteMetaImpl::AllTables(std::vector &table_schema_array) { - try { - server::MetricCollector metric; - - auto selected = ConnectorPtr->select(columns(&TableSchema::id_, - &TableSchema::table_id_, - &TableSchema::dimension_, - &TableSchema::created_on_, - &TableSchema::flag_, - &TableSchema::index_file_size_, - &TableSchema::engine_type_, - &TableSchema::nlist_, - &TableSchema::metric_type_), - where(c(&TableSchema::state_) != (int) TableSchema::TO_DELETE)); - for (auto &table : selected) { - TableSchema schema; - schema.id_ = std::get<0>(table); - schema.table_id_ = std::get<1>(table); - schema.dimension_ = std::get<2>(table); - schema.created_on_ = std::get<3>(table); - schema.flag_ = std::get<4>(table); - schema.index_file_size_ = std::get<5>(table); - schema.engine_type_ = std::get<6>(table); - schema.nlist_ = std::get<7>(table); - schema.metric_type_ = std::get<8>(table); - - table_schema_array.emplace_back(schema); - } - } catch (std::exception &e) { - return HandleException("Encounter exception when lookup all tables", e.what()); - } - - return Status::OK(); -} - Status SqliteMetaImpl::CreateTableFile(TableFileSchema &file_schema) { if (file_schema.date_ == EmptyDate) { @@ -627,6 +396,641 @@ SqliteMetaImpl::CreateTableFile(TableFileSchema &file_schema) { return Status::OK(); } +// TODO(myh): Delete single vecotor by id +Status +SqliteMetaImpl::DropDataByDate(const std::string &table_id, + const DatesT &dates) { + if (dates.empty()) { + return Status::OK(); + } + + TableSchema table_schema; + table_schema.table_id_ = table_id; + auto status = DescribeTable(table_schema); + if (!status.ok()) { + return status; + } + + try { + // sqlite_orm has a bug, 'in' statement cannot handle too many elements + // so we split one query into multi-queries, this is a work-around!! + std::vector split_dates; + split_dates.push_back(DatesT()); + const size_t batch_size = 30; + for (DateT date : dates) { + DatesT& last_batch = *split_dates.rbegin(); + last_batch.push_back(date); + if (last_batch.size() > batch_size) { + split_dates.push_back(DatesT()); + } + } + + // multi-threads call sqlite update may get exception('bad logic', etc), so we add a lock here + std::lock_guard meta_lock(meta_mutex_); + + for (auto& batch_dates : split_dates) { + if (batch_dates.empty()) { + continue; + } + + ConnectorPtr->update_all( + set(c(&TableFileSchema::file_type_) = (int)TableFileSchema::TO_DELETE, + c(&TableFileSchema::updated_time_) = utils::GetMicroSecTimeStamp()), + where(c(&TableFileSchema::table_id_) == table_id and in(&TableFileSchema::date_, batch_dates))); + } + + ENGINE_LOG_DEBUG << "Successfully drop data by date, table id = " << table_schema.table_id_; + } catch (std::exception &e) { + return HandleException("Encounter exception when drop partition", e.what()); + } + + return Status::OK(); +} + +Status +SqliteMetaImpl::GetTableFiles(const std::string &table_id, + const std::vector &ids, + TableFilesSchema &table_files) { + try { + table_files.clear(); + auto files = ConnectorPtr->select(columns(&TableFileSchema::id_, + &TableFileSchema::file_id_, + &TableFileSchema::file_type_, + &TableFileSchema::file_size_, + &TableFileSchema::row_count_, + &TableFileSchema::date_, + &TableFileSchema::engine_type_, + &TableFileSchema::created_on_), + where(c(&TableFileSchema::table_id_) == table_id and + in(&TableFileSchema::id_, ids) and + c(&TableFileSchema::file_type_) != (int) TableFileSchema::TO_DELETE)); + TableSchema table_schema; + table_schema.table_id_ = table_id; + auto status = DescribeTable(table_schema); + if (!status.ok()) { + return status; + } + + Status result; + for (auto &file : files) { + TableFileSchema file_schema; + file_schema.table_id_ = table_id; + file_schema.id_ = std::get<0>(file); + file_schema.file_id_ = std::get<1>(file); + file_schema.file_type_ = std::get<2>(file); + file_schema.file_size_ = std::get<3>(file); + file_schema.row_count_ = std::get<4>(file); + file_schema.date_ = std::get<5>(file); + file_schema.engine_type_ = std::get<6>(file); + file_schema.created_on_ = std::get<7>(file); + file_schema.dimension_ = table_schema.dimension_; + file_schema.index_file_size_ = table_schema.index_file_size_; + file_schema.nlist_ = table_schema.nlist_; + file_schema.metric_type_ = table_schema.metric_type_; + + utils::GetTableFilePath(options_, file_schema); + + table_files.emplace_back(file_schema); + } + + ENGINE_LOG_DEBUG << "Get table files by id"; + return result; + } catch (std::exception &e) { + return HandleException("Encounter exception when lookup table files", e.what()); + } +} + +Status +SqliteMetaImpl::UpdateTableFlag(const std::string &table_id, int64_t flag) { + try { + server::MetricCollector metric; + + //set all backup file to raw + ConnectorPtr->update_all( + set( + c(&TableSchema::flag_) = flag), + where( + c(&TableSchema::table_id_) == table_id)); + ENGINE_LOG_DEBUG << "Successfully update table flag, table id = " << table_id; + } catch (std::exception &e) { + std::string msg = "Encounter exception when update table flag: table_id = " + table_id; + return HandleException(msg, e.what()); + } + + return Status::OK(); +} + +Status +SqliteMetaImpl::UpdateTableFile(TableFileSchema &file_schema) { + file_schema.updated_time_ = utils::GetMicroSecTimeStamp(); + try { + server::MetricCollector metric; + + // multi-threads call sqlite update may get exception('bad logic', etc), so we add a lock here + std::lock_guard meta_lock(meta_mutex_); + + auto tables = ConnectorPtr->select(columns(&TableSchema::state_), + where(c(&TableSchema::table_id_) == file_schema.table_id_)); + + //if the table has been deleted, just mark the table file as TO_DELETE + //clean thread will delete the file later + if (tables.size() < 1 || std::get<0>(tables[0]) == (int) TableSchema::TO_DELETE) { + file_schema.file_type_ = TableFileSchema::TO_DELETE; + } + + ConnectorPtr->update(file_schema); + + ENGINE_LOG_DEBUG << "Update single table file, file id = " << file_schema.file_id_; + } catch (std::exception &e) { + std::string msg = "Exception update table file: table_id = " + file_schema.table_id_ + + " file_id = " + file_schema.file_id_; + return HandleException(msg, e.what()); + } + return Status::OK(); +} + +Status +SqliteMetaImpl::UpdateTableFiles(TableFilesSchema &files) { + try { + server::MetricCollector metric; + + //multi-threads call sqlite update may get exception('bad logic', etc), so we add a lock here + std::lock_guard meta_lock(meta_mutex_); + + std::map has_tables; + for (auto &file : files) { + if (has_tables.find(file.table_id_) != has_tables.end()) { + continue; + } + auto tables = ConnectorPtr->select(columns(&TableSchema::id_), + where(c(&TableSchema::table_id_) == file.table_id_ + and c(&TableSchema::state_) != (int) TableSchema::TO_DELETE)); + if (tables.size() >= 1) { + has_tables[file.table_id_] = true; + } else { + has_tables[file.table_id_] = false; + } + } + + auto commited = ConnectorPtr->transaction([&]() mutable { + for (auto &file : files) { + if (!has_tables[file.table_id_]) { + file.file_type_ = TableFileSchema::TO_DELETE; + } + + file.updated_time_ = utils::GetMicroSecTimeStamp(); + ConnectorPtr->update(file); + } + return true; + }); + + if (!commited) { + return HandleException("UpdateTableFiles error: sqlite transaction failed"); + } + + ENGINE_LOG_DEBUG << "Update " << files.size() << " table files"; + } catch (std::exception &e) { + return HandleException("Encounter exception when update table files", e.what()); + } + return Status::OK(); +} + +Status +SqliteMetaImpl::UpdateTableIndex(const std::string& table_id, const TableIndex& index) { + try { + server::MetricCollector metric; + + // multi-threads call sqlite update may get exception('bad logic', etc), so we add a lock here + std::lock_guard meta_lock(meta_mutex_); + + auto tables = ConnectorPtr->select(columns(&TableSchema::id_, + &TableSchema::state_, + &TableSchema::dimension_, + &TableSchema::created_on_, + &TableSchema::flag_, + &TableSchema::index_file_size_, + &TableSchema::owner_table_, + &TableSchema::partition_tag_, + &TableSchema::version_), + where(c(&TableSchema::table_id_) == table_id + and c(&TableSchema::state_) != (int) TableSchema::TO_DELETE)); + + if (tables.size() > 0) { + meta::TableSchema table_schema; + table_schema.id_ = std::get<0>(tables[0]); + table_schema.table_id_ = table_id; + table_schema.state_ = std::get<1>(tables[0]); + table_schema.dimension_ = std::get<2>(tables[0]); + table_schema.created_on_ = std::get<3>(tables[0]); + table_schema.flag_ = std::get<4>(tables[0]); + table_schema.index_file_size_ = std::get<5>(tables[0]); + table_schema.owner_table_ = std::get<6>(tables[0]); + table_schema.partition_tag_ = std::get<7>(tables[0]); + table_schema.version_ = std::get<8>(tables[0]); + table_schema.engine_type_ = index.engine_type_; + table_schema.nlist_ = index.nlist_; + table_schema.metric_type_ = index.metric_type_; + + ConnectorPtr->update(table_schema); + } else { + return Status(DB_NOT_FOUND, "Table " + table_id + " not found"); + } + + //set all backup file to raw + ConnectorPtr->update_all( + set( + c(&TableFileSchema::file_type_) = (int) TableFileSchema::RAW, + c(&TableFileSchema::updated_time_) = utils::GetMicroSecTimeStamp()), + where( + c(&TableFileSchema::table_id_) == table_id and + c(&TableFileSchema::file_type_) == (int) TableFileSchema::BACKUP)); + + ENGINE_LOG_DEBUG << "Successfully update table index, table id = " << table_id; + } catch (std::exception& e) { + std::string msg = "Encounter exception when update table index: table_id = " + table_id; + return HandleException(msg, e.what()); + } + + return Status::OK(); +} + +Status +SqliteMetaImpl::UpdateTableFilesToIndex(const std::string &table_id) { + try { + server::MetricCollector metric; + + //multi-threads call sqlite update may get exception('bad logic', etc), so we add a lock here + std::lock_guard meta_lock(meta_mutex_); + + ConnectorPtr->update_all( + set( + c(&TableFileSchema::file_type_) = (int) TableFileSchema::TO_INDEX), + where( + c(&TableFileSchema::table_id_) == table_id and + c(&TableFileSchema::file_type_) == (int) TableFileSchema::RAW)); + + ENGINE_LOG_DEBUG << "Update files to to_index, table id = " << table_id; + } catch (std::exception &e) { + return HandleException("Encounter exception when update table files to to_index", e.what()); + } + + return Status::OK(); +} + +Status +SqliteMetaImpl::DescribeTableIndex(const std::string& table_id, TableIndex& index) { + try { + server::MetricCollector metric; + + auto groups = ConnectorPtr->select(columns(&TableSchema::engine_type_, + &TableSchema::nlist_, + &TableSchema::metric_type_), + where(c(&TableSchema::table_id_) == table_id + and c(&TableSchema::state_) != (int) TableSchema::TO_DELETE)); + + if (groups.size() == 1) { + index.engine_type_ = std::get<0>(groups[0]); + index.nlist_ = std::get<1>(groups[0]); + index.metric_type_ = std::get<2>(groups[0]); + } else { + return Status(DB_NOT_FOUND, "Table " + table_id + " not found"); + } + } catch (std::exception& e) { + return HandleException("Encounter exception when describe index", e.what()); + } + + return Status::OK(); +} + +Status +SqliteMetaImpl::DropTableIndex(const std::string& table_id) { + try { + server::MetricCollector metric; + + // multi-threads call sqlite update may get exception('bad logic', etc), so we add a lock here + std::lock_guard meta_lock(meta_mutex_); + + //soft delete index files + ConnectorPtr->update_all( + set( + c(&TableFileSchema::file_type_) = (int) TableFileSchema::TO_DELETE, + c(&TableFileSchema::updated_time_) = utils::GetMicroSecTimeStamp()), + where( + c(&TableFileSchema::table_id_) == table_id and + c(&TableFileSchema::file_type_) == (int) TableFileSchema::INDEX)); + + //set all backup file to raw + ConnectorPtr->update_all( + set( + c(&TableFileSchema::file_type_) = (int) TableFileSchema::RAW, + c(&TableFileSchema::updated_time_) = utils::GetMicroSecTimeStamp()), + where( + c(&TableFileSchema::table_id_) == table_id and + c(&TableFileSchema::file_type_) == (int) TableFileSchema::BACKUP)); + + //set table index type to raw + ConnectorPtr->update_all( + set( + c(&TableSchema::engine_type_) = DEFAULT_ENGINE_TYPE, + c(&TableSchema::nlist_) = DEFAULT_NLIST, + c(&TableSchema::metric_type_) = DEFAULT_METRIC_TYPE), + where( + c(&TableSchema::table_id_) == table_id)); + + ENGINE_LOG_DEBUG << "Successfully drop table index, table id = " << table_id; + } catch (std::exception &e) { + return HandleException("Encounter exception when delete table index files", e.what()); + } + + return Status::OK(); +} + +Status +SqliteMetaImpl::CreatePartition(const std::string& table_id, const std::string& partition_name, const std::string& tag) { + server::MetricCollector metric; + + TableSchema table_schema; + table_schema.table_id_ = table_id; + auto status = DescribeTable(table_schema); + if (!status.ok()) { + return status; + } + + // not allow create partition under partition + if(!table_schema.owner_table_.empty()) { + return Status(DB_ERROR, "Nested partition is not allowed"); + } + + // trim side-blank of tag, only compare valid characters + // for example: " ab cd " is treated as "ab cd" + std::string valid_tag = tag; + server::StringHelpFunctions::TrimStringBlank(valid_tag); + + // not allow duplicated partition + std::string exist_partition; + GetPartitionName(table_id, valid_tag, exist_partition); + if(!exist_partition.empty()) { + return Status(DB_ERROR, "Duplicate partition is not allowed"); + } + + if (partition_name == "") { + // generate unique partition name + NextTableId(table_schema.table_id_); + } else { + table_schema.table_id_ = partition_name; + } + + table_schema.id_ = -1; + table_schema.flag_ = 0; + table_schema.created_on_ = utils::GetMicroSecTimeStamp(); + table_schema.owner_table_ = table_id; + table_schema.partition_tag_ = valid_tag; + + status = CreateTable(table_schema); + if (status.code() == DB_ALREADY_EXIST) { + return Status(DB_ALREADY_EXIST, "Partition already exists"); + } + + return status; +} + +Status +SqliteMetaImpl::DropPartition(const std::string& partition_name) { + return DropTable(partition_name); +} + +Status +SqliteMetaImpl::ShowPartitions(const std::string& table_id, std::vector& partiton_schema_array) { + try { + server::MetricCollector metric; + + auto partitions = ConnectorPtr->select(columns(&TableSchema::table_id_), + where(c(&TableSchema::owner_table_) == table_id + and c(&TableSchema::state_) != (int) TableSchema::TO_DELETE)); + for(size_t i = 0; i < partitions.size(); i++) { + std::string partition_name = std::get<0>(partitions[i]); + meta::TableSchema partition_schema; + partition_schema.table_id_ = partition_name; + DescribeTable(partition_schema); + partiton_schema_array.emplace_back(partition_schema); + } + } catch (std::exception &e) { + return HandleException("Encounter exception when show partitions", e.what()); + } + + return Status::OK(); +} + +Status +SqliteMetaImpl::GetPartitionName(const std::string& table_id, const std::string& tag, std::string& partition_name) { + try { + server::MetricCollector metric; + + // trim side-blank of tag, only compare valid characters + // for example: " ab cd " is treated as "ab cd" + std::string valid_tag = tag; + server::StringHelpFunctions::TrimStringBlank(valid_tag); + + auto name = ConnectorPtr->select(columns(&TableSchema::table_id_), + where(c(&TableSchema::owner_table_) == table_id + and c(&TableSchema::partition_tag_) == valid_tag)); + if (name.size() > 0) { + partition_name = std::get<0>(name[0]); + } else { + return Status(DB_NOT_FOUND, "Table " + table_id + "'s partition " + valid_tag + " not found"); + } + } catch (std::exception &e) { + return HandleException("Encounter exception when get partition name", e.what()); + } + + return Status::OK(); +} + +Status +SqliteMetaImpl::FilesToSearch(const std::string& table_id, + const std::vector& ids, + const DatesT& dates, + DatePartionedTableFilesSchema& files) { + files.clear(); + server::MetricCollector metric; + + try { + auto select_columns = + columns(&TableFileSchema::id_, &TableFileSchema::table_id_, &TableFileSchema::file_id_, + &TableFileSchema::file_type_, &TableFileSchema::file_size_, &TableFileSchema::row_count_, + &TableFileSchema::date_, &TableFileSchema::engine_type_); + + auto match_tableid = c(&TableFileSchema::table_id_) == table_id; + + std::vector file_types = {(int)TableFileSchema::RAW, (int)TableFileSchema::TO_INDEX, + (int)TableFileSchema::INDEX}; + auto match_type = in(&TableFileSchema::file_type_, file_types); + + TableSchema table_schema; + table_schema.table_id_ = table_id; + auto status = DescribeTable(table_schema); + if (!status.ok()) { + return status; + } + + // sqlite_orm has a bug, 'in' statement cannot handle too many elements + // so we split one query into multi-queries, this is a work-around!! + std::vector split_dates; + split_dates.push_back(DatesT()); + const size_t batch_size = 30; + for (DateT date : dates) { + DatesT& last_batch = *split_dates.rbegin(); + last_batch.push_back(date); + if (last_batch.size() > batch_size) { + split_dates.push_back(DatesT()); + } + } + + // perform query + decltype(ConnectorPtr->select(select_columns)) selected; + if (dates.empty() && ids.empty()) { + auto filter = where(match_tableid and match_type); + selected = ConnectorPtr->select(select_columns, filter); + } else if (dates.empty() && !ids.empty()) { + auto match_fileid = in(&TableFileSchema::id_, ids); + auto filter = where(match_tableid and match_fileid and match_type); + selected = ConnectorPtr->select(select_columns, filter); + } else if (!dates.empty() && ids.empty()) { + for (auto& batch_dates : split_dates) { + if (batch_dates.empty()) { + continue; + } + auto match_date = in(&TableFileSchema::date_, batch_dates); + auto filter = where(match_tableid and match_date and match_type); + auto batch_selected = ConnectorPtr->select(select_columns, filter); + for (auto& file : batch_selected) { + selected.push_back(file); + } + } + + } else if (!dates.empty() && !ids.empty()) { + for (auto& batch_dates : split_dates) { + if (batch_dates.empty()) { + continue; + } + auto match_fileid = in(&TableFileSchema::id_, ids); + auto match_date = in(&TableFileSchema::date_, batch_dates); + auto filter = where(match_tableid and match_fileid and match_date and match_type); + auto batch_selected = ConnectorPtr->select(select_columns, filter); + for (auto& file : batch_selected) { + selected.push_back(file); + } + } + } + + Status ret; + TableFileSchema table_file; + for (auto& file : selected) { + table_file.id_ = std::get<0>(file); + table_file.table_id_ = std::get<1>(file); + table_file.file_id_ = std::get<2>(file); + table_file.file_type_ = std::get<3>(file); + table_file.file_size_ = std::get<4>(file); + table_file.row_count_ = std::get<5>(file); + table_file.date_ = std::get<6>(file); + table_file.engine_type_ = std::get<7>(file); + table_file.dimension_ = table_schema.dimension_; + table_file.index_file_size_ = table_schema.index_file_size_; + table_file.nlist_ = table_schema.nlist_; + table_file.metric_type_ = table_schema.metric_type_; + + auto status = utils::GetTableFilePath(options_, table_file); + if (!status.ok()) { + ret = status; + } + + auto dateItr = files.find(table_file.date_); + if (dateItr == files.end()) { + files[table_file.date_] = TableFilesSchema(); + } + files[table_file.date_].push_back(table_file); + } + if (files.empty()) { + ENGINE_LOG_ERROR << "No file to search for table: " << table_id; + } + + if (selected.size() > 0) { + ENGINE_LOG_DEBUG << "Collect " << selected.size() << " to-search files"; + } + return ret; + } catch (std::exception& e) { + return HandleException("Encounter exception when iterate index files", e.what()); + } +} + +Status +SqliteMetaImpl::FilesToMerge(const std::string& table_id, DatePartionedTableFilesSchema& files) { + files.clear(); + + try { + server::MetricCollector metric; + + // check table existence + TableSchema table_schema; + table_schema.table_id_ = table_id; + auto status = DescribeTable(table_schema); + if (!status.ok()) { + return status; + } + + // get files to merge + auto selected = ConnectorPtr->select( + columns(&TableFileSchema::id_, &TableFileSchema::table_id_, &TableFileSchema::file_id_, + &TableFileSchema::file_type_, &TableFileSchema::file_size_, &TableFileSchema::row_count_, + &TableFileSchema::date_, &TableFileSchema::created_on_), + where(c(&TableFileSchema::file_type_) == (int)TableFileSchema::RAW and + c(&TableFileSchema::table_id_) == table_id), + order_by(&TableFileSchema::file_size_).desc()); + + Status result; + int64_t to_merge_files = 0; + for (auto& file : selected) { + TableFileSchema table_file; + table_file.file_size_ = std::get<4>(file); + if (table_file.file_size_ >= table_schema.index_file_size_) { + continue; // skip large file + } + + table_file.id_ = std::get<0>(file); + table_file.table_id_ = std::get<1>(file); + table_file.file_id_ = std::get<2>(file); + table_file.file_type_ = std::get<3>(file); + table_file.row_count_ = std::get<5>(file); + table_file.date_ = std::get<6>(file); + table_file.created_on_ = std::get<7>(file); + table_file.dimension_ = table_schema.dimension_; + table_file.index_file_size_ = table_schema.index_file_size_; + table_file.nlist_ = table_schema.nlist_; + table_file.metric_type_ = table_schema.metric_type_; + + auto status = utils::GetTableFilePath(options_, table_file); + if (!status.ok()) { + result = status; + } + + auto dateItr = files.find(table_file.date_); + if (dateItr == files.end()) { + files[table_file.date_] = TableFilesSchema(); + } + + files[table_file.date_].push_back(table_file); + to_merge_files++; + } + + if (to_merge_files > 0) { + ENGINE_LOG_TRACE << "Collect " << to_merge_files << " to-merge files"; + } + return result; + } catch (std::exception& e) { + return HandleException("Encounter exception when iterate merge files", e.what()); + } +} + Status SqliteMetaImpl::FilesToIndex(TableFilesSchema &files) { files.clear(); @@ -644,7 +1048,7 @@ SqliteMetaImpl::FilesToIndex(TableFilesSchema &files) { &TableFileSchema::engine_type_, &TableFileSchema::created_on_), where(c(&TableFileSchema::file_type_) - == (int) TableFileSchema::TO_INDEX)); + == (int) TableFileSchema::TO_INDEX)); std::map groups; TableFileSchema table_file; @@ -692,267 +1096,72 @@ SqliteMetaImpl::FilesToIndex(TableFilesSchema &files) { } Status -SqliteMetaImpl::FilesToSearch(const std::string &table_id, - const std::vector &ids, - const DatesT &dates, - DatePartionedTableFilesSchema &files) { - files.clear(); - server::MetricCollector metric; +SqliteMetaImpl::FilesByType(const std::string &table_id, + const std::vector &file_types, + std::vector &file_ids) { + if (file_types.empty()) { + return Status(DB_ERROR, "file types array is empty"); + } try { - auto select_columns = columns(&TableFileSchema::id_, - &TableFileSchema::table_id_, - &TableFileSchema::file_id_, - &TableFileSchema::file_type_, - &TableFileSchema::file_size_, - &TableFileSchema::row_count_, - &TableFileSchema::date_, - &TableFileSchema::engine_type_); + file_ids.clear(); + auto selected = ConnectorPtr->select(columns(&TableFileSchema::file_id_, + &TableFileSchema::file_type_), + where(in(&TableFileSchema::file_type_, file_types) + and c(&TableFileSchema::table_id_) == table_id)); - auto match_tableid = c(&TableFileSchema::table_id_) == table_id; - - std::vector file_types = { - (int) TableFileSchema::RAW, - (int) TableFileSchema::TO_INDEX, - (int) TableFileSchema::INDEX - }; - auto match_type = in(&TableFileSchema::file_type_, file_types); - - TableSchema table_schema; - table_schema.table_id_ = table_id; - auto status = DescribeTable(table_schema); - if (!status.ok()) { return status; } - - //sqlite_orm has a bug, 'in' statement cannot handle too many elements - //so we split one query into multi-queries, this is a work-around!! - std::vector split_dates; - split_dates.push_back(DatesT()); - const size_t batch_size = 30; - for(DateT date : dates) { - DatesT& last_batch = *split_dates.rbegin(); - last_batch.push_back(date); - if(last_batch.size() > batch_size) { - split_dates.push_back(DatesT()); - } - } - - //perform query - decltype(ConnectorPtr->select(select_columns)) selected; - if (dates.empty() && ids.empty()) { - auto filter = where(match_tableid and match_type); - selected = ConnectorPtr->select(select_columns, filter); - } else if (dates.empty() && !ids.empty()) { - auto match_fileid = in(&TableFileSchema::id_, ids); - auto filter = where(match_tableid and match_fileid and match_type); - selected = ConnectorPtr->select(select_columns, filter); - } else if (!dates.empty() && ids.empty()) { - for(auto& batch_dates : split_dates) { - if(batch_dates.empty()) { - continue; - } - auto match_date = in(&TableFileSchema::date_, batch_dates); - auto filter = where(match_tableid and match_date and match_type); - auto batch_selected = ConnectorPtr->select(select_columns, filter); - for (auto &file : batch_selected) { - selected.push_back(file); + if (selected.size() >= 1) { + int raw_count = 0, new_count = 0, new_merge_count = 0, new_index_count = 0; + int to_index_count = 0, index_count = 0, backup_count = 0; + for (auto &file : selected) { + file_ids.push_back(std::get<0>(file)); + switch (std::get<1>(file)) { + case (int) TableFileSchema::RAW:raw_count++; + break; + case (int) TableFileSchema::NEW:new_count++; + break; + case (int) TableFileSchema::NEW_MERGE:new_merge_count++; + break; + case (int) TableFileSchema::NEW_INDEX:new_index_count++; + break; + case (int) TableFileSchema::TO_INDEX:to_index_count++; + break; + case (int) TableFileSchema::INDEX:index_count++; + break; + case (int) TableFileSchema::BACKUP:backup_count++; + break; + default:break; } } - } else if (!dates.empty() && !ids.empty()) { - for(auto& batch_dates : split_dates) { - if(batch_dates.empty()) { - continue; - } - auto match_fileid = in(&TableFileSchema::id_, ids); - auto match_date = in(&TableFileSchema::date_, batch_dates); - auto filter = where(match_tableid and match_fileid and match_date and match_type); - auto batch_selected = ConnectorPtr->select(select_columns, filter); - for (auto &file : batch_selected) { - selected.push_back(file); - } - } + ENGINE_LOG_DEBUG << "Table " << table_id << " currently has raw files:" << raw_count + << " new files:" << new_count << " new_merge files:" << new_merge_count + << " new_index files:" << new_index_count << " to_index files:" << to_index_count + << " index files:" << index_count << " backup files:" << backup_count; } - - Status ret; - TableFileSchema table_file; - for (auto &file : selected) { - table_file.id_ = std::get<0>(file); - table_file.table_id_ = std::get<1>(file); - table_file.file_id_ = std::get<2>(file); - table_file.file_type_ = std::get<3>(file); - table_file.file_size_ = std::get<4>(file); - table_file.row_count_ = std::get<5>(file); - table_file.date_ = std::get<6>(file); - table_file.engine_type_ = std::get<7>(file); - table_file.dimension_ = table_schema.dimension_; - table_file.index_file_size_ = table_schema.index_file_size_; - table_file.nlist_ = table_schema.nlist_; - table_file.metric_type_ = table_schema.metric_type_; - - auto status = utils::GetTableFilePath(options_, table_file); - if (!status.ok()) { - ret = status; - } - - auto dateItr = files.find(table_file.date_); - if (dateItr == files.end()) { - files[table_file.date_] = TableFilesSchema(); - } - files[table_file.date_].push_back(table_file); - } - if (files.empty()) { - ENGINE_LOG_ERROR << "No file to search for table: " << table_id; - } - - if (selected.size() > 0) { - ENGINE_LOG_DEBUG << "Collect " << selected.size() << " to-search files"; - } - return ret; } catch (std::exception &e) { - return HandleException("Encounter exception when iterate index files", e.what()); + return HandleException("Encounter exception when check non index files", e.what()); } + return Status::OK(); } -Status -SqliteMetaImpl::FilesToMerge(const std::string &table_id, - DatePartionedTableFilesSchema &files) { - files.clear(); - - try { - server::MetricCollector metric; - - //check table existence - TableSchema table_schema; - table_schema.table_id_ = table_id; - auto status = DescribeTable(table_schema); - if (!status.ok()) { - return status; - } - - //get files to merge - auto selected = ConnectorPtr->select(columns(&TableFileSchema::id_, - &TableFileSchema::table_id_, - &TableFileSchema::file_id_, - &TableFileSchema::file_type_, - &TableFileSchema::file_size_, - &TableFileSchema::row_count_, - &TableFileSchema::date_, - &TableFileSchema::created_on_), - where(c(&TableFileSchema::file_type_) == (int) TableFileSchema::RAW and - c(&TableFileSchema::table_id_) == table_id), - order_by(&TableFileSchema::file_size_).desc()); - - Status result; - for (auto &file : selected) { - TableFileSchema table_file; - table_file.file_size_ = std::get<4>(file); - if (table_file.file_size_ >= table_schema.index_file_size_) { - continue;//skip large file - } - - table_file.id_ = std::get<0>(file); - table_file.table_id_ = std::get<1>(file); - table_file.file_id_ = std::get<2>(file); - table_file.file_type_ = std::get<3>(file); - table_file.row_count_ = std::get<5>(file); - table_file.date_ = std::get<6>(file); - table_file.created_on_ = std::get<7>(file); - table_file.dimension_ = table_schema.dimension_; - table_file.index_file_size_ = table_schema.index_file_size_; - table_file.nlist_ = table_schema.nlist_; - table_file.metric_type_ = table_schema.metric_type_; - - auto status = utils::GetTableFilePath(options_, table_file); - if (!status.ok()) { - result = status; - } - - auto dateItr = files.find(table_file.date_); - if (dateItr == files.end()) { - files[table_file.date_] = TableFilesSchema(); - } - files[table_file.date_].push_back(table_file); - } - - if (selected.size() > 0) { - ENGINE_LOG_DEBUG << "Collect " << selected.size() << " to-merge files"; - } - return result; - } catch (std::exception &e) { - return HandleException("Encounter exception when iterate merge files", e.what()); - } -} - -Status -SqliteMetaImpl::GetTableFiles(const std::string &table_id, - const std::vector &ids, - TableFilesSchema &table_files) { - try { - table_files.clear(); - auto files = ConnectorPtr->select(columns(&TableFileSchema::id_, - &TableFileSchema::file_id_, - &TableFileSchema::file_type_, - &TableFileSchema::file_size_, - &TableFileSchema::row_count_, - &TableFileSchema::date_, - &TableFileSchema::engine_type_, - &TableFileSchema::created_on_), - where(c(&TableFileSchema::table_id_) == table_id and - in(&TableFileSchema::id_, ids) and - c(&TableFileSchema::file_type_) != (int) TableFileSchema::TO_DELETE)); - - TableSchema table_schema; - table_schema.table_id_ = table_id; - auto status = DescribeTable(table_schema); - if (!status.ok()) { - return status; - } - - Status result; - for (auto &file : files) { - TableFileSchema file_schema; - file_schema.table_id_ = table_id; - file_schema.id_ = std::get<0>(file); - file_schema.file_id_ = std::get<1>(file); - file_schema.file_type_ = std::get<2>(file); - file_schema.file_size_ = std::get<3>(file); - file_schema.row_count_ = std::get<4>(file); - file_schema.date_ = std::get<5>(file); - file_schema.engine_type_ = std::get<6>(file); - file_schema.created_on_ = std::get<7>(file); - file_schema.dimension_ = table_schema.dimension_; - file_schema.index_file_size_ = table_schema.index_file_size_; - file_schema.nlist_ = table_schema.nlist_; - file_schema.metric_type_ = table_schema.metric_type_; - - utils::GetTableFilePath(options_, file_schema); - - table_files.emplace_back(file_schema); - } - - ENGINE_LOG_DEBUG << "Get table files by id"; - return result; - } catch (std::exception &e) { - return HandleException("Encounter exception when lookup table files", e.what()); - } -} // TODO(myh): Support swap to cloud storage Status SqliteMetaImpl::Archive() { - auto &criterias = options_.archive_conf_.GetCriterias(); + auto& criterias = options_.archive_conf_.GetCriterias(); if (criterias.size() == 0) { return Status::OK(); } for (auto kv : criterias) { - auto &criteria = kv.first; - auto &limit = kv.second; + auto& criteria = kv.first; + auto& limit = kv.second; if (criteria == engine::ARCHIVE_CONF_DAYS) { int64_t usecs = limit * D_SEC * US_PS; int64_t now = utils::GetMicroSecTimeStamp(); try { - //multi-threads call sqlite update may get exception('bad logic', etc), so we add a lock here + // multi-threads call sqlite update may get exception('bad logic', etc), so we add a lock here std::lock_guard meta_lock(meta_mutex_); ConnectorPtr->update_all( @@ -960,7 +1169,7 @@ SqliteMetaImpl::Archive() { c(&TableFileSchema::file_type_) = (int) TableFileSchema::TO_DELETE), where( c(&TableFileSchema::created_on_) < (int64_t) (now - usecs) and - c(&TableFileSchema::file_type_) != (int) TableFileSchema::TO_DELETE)); + c(&TableFileSchema::file_type_) != (int) TableFileSchema::TO_DELETE)); } catch (std::exception &e) { return HandleException("Encounter exception when update table files", e.what()); } @@ -971,7 +1180,7 @@ SqliteMetaImpl::Archive() { uint64_t sum = 0; Size(sum); - int64_t to_delete = (int64_t) sum - limit * G; + int64_t to_delete = (int64_t)sum - limit * G; DiscardFiles(to_delete); ENGINE_LOG_DEBUG << "Archive files to free disk"; @@ -982,20 +1191,206 @@ SqliteMetaImpl::Archive() { } Status -SqliteMetaImpl::Size(uint64_t &result) { +SqliteMetaImpl::Size(uint64_t& result) { result = 0; try { auto selected = ConnectorPtr->select(columns(sum(&TableFileSchema::file_size_)), - where( - c(&TableFileSchema::file_type_) != (int) TableFileSchema::TO_DELETE)); - for (auto &total_size : selected) { + where(c(&TableFileSchema::file_type_) != (int)TableFileSchema::TO_DELETE)); + for (auto& total_size : selected) { if (!std::get<0>(total_size)) { continue; } - result += (uint64_t) (*std::get<0>(total_size)); + result += (uint64_t)(*std::get<0>(total_size)); + } + } catch (std::exception& e) { + return HandleException("Encounter exception when calculte db size", e.what()); + } + + return Status::OK(); +} + +Status +SqliteMetaImpl::CleanUp() { + try { + server::MetricCollector metric; + + // multi-threads call sqlite update may get exception('bad logic', etc), so we add a lock here + std::lock_guard meta_lock(meta_mutex_); + + std::vector file_types = { + (int) TableFileSchema::NEW, + (int) TableFileSchema::NEW_INDEX, + (int) TableFileSchema::NEW_MERGE + }; + auto files = + ConnectorPtr->select(columns(&TableFileSchema::id_), where(in(&TableFileSchema::file_type_, file_types))); + + auto commited = ConnectorPtr->transaction([&]() mutable { + for (auto &file : files) { + ENGINE_LOG_DEBUG << "Remove table file type as NEW"; + ConnectorPtr->remove(std::get<0>(file)); + } + return true; + }); + + if (!commited) { + return HandleException("CleanUp error: sqlite transaction failed"); + } + + if (files.size() > 0) { + ENGINE_LOG_DEBUG << "Clean " << files.size() << " files"; } } catch (std::exception &e) { - return HandleException("Encounter exception when calculte db size", e.what()); + return HandleException("Encounter exception when clean table file", e.what()); + } + + return Status::OK(); +} + +Status +SqliteMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) { + auto now = utils::GetMicroSecTimeStamp(); + std::set table_ids; + + // remove to_delete files + try { + server::MetricCollector metric; + + // multi-threads call sqlite update may get exception('bad logic', etc), so we add a lock here + std::lock_guard meta_lock(meta_mutex_); + + auto files = ConnectorPtr->select(columns(&TableFileSchema::id_, + &TableFileSchema::table_id_, + &TableFileSchema::file_id_, + &TableFileSchema::date_), + where( + c(&TableFileSchema::file_type_) == + (int) TableFileSchema::TO_DELETE + and + c(&TableFileSchema::updated_time_) + < now - seconds * US_PS)); + + auto commited = ConnectorPtr->transaction([&]() mutable { + TableFileSchema table_file; + for (auto& file : files) { + table_file.id_ = std::get<0>(file); + table_file.table_id_ = std::get<1>(file); + table_file.file_id_ = std::get<2>(file); + table_file.date_ = std::get<3>(file); + + utils::DeleteTableFilePath(options_, table_file); + ENGINE_LOG_DEBUG << "Removing file id:" << table_file.file_id_ << " location:" << table_file.location_; + ConnectorPtr->remove(table_file.id_); + + table_ids.insert(table_file.table_id_); + } + return true; + }); + + if (!commited) { + return HandleException("CleanUpFilesWithTTL error: sqlite transaction failed"); + } + + if (files.size() > 0) { + ENGINE_LOG_DEBUG << "Clean " << files.size() << " files deleted in " << seconds << " seconds"; + } + } catch (std::exception& e) { + return HandleException("Encounter exception when clean table files", e.what()); + } + + // remove to_delete tables + try { + server::MetricCollector metric; + + // multi-threads call sqlite update may get exception('bad logic', etc), so we add a lock here + std::lock_guard meta_lock(meta_mutex_); + + auto tables = ConnectorPtr->select(columns(&TableSchema::id_, &TableSchema::table_id_), + where(c(&TableSchema::state_) == (int)TableSchema::TO_DELETE)); + + auto commited = ConnectorPtr->transaction([&]() mutable { + for (auto& table : tables) { + utils::DeleteTablePath(options_, std::get<1>(table), false); // only delete empty folder + ConnectorPtr->remove(std::get<0>(table)); + } + + return true; + }); + + if (!commited) { + return HandleException("CleanUpFilesWithTTL error: sqlite transaction failed"); + } + + if (tables.size() > 0) { + ENGINE_LOG_DEBUG << "Remove " << tables.size() << " tables from meta"; + } + } catch (std::exception& e) { + return HandleException("Encounter exception when clean table files", e.what()); + } + + // remove deleted table folder + // don't remove table folder until all its files has been deleted + try { + server::MetricCollector metric; + + int64_t remove_tables = 0; + for (auto& table_id : table_ids) { + auto selected = ConnectorPtr->select(columns(&TableFileSchema::file_id_), + where(c(&TableFileSchema::table_id_) == table_id)); + if (selected.size() == 0) { + utils::DeleteTablePath(options_, table_id); + remove_tables++; + } + } + + if (remove_tables) { + ENGINE_LOG_DEBUG << "Remove " << remove_tables << " tables folder"; + } + } catch (std::exception& e) { + return HandleException("Encounter exception when delete table folder", e.what()); + } + + return Status::OK(); +} + +Status +SqliteMetaImpl::Count(const std::string &table_id, uint64_t &result) { + try { + server::MetricCollector metric; + + std::vector file_types = {(int)TableFileSchema::RAW, (int)TableFileSchema::TO_INDEX, + (int)TableFileSchema::INDEX}; + auto selected = ConnectorPtr->select( + columns(&TableFileSchema::row_count_), + where(in(&TableFileSchema::file_type_, file_types) and c(&TableFileSchema::table_id_) == table_id)); + + TableSchema table_schema; + table_schema.table_id_ = table_id; + auto status = DescribeTable(table_schema); + + if (!status.ok()) { + return status; + } + + result = 0; + for (auto& file : selected) { + result += std::get<0>(file); + } + } catch (std::exception& e) { + return HandleException("Encounter exception when calculate table file size", e.what()); + } + return Status::OK(); +} + +Status +SqliteMetaImpl::DropAll() { + ENGINE_LOG_DEBUG << "Drop all sqlite meta"; + + try { + ConnectorPtr->drop_table(META_TABLES); + ConnectorPtr->drop_table(META_TABLEFILES); + } catch (std::exception& e) { + return HandleException("Encounter exception when drop all meta", e.what()); } return Status::OK(); @@ -1019,7 +1414,7 @@ SqliteMetaImpl::DiscardFiles(int64_t to_discard_size) { auto selected = ConnectorPtr->select(columns(&TableFileSchema::id_, &TableFileSchema::file_size_), where(c(&TableFileSchema::file_type_) - != (int) TableFileSchema::TO_DELETE), + != (int) TableFileSchema::TO_DELETE), order_by(&TableFileSchema::id_), limit(10)); @@ -1060,293 +1455,6 @@ SqliteMetaImpl::DiscardFiles(int64_t to_discard_size) { return DiscardFiles(to_discard_size); } -Status -SqliteMetaImpl::UpdateTableFile(TableFileSchema &file_schema) { - file_schema.updated_time_ = utils::GetMicroSecTimeStamp(); - try { - server::MetricCollector metric; - - //multi-threads call sqlite update may get exception('bad logic', etc), so we add a lock here - std::lock_guard meta_lock(meta_mutex_); - - auto tables = ConnectorPtr->select(columns(&TableSchema::state_), - where(c(&TableSchema::table_id_) == file_schema.table_id_)); - - //if the table has been deleted, just mark the table file as TO_DELETE - //clean thread will delete the file later - if (tables.size() < 1 || std::get<0>(tables[0]) == (int) TableSchema::TO_DELETE) { - file_schema.file_type_ = TableFileSchema::TO_DELETE; - } - - ConnectorPtr->update(file_schema); - - ENGINE_LOG_DEBUG << "Update single table file, file id = " << file_schema.file_id_; - } catch (std::exception &e) { - std::string msg = "Exception update table file: table_id = " + file_schema.table_id_ - + " file_id = " + file_schema.file_id_; - return HandleException(msg, e.what()); - } - return Status::OK(); -} - -Status -SqliteMetaImpl::UpdateTableFilesToIndex(const std::string &table_id) { - try { - server::MetricCollector metric; - - //multi-threads call sqlite update may get exception('bad logic', etc), so we add a lock here - std::lock_guard meta_lock(meta_mutex_); - - ConnectorPtr->update_all( - set( - c(&TableFileSchema::file_type_) = (int) TableFileSchema::TO_INDEX), - where( - c(&TableFileSchema::table_id_) == table_id and - c(&TableFileSchema::file_type_) == (int) TableFileSchema::RAW)); - - ENGINE_LOG_DEBUG << "Update files to to_index, table id = " << table_id; - } catch (std::exception &e) { - return HandleException("Encounter exception when update table files to to_index", e.what()); - } - - return Status::OK(); -} - -Status -SqliteMetaImpl::UpdateTableFiles(TableFilesSchema &files) { - try { - server::MetricCollector metric; - - //multi-threads call sqlite update may get exception('bad logic', etc), so we add a lock here - std::lock_guard meta_lock(meta_mutex_); - - std::map has_tables; - for (auto &file : files) { - if (has_tables.find(file.table_id_) != has_tables.end()) { - continue; - } - auto tables = ConnectorPtr->select(columns(&TableSchema::id_), - where(c(&TableSchema::table_id_) == file.table_id_ - and c(&TableSchema::state_) != (int) TableSchema::TO_DELETE)); - if (tables.size() >= 1) { - has_tables[file.table_id_] = true; - } else { - has_tables[file.table_id_] = false; - } - } - - auto commited = ConnectorPtr->transaction([&]() mutable { - for (auto &file : files) { - if (!has_tables[file.table_id_]) { - file.file_type_ = TableFileSchema::TO_DELETE; - } - - file.updated_time_ = utils::GetMicroSecTimeStamp(); - ConnectorPtr->update(file); - } - return true; - }); - - if (!commited) { - return HandleException("UpdateTableFiles error: sqlite transaction failed"); - } - - ENGINE_LOG_DEBUG << "Update " << files.size() << " table files"; - } catch (std::exception &e) { - return HandleException("Encounter exception when update table files", e.what()); - } - return Status::OK(); -} - -Status -SqliteMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) { - auto now = utils::GetMicroSecTimeStamp(); - std::set table_ids; - - //remove to_delete files - try { - server::MetricCollector metric; - - //multi-threads call sqlite update may get exception('bad logic', etc), so we add a lock here - std::lock_guard meta_lock(meta_mutex_); - - auto files = ConnectorPtr->select(columns(&TableFileSchema::id_, - &TableFileSchema::table_id_, - &TableFileSchema::file_id_, - &TableFileSchema::date_), - where( - c(&TableFileSchema::file_type_) == - (int) TableFileSchema::TO_DELETE - and - c(&TableFileSchema::updated_time_) - < now - seconds * US_PS)); - - auto commited = ConnectorPtr->transaction([&]() mutable { - TableFileSchema table_file; - for (auto &file : files) { - table_file.id_ = std::get<0>(file); - table_file.table_id_ = std::get<1>(file); - table_file.file_id_ = std::get<2>(file); - table_file.date_ = std::get<3>(file); - - utils::DeleteTableFilePath(options_, table_file); - ENGINE_LOG_DEBUG << "Removing file id:" << table_file.file_id_ << " location:" << table_file.location_; - ConnectorPtr->remove(table_file.id_); - - table_ids.insert(table_file.table_id_); - } - return true; - }); - - if (!commited) { - return HandleException("CleanUpFilesWithTTL error: sqlite transaction failed"); - } - - if (files.size() > 0) { - ENGINE_LOG_DEBUG << "Clean " << files.size() << " files deleted in " << seconds << " seconds"; - } - } catch (std::exception &e) { - return HandleException("Encounter exception when clean table files", e.what()); - } - - //remove to_delete tables - try { - server::MetricCollector metric; - - //multi-threads call sqlite update may get exception('bad logic', etc), so we add a lock here - std::lock_guard meta_lock(meta_mutex_); - - auto tables = ConnectorPtr->select(columns(&TableSchema::id_, - &TableSchema::table_id_), - where(c(&TableSchema::state_) == (int) TableSchema::TO_DELETE)); - - auto commited = ConnectorPtr->transaction([&]() mutable { - for (auto &table : tables) { - utils::DeleteTablePath(options_, std::get<1>(table), false);//only delete empty folder - ConnectorPtr->remove(std::get<0>(table)); - } - - return true; - }); - - if (!commited) { - return HandleException("CleanUpFilesWithTTL error: sqlite transaction failed"); - } - - if (tables.size() > 0) { - ENGINE_LOG_DEBUG << "Remove " << tables.size() << " tables from meta"; - } - } catch (std::exception &e) { - return HandleException("Encounter exception when clean table files", e.what()); - } - - //remove deleted table folder - //don't remove table folder until all its files has been deleted - try { - server::MetricCollector metric; - - for (auto &table_id : table_ids) { - auto selected = ConnectorPtr->select(columns(&TableFileSchema::file_id_), - where(c(&TableFileSchema::table_id_) == table_id)); - if (selected.size() == 0) { - utils::DeleteTablePath(options_, table_id); - } - } - - if (table_ids.size() > 0) { - ENGINE_LOG_DEBUG << "Remove " << table_ids.size() << " tables folder"; - } - } catch (std::exception &e) { - return HandleException("Encounter exception when delete table folder", e.what()); - } - - return Status::OK(); -} - -Status -SqliteMetaImpl::CleanUp() { - try { - server::MetricCollector metric; - - //multi-threads call sqlite update may get exception('bad logic', etc), so we add a lock here - std::lock_guard meta_lock(meta_mutex_); - - std::vector file_types = { - (int) TableFileSchema::NEW, - (int) TableFileSchema::NEW_INDEX, - (int) TableFileSchema::NEW_MERGE - }; - auto files = - ConnectorPtr->select(columns(&TableFileSchema::id_), where(in(&TableFileSchema::file_type_, file_types))); - - auto commited = ConnectorPtr->transaction([&]() mutable { - for (auto &file : files) { - ENGINE_LOG_DEBUG << "Remove table file type as NEW"; - ConnectorPtr->remove(std::get<0>(file)); - } - return true; - }); - - if (!commited) { - return HandleException("CleanUp error: sqlite transaction failed"); - } - - if (files.size() > 0) { - ENGINE_LOG_DEBUG << "Clean " << files.size() << " files"; - } - } catch (std::exception &e) { - return HandleException("Encounter exception when clean table file", e.what()); - } - - return Status::OK(); -} - -Status -SqliteMetaImpl::Count(const std::string &table_id, uint64_t &result) { - try { - server::MetricCollector metric; - - std::vector file_types = { - (int) TableFileSchema::RAW, - (int) TableFileSchema::TO_INDEX, - (int) TableFileSchema::INDEX - }; - auto selected = ConnectorPtr->select(columns(&TableFileSchema::row_count_), - where(in(&TableFileSchema::file_type_, file_types) - and c(&TableFileSchema::table_id_) == table_id)); - - TableSchema table_schema; - table_schema.table_id_ = table_id; - auto status = DescribeTable(table_schema); - - if (!status.ok()) { - return status; - } - - result = 0; - for (auto &file : selected) { - result += std::get<0>(file); - } - } catch (std::exception &e) { - return HandleException("Encounter exception when calculate table file size", e.what()); - } - return Status::OK(); -} - -Status -SqliteMetaImpl::DropAll() { - ENGINE_LOG_DEBUG << "Drop all sqlite meta"; - - try { - ConnectorPtr->drop_table(META_TABLES); - ConnectorPtr->drop_table(META_TABLEFILES); - } catch (std::exception &e) { - return HandleException("Encounter exception when drop all meta", e.what()); - } - - return Status::OK(); -} - } // namespace meta } // namespace engine } // namespace milvus diff --git a/core/src/db/meta/SqliteMetaImpl.h b/core/src/db/meta/SqliteMetaImpl.h index 0fc3f3c4ba..84d97ed49d 100644 --- a/core/src/db/meta/SqliteMetaImpl.h +++ b/core/src/db/meta/SqliteMetaImpl.h @@ -49,7 +49,7 @@ class SqliteMetaImpl : public Meta { AllTables(std::vector& table_schema_array) override; Status - DeleteTable(const std::string& table_id) override; + DropTable(const std::string& table_id) override; Status DeleteTableFiles(const std::string& table_id) override; @@ -58,21 +58,26 @@ class SqliteMetaImpl : public Meta { CreateTableFile(TableFileSchema& file_schema) override; Status - DropPartitionsByDates(const std::string& table_id, const DatesT& dates) override; + DropDataByDate(const std::string& table_id, const DatesT& dates) override; Status GetTableFiles(const std::string& table_id, const std::vector& ids, TableFilesSchema& table_files) override; - Status - FilesByType(const std::string& table_id, const std::vector& file_types, - std::vector& file_ids) override; - Status UpdateTableIndex(const std::string& table_id, const TableIndex& index) override; Status UpdateTableFlag(const std::string& table_id, int64_t flag) override; + Status + UpdateTableFile(TableFileSchema& file_schema) override; + + Status + UpdateTableFilesToIndex(const std::string& table_id) override; + + Status + UpdateTableFiles(TableFilesSchema& files) override; + Status DescribeTableIndex(const std::string& table_id, TableIndex& index) override; @@ -80,13 +85,16 @@ class SqliteMetaImpl : public Meta { DropTableIndex(const std::string& table_id) override; Status - UpdateTableFilesToIndex(const std::string& table_id) override; + CreatePartition(const std::string& table_id, const std::string& partition_name, const std::string& tag) override; Status - UpdateTableFile(TableFileSchema& file_schema) override; + DropPartition(const std::string& partition_name) override; Status - UpdateTableFiles(TableFilesSchema& files) override; + ShowPartitions(const std::string& table_id, std::vector& partiton_schema_array) override; + + Status + GetPartitionName(const std::string& table_id, const std::string& tag, std::string& partition_name) override; Status FilesToSearch(const std::string& table_id, const std::vector& ids, const DatesT& dates, @@ -99,11 +107,15 @@ class SqliteMetaImpl : public Meta { FilesToIndex(TableFilesSchema&) override; Status - Archive() override; + FilesByType(const std::string& table_id, const std::vector& file_types, + std::vector& file_ids) override; Status Size(uint64_t& result) override; + Status + Archive() override; + Status CleanUp() override; diff --git a/core/src/external/easyloggingpp/easylogging++.cc b/core/src/external/easyloggingpp/easylogging++.cc deleted file mode 100644 index 0c3bb0d375..0000000000 --- a/core/src/external/easyloggingpp/easylogging++.cc +++ /dev/null @@ -1,3112 +0,0 @@ -// -// Bismillah ar-Rahmaan ar-Raheem -// -// Easylogging++ v9.96.7 -// Cross-platform logging library for C++ applications -// -// Copyright (c) 2012-2018 Zuhd Web Services -// Copyright (c) 2012-2018 @abumusamq -// -// This library is released under the MIT Licence. -// https://github.com/zuhd-org/easyloggingpp/blob/master/LICENSE -// -// https://zuhd.org -// http://muflihun.com -// - -#include "easylogging++.h" - -#if defined(AUTO_INITIALIZE_EASYLOGGINGPP) -INITIALIZE_EASYLOGGINGPP -#endif - -namespace el { - -// el::base -namespace base { -// el::base::consts -namespace consts { - -// Level log values - These are values that are replaced in place of %level format specifier -// Extra spaces after format specifiers are only for readability purposes in log files -static const base::type::char_t* kInfoLevelLogValue = ELPP_LITERAL("INFO"); -static const base::type::char_t* kDebugLevelLogValue = ELPP_LITERAL("DEBUG"); -static const base::type::char_t* kWarningLevelLogValue = ELPP_LITERAL("WARNING"); -static const base::type::char_t* kErrorLevelLogValue = ELPP_LITERAL("ERROR"); -static const base::type::char_t* kFatalLevelLogValue = ELPP_LITERAL("FATAL"); -static const base::type::char_t* kVerboseLevelLogValue = - ELPP_LITERAL("VERBOSE"); // will become VERBOSE-x where x = verbose level -static const base::type::char_t* kTraceLevelLogValue = ELPP_LITERAL("TRACE"); -static const base::type::char_t* kInfoLevelShortLogValue = ELPP_LITERAL("I"); -static const base::type::char_t* kDebugLevelShortLogValue = ELPP_LITERAL("D"); -static const base::type::char_t* kWarningLevelShortLogValue = ELPP_LITERAL("W"); -static const base::type::char_t* kErrorLevelShortLogValue = ELPP_LITERAL("E"); -static const base::type::char_t* kFatalLevelShortLogValue = ELPP_LITERAL("F"); -static const base::type::char_t* kVerboseLevelShortLogValue = ELPP_LITERAL("V"); -static const base::type::char_t* kTraceLevelShortLogValue = ELPP_LITERAL("T"); -// Format specifiers - These are used to define log format -static const base::type::char_t* kAppNameFormatSpecifier = ELPP_LITERAL("%app"); -static const base::type::char_t* kLoggerIdFormatSpecifier = ELPP_LITERAL("%logger"); -static const base::type::char_t* kThreadIdFormatSpecifier = ELPP_LITERAL("%thread"); -static const base::type::char_t* kSeverityLevelFormatSpecifier = ELPP_LITERAL("%level"); -static const base::type::char_t* kSeverityLevelShortFormatSpecifier = ELPP_LITERAL("%levshort"); -static const base::type::char_t* kDateTimeFormatSpecifier = ELPP_LITERAL("%datetime"); -static const base::type::char_t* kLogFileFormatSpecifier = ELPP_LITERAL("%file"); -static const base::type::char_t* kLogFileBaseFormatSpecifier = ELPP_LITERAL("%fbase"); -static const base::type::char_t* kLogLineFormatSpecifier = ELPP_LITERAL("%line"); -static const base::type::char_t* kLogLocationFormatSpecifier = ELPP_LITERAL("%loc"); -static const base::type::char_t* kLogFunctionFormatSpecifier = ELPP_LITERAL("%func"); -static const base::type::char_t* kCurrentUserFormatSpecifier = ELPP_LITERAL("%user"); -static const base::type::char_t* kCurrentHostFormatSpecifier = ELPP_LITERAL("%host"); -static const base::type::char_t* kMessageFormatSpecifier = ELPP_LITERAL("%msg"); -static const base::type::char_t* kVerboseLevelFormatSpecifier = ELPP_LITERAL("%vlevel"); -static const char* kDateTimeFormatSpecifierForFilename = "%datetime"; -// Date/time -static const char* kDays[7] = { "Sundayaaa", "Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday" }; -static const char* kDaysAbbrev[7] = { "Sun", "Mon", "Tue", "Wed", "Thu", "Fri", "Sat" }; -static const char* kMonths[12] = { "January", "February", "March", "Apri", "May", "June", "July", "August", - "September", "October", "November", "December" - }; -static const char* kMonthsAbbrev[12] = { "Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec" }; -static const char* kDefaultDateTimeFormat = "%Y-%M-%d %H:%m:%s,%g"; -static const char* kDefaultDateTimeFormatInFilename = "%Y-%M-%d_%H-%m"; -static const int kYearBase = 1900; -static const char* kAm = "AM"; -static const char* kPm = "PM"; -// Miscellaneous constants - -static const char* kNullPointer = "nullptr"; -#if ELPP_VARIADIC_TEMPLATES_SUPPORTED -#endif // ELPP_VARIADIC_TEMPLATES_SUPPORTED -static const base::type::VerboseLevel kMaxVerboseLevel = 9; -static const char* kUnknownUser = "user"; -static const char* kUnknownHost = "unknown-host"; - - -//---------------- DEFAULT LOG FILE ----------------------- - -#if defined(ELPP_NO_DEFAULT_LOG_FILE) -# if ELPP_OS_UNIX -static const char* kDefaultLogFile = "/dev/null"; -# elif ELPP_OS_WINDOWS -static const char* kDefaultLogFile = "nul"; -# endif // ELPP_OS_UNIX -#elif defined(ELPP_DEFAULT_LOG_FILE) -static const char* kDefaultLogFile = ELPP_DEFAULT_LOG_FILE; -#else -static const char* kDefaultLogFile = "myeasylog.log"; -#endif // defined(ELPP_NO_DEFAULT_LOG_FILE) - - -#if !defined(ELPP_DISABLE_LOG_FILE_FROM_ARG) -static const char* kDefaultLogFileParam = "--default-log-file"; -#endif // !defined(ELPP_DISABLE_LOG_FILE_FROM_ARG) -#if defined(ELPP_LOGGING_FLAGS_FROM_ARG) -static const char* kLoggingFlagsParam = "--logging-flags"; -#endif // defined(ELPP_LOGGING_FLAGS_FROM_ARG) -static const char* kValidLoggerIdSymbols = - "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-._"; -static const char* kConfigurationComment = "##"; -static const char* kConfigurationLevel = "*"; -static const char* kConfigurationLoggerId = "--"; -} -// el::base::utils -namespace utils { - -/// @brief Aborts application due with user-defined status -static void abort(int status, const std::string& reason) { - // Both status and reason params are there for debugging with tools like gdb etc - ELPP_UNUSED(status); - ELPP_UNUSED(reason); -#if defined(ELPP_COMPILER_MSVC) && defined(_M_IX86) && defined(_DEBUG) - // Ignore msvc critical error dialog - break instead (on debug mode) - _asm int 3 -#else - ::abort(); -#endif // defined(ELPP_COMPILER_MSVC) && defined(_M_IX86) && defined(_DEBUG) -} - -} // namespace utils -} // namespace base - -// el - -// LevelHelper - -const char* LevelHelper::convertToString(Level level) { - // Do not use switch over strongly typed enums because Intel C++ compilers dont support them yet. - if (level == Level::Global) return "GLOBAL"; - if (level == Level::Debug) return "DEBUG"; - if (level == Level::Info) return "INFO"; - if (level == Level::Warning) return "WARNING"; - if (level == Level::Error) return "ERROR"; - if (level == Level::Fatal) return "FATAL"; - if (level == Level::Verbose) return "VERBOSE"; - if (level == Level::Trace) return "TRACE"; - return "UNKNOWN"; -} - -struct StringToLevelItem { - const char* levelString; - Level level; -}; - -static struct StringToLevelItem stringToLevelMap[] = { - { "global", Level::Global }, - { "debug", Level::Debug }, - { "info", Level::Info }, - { "warning", Level::Warning }, - { "error", Level::Error }, - { "fatal", Level::Fatal }, - { "verbose", Level::Verbose }, - { "trace", Level::Trace } -}; - -Level LevelHelper::convertFromString(const char* levelStr) { - for (auto& item : stringToLevelMap) { - if (base::utils::Str::cStringCaseEq(levelStr, item.levelString)) { - return item.level; - } - } - return Level::Unknown; -} - -void LevelHelper::forEachLevel(base::type::EnumType* startIndex, const std::function& fn) { - base::type::EnumType lIndexMax = LevelHelper::kMaxValid; - do { - if (fn()) { - break; - } - *startIndex = static_cast(*startIndex << 1); - } while (*startIndex <= lIndexMax); -} - -// ConfigurationTypeHelper - -const char* ConfigurationTypeHelper::convertToString(ConfigurationType configurationType) { - // Do not use switch over strongly typed enums because Intel C++ compilers dont support them yet. - if (configurationType == ConfigurationType::Enabled) return "ENABLED"; - if (configurationType == ConfigurationType::Filename) return "FILENAME"; - if (configurationType == ConfigurationType::Format) return "FORMAT"; - if (configurationType == ConfigurationType::ToFile) return "TO_FILE"; - if (configurationType == ConfigurationType::ToStandardOutput) return "TO_STANDARD_OUTPUT"; - if (configurationType == ConfigurationType::SubsecondPrecision) return "SUBSECOND_PRECISION"; - if (configurationType == ConfigurationType::PerformanceTracking) return "PERFORMANCE_TRACKING"; - if (configurationType == ConfigurationType::MaxLogFileSize) return "MAX_LOG_FILE_SIZE"; - if (configurationType == ConfigurationType::LogFlushThreshold) return "LOG_FLUSH_THRESHOLD"; - return "UNKNOWN"; -} - -struct ConfigurationStringToTypeItem { - const char* configString; - ConfigurationType configType; -}; - -static struct ConfigurationStringToTypeItem configStringToTypeMap[] = { - { "enabled", ConfigurationType::Enabled }, - { "to_file", ConfigurationType::ToFile }, - { "to_standard_output", ConfigurationType::ToStandardOutput }, - { "format", ConfigurationType::Format }, - { "filename", ConfigurationType::Filename }, - { "subsecond_precision", ConfigurationType::SubsecondPrecision }, - { "milliseconds_width", ConfigurationType::MillisecondsWidth }, - { "performance_tracking", ConfigurationType::PerformanceTracking }, - { "max_log_file_size", ConfigurationType::MaxLogFileSize }, - { "log_flush_threshold", ConfigurationType::LogFlushThreshold }, -}; - -ConfigurationType ConfigurationTypeHelper::convertFromString(const char* configStr) { - for (auto& item : configStringToTypeMap) { - if (base::utils::Str::cStringCaseEq(configStr, item.configString)) { - return item.configType; - } - } - return ConfigurationType::Unknown; -} - -void ConfigurationTypeHelper::forEachConfigType(base::type::EnumType* startIndex, const std::function& fn) { - base::type::EnumType cIndexMax = ConfigurationTypeHelper::kMaxValid; - do { - if (fn()) { - break; - } - *startIndex = static_cast(*startIndex << 1); - } while (*startIndex <= cIndexMax); -} - -// Configuration - -Configuration::Configuration(const Configuration& c) : - m_level(c.m_level), - m_configurationType(c.m_configurationType), - m_value(c.m_value) { -} - -Configuration& Configuration::operator=(const Configuration& c) { - if (&c != this) { - m_level = c.m_level; - m_configurationType = c.m_configurationType; - m_value = c.m_value; - } - return *this; -} - -/// @brief Full constructor used to sets value of configuration -Configuration::Configuration(Level level, ConfigurationType configurationType, const std::string& value) : - m_level(level), - m_configurationType(configurationType), - m_value(value) { -} - -void Configuration::log(el::base::type::ostream_t& os) const { - os << LevelHelper::convertToString(m_level) - << ELPP_LITERAL(" ") << ConfigurationTypeHelper::convertToString(m_configurationType) - << ELPP_LITERAL(" = ") << m_value.c_str(); -} - -/// @brief Used to find configuration from configuration (pointers) repository. Avoid using it. -Configuration::Predicate::Predicate(Level level, ConfigurationType configurationType) : - m_level(level), - m_configurationType(configurationType) { -} - -bool Configuration::Predicate::operator()(const Configuration* conf) const { - return ((conf != nullptr) && (conf->level() == m_level) && (conf->configurationType() == m_configurationType)); -} - -// Configurations - -Configurations::Configurations(void) : - m_configurationFile(std::string()), - m_isFromFile(false) { -} - -Configurations::Configurations(const std::string& configurationFile, bool useDefaultsForRemaining, - Configurations* base) : - m_configurationFile(configurationFile), - m_isFromFile(false) { - parseFromFile(configurationFile, base); - if (useDefaultsForRemaining) { - setRemainingToDefault(); - } -} - -bool Configurations::parseFromFile(const std::string& configurationFile, Configurations* base) { - // We initial assertion with true because if we have assertion diabled, we want to pass this - // check and if assertion is enabled we will have values re-assigned any way. - bool assertionPassed = true; - ELPP_ASSERT((assertionPassed = base::utils::File::pathExists(configurationFile.c_str(), true)) == true, - "Configuration file [" << configurationFile << "] does not exist!"); - if (!assertionPassed) { - return false; - } - bool success = Parser::parseFromFile(configurationFile, this, base); - m_isFromFile = success; - return success; -} - -bool Configurations::parseFromText(const std::string& configurationsString, Configurations* base) { - bool success = Parser::parseFromText(configurationsString, this, base); - if (success) { - m_isFromFile = false; - } - return success; -} - -void Configurations::setFromBase(Configurations* base) { - if (base == nullptr || base == this) { - return; - } - base::threading::ScopedLock scopedLock(base->lock()); - for (Configuration*& conf : base->list()) { - set(conf); - } -} - -bool Configurations::hasConfiguration(ConfigurationType configurationType) { - base::type::EnumType lIndex = LevelHelper::kMinValid; - bool result = false; - LevelHelper::forEachLevel(&lIndex, [&](void) -> bool { - if (hasConfiguration(LevelHelper::castFromInt(lIndex), configurationType)) { - result = true; - } - return result; - }); - return result; -} - -bool Configurations::hasConfiguration(Level level, ConfigurationType configurationType) { - base::threading::ScopedLock scopedLock(lock()); -#if ELPP_COMPILER_INTEL - // We cant specify template types here, Intel C++ throws compilation error - // "error: type name is not allowed" - return RegistryWithPred::get(level, configurationType) != nullptr; -#else - return RegistryWithPred::get(level, configurationType) != nullptr; -#endif // ELPP_COMPILER_INTEL -} - -void Configurations::set(Level level, ConfigurationType configurationType, const std::string& value) { - base::threading::ScopedLock scopedLock(lock()); - unsafeSet(level, configurationType, value); // This is not unsafe anymore as we have locked mutex - if (level == Level::Global) { - unsafeSetGlobally(configurationType, value, false); // Again this is not unsafe either - } -} - -void Configurations::set(Configuration* conf) { - if (conf == nullptr) { - return; - } - set(conf->level(), conf->configurationType(), conf->value()); -} - -void Configurations::setToDefault(void) { - setGlobally(ConfigurationType::Enabled, std::string("true"), true); - setGlobally(ConfigurationType::Filename, std::string(base::consts::kDefaultLogFile), true); -#if defined(ELPP_NO_LOG_TO_FILE) - setGlobally(ConfigurationType::ToFile, std::string("false"), true); -#else - setGlobally(ConfigurationType::ToFile, std::string("true"), true); -#endif // defined(ELPP_NO_LOG_TO_FILE) - setGlobally(ConfigurationType::ToStandardOutput, std::string("true"), true); - setGlobally(ConfigurationType::SubsecondPrecision, std::string("3"), true); - setGlobally(ConfigurationType::PerformanceTracking, std::string("true"), true); - setGlobally(ConfigurationType::MaxLogFileSize, std::string("0"), true); - setGlobally(ConfigurationType::LogFlushThreshold, std::string("0"), true); - - setGlobally(ConfigurationType::Format, std::string("%datetime %level [%logger] %msg"), true); - set(Level::Debug, ConfigurationType::Format, - std::string("%datetime %level [%logger] [%user@%host] [%func] [%loc] %msg")); - // INFO and WARNING are set to default by Level::Global - set(Level::Error, ConfigurationType::Format, std::string("%datetime %level [%logger] %msg")); - set(Level::Fatal, ConfigurationType::Format, std::string("%datetime %level [%logger] %msg")); - set(Level::Verbose, ConfigurationType::Format, std::string("%datetime %level-%vlevel [%logger] %msg")); - set(Level::Trace, ConfigurationType::Format, std::string("%datetime %level [%logger] [%func] [%loc] %msg")); -} - -void Configurations::setRemainingToDefault(void) { - base::threading::ScopedLock scopedLock(lock()); -#if defined(ELPP_NO_LOG_TO_FILE) - unsafeSetIfNotExist(Level::Global, ConfigurationType::Enabled, std::string("false")); -#else - unsafeSetIfNotExist(Level::Global, ConfigurationType::Enabled, std::string("true")); -#endif // defined(ELPP_NO_LOG_TO_FILE) - unsafeSetIfNotExist(Level::Global, ConfigurationType::Filename, std::string(base::consts::kDefaultLogFile)); - unsafeSetIfNotExist(Level::Global, ConfigurationType::ToStandardOutput, std::string("true")); - unsafeSetIfNotExist(Level::Global, ConfigurationType::SubsecondPrecision, std::string("3")); - unsafeSetIfNotExist(Level::Global, ConfigurationType::PerformanceTracking, std::string("true")); - unsafeSetIfNotExist(Level::Global, ConfigurationType::MaxLogFileSize, std::string("0")); - unsafeSetIfNotExist(Level::Global, ConfigurationType::Format, std::string("%datetime %level [%logger] %msg")); - unsafeSetIfNotExist(Level::Debug, ConfigurationType::Format, - std::string("%datetime %level [%logger] [%user@%host] [%func] [%loc] %msg")); - // INFO and WARNING are set to default by Level::Global - unsafeSetIfNotExist(Level::Error, ConfigurationType::Format, std::string("%datetime %level [%logger] %msg")); - unsafeSetIfNotExist(Level::Fatal, ConfigurationType::Format, std::string("%datetime %level [%logger] %msg")); - unsafeSetIfNotExist(Level::Verbose, ConfigurationType::Format, std::string("%datetime %level-%vlevel [%logger] %msg")); - unsafeSetIfNotExist(Level::Trace, ConfigurationType::Format, - std::string("%datetime %level [%logger] [%func] [%loc] %msg")); -} - -bool Configurations::Parser::parseFromFile(const std::string& configurationFile, Configurations* sender, - Configurations* base) { - sender->setFromBase(base); - std::ifstream fileStream_(configurationFile.c_str(), std::ifstream::in); - ELPP_ASSERT(fileStream_.is_open(), "Unable to open configuration file [" << configurationFile << "] for parsing."); - bool parsedSuccessfully = false; - std::string line = std::string(); - Level currLevel = Level::Unknown; - std::string currConfigStr = std::string(); - std::string currLevelStr = std::string(); - while (fileStream_.good()) { - std::getline(fileStream_, line); - parsedSuccessfully = parseLine(&line, &currConfigStr, &currLevelStr, &currLevel, sender); - ELPP_ASSERT(parsedSuccessfully, "Unable to parse configuration line: " << line); - } - return parsedSuccessfully; -} - -bool Configurations::Parser::parseFromText(const std::string& configurationsString, Configurations* sender, - Configurations* base) { - sender->setFromBase(base); - bool parsedSuccessfully = false; - std::stringstream ss(configurationsString); - std::string line = std::string(); - Level currLevel = Level::Unknown; - std::string currConfigStr = std::string(); - std::string currLevelStr = std::string(); - while (std::getline(ss, line)) { - parsedSuccessfully = parseLine(&line, &currConfigStr, &currLevelStr, &currLevel, sender); - ELPP_ASSERT(parsedSuccessfully, "Unable to parse configuration line: " << line); - } - return parsedSuccessfully; -} - -void Configurations::Parser::ignoreComments(std::string* line) { - std::size_t foundAt = 0; - std::size_t quotesStart = line->find("\""); - std::size_t quotesEnd = std::string::npos; - if (quotesStart != std::string::npos) { - quotesEnd = line->find("\"", quotesStart + 1); - while (quotesEnd != std::string::npos && line->at(quotesEnd - 1) == '\\') { - // Do not erase slash yet - we will erase it in parseLine(..) while loop - quotesEnd = line->find("\"", quotesEnd + 2); - } - } - if ((foundAt = line->find(base::consts::kConfigurationComment)) != std::string::npos) { - if (foundAt < quotesEnd) { - foundAt = line->find(base::consts::kConfigurationComment, quotesEnd + 1); - } - *line = line->substr(0, foundAt); - } -} - -bool Configurations::Parser::isLevel(const std::string& line) { - return base::utils::Str::startsWith(line, std::string(base::consts::kConfigurationLevel)); -} - -bool Configurations::Parser::isComment(const std::string& line) { - return base::utils::Str::startsWith(line, std::string(base::consts::kConfigurationComment)); -} - -bool Configurations::Parser::isConfig(const std::string& line) { - std::size_t assignment = line.find('='); - return line != "" && - ((line[0] >= 'A' && line[0] <= 'Z') || (line[0] >= 'a' && line[0] <= 'z')) && - (assignment != std::string::npos) && - (line.size() > assignment); -} - -bool Configurations::Parser::parseLine(std::string* line, std::string* currConfigStr, std::string* currLevelStr, - Level* currLevel, - Configurations* conf) { - ConfigurationType currConfig = ConfigurationType::Unknown; - std::string currValue = std::string(); - *line = base::utils::Str::trim(*line); - if (isComment(*line)) return true; - ignoreComments(line); - *line = base::utils::Str::trim(*line); - if (line->empty()) { - // Comment ignored - return true; - } - if (isLevel(*line)) { - if (line->size() <= 2) { - return true; - } - *currLevelStr = line->substr(1, line->size() - 2); - *currLevelStr = base::utils::Str::toUpper(*currLevelStr); - *currLevelStr = base::utils::Str::trim(*currLevelStr); - *currLevel = LevelHelper::convertFromString(currLevelStr->c_str()); - return true; - } - if (isConfig(*line)) { - std::size_t assignment = line->find('='); - *currConfigStr = line->substr(0, assignment); - *currConfigStr = base::utils::Str::toUpper(*currConfigStr); - *currConfigStr = base::utils::Str::trim(*currConfigStr); - currConfig = ConfigurationTypeHelper::convertFromString(currConfigStr->c_str()); - currValue = line->substr(assignment + 1); - currValue = base::utils::Str::trim(currValue); - std::size_t quotesStart = currValue.find("\"", 0); - std::size_t quotesEnd = std::string::npos; - if (quotesStart != std::string::npos) { - quotesEnd = currValue.find("\"", quotesStart + 1); - while (quotesEnd != std::string::npos && currValue.at(quotesEnd - 1) == '\\') { - currValue = currValue.erase(quotesEnd - 1, 1); - quotesEnd = currValue.find("\"", quotesEnd + 2); - } - } - if (quotesStart != std::string::npos && quotesEnd != std::string::npos) { - // Quote provided - check and strip if valid - ELPP_ASSERT((quotesStart < quotesEnd), "Configuration error - No ending quote found in [" - << currConfigStr << "]"); - ELPP_ASSERT((quotesStart + 1 != quotesEnd), "Empty configuration value for [" << currConfigStr << "]"); - if ((quotesStart != quotesEnd) && (quotesStart + 1 != quotesEnd)) { - // Explicit check in case if assertion is disabled - currValue = currValue.substr(quotesStart + 1, quotesEnd - 1); - } - } - } - ELPP_ASSERT(*currLevel != Level::Unknown, "Unrecognized severity level [" << *currLevelStr << "]"); - ELPP_ASSERT(currConfig != ConfigurationType::Unknown, "Unrecognized configuration [" << *currConfigStr << "]"); - if (*currLevel == Level::Unknown || currConfig == ConfigurationType::Unknown) { - return false; // unrecognizable level or config - } - conf->set(*currLevel, currConfig, currValue); - return true; -} - -void Configurations::unsafeSetIfNotExist(Level level, ConfigurationType configurationType, const std::string& value) { - Configuration* conf = RegistryWithPred::get(level, configurationType); - if (conf == nullptr) { - unsafeSet(level, configurationType, value); - } -} - -void Configurations::unsafeSet(Level level, ConfigurationType configurationType, const std::string& value) { - Configuration* conf = RegistryWithPred::get(level, configurationType); - if (conf == nullptr) { - registerNew(new Configuration(level, configurationType, value)); - } else { - conf->setValue(value); - } - if (level == Level::Global) { - unsafeSetGlobally(configurationType, value, false); - } -} - -void Configurations::setGlobally(ConfigurationType configurationType, const std::string& value, - bool includeGlobalLevel) { - if (includeGlobalLevel) { - set(Level::Global, configurationType, value); - } - base::type::EnumType lIndex = LevelHelper::kMinValid; - LevelHelper::forEachLevel(&lIndex, [&](void) -> bool { - set(LevelHelper::castFromInt(lIndex), configurationType, value); - return false; // Do not break lambda function yet as we need to set all levels regardless - }); -} - -void Configurations::unsafeSetGlobally(ConfigurationType configurationType, const std::string& value, - bool includeGlobalLevel) { - if (includeGlobalLevel) { - unsafeSet(Level::Global, configurationType, value); - } - base::type::EnumType lIndex = LevelHelper::kMinValid; - LevelHelper::forEachLevel(&lIndex, [&](void) -> bool { - unsafeSet(LevelHelper::castFromInt(lIndex), configurationType, value); - return false; // Do not break lambda function yet as we need to set all levels regardless - }); -} - -// LogBuilder - -void LogBuilder::convertToColoredOutput(base::type::string_t* logLine, Level level) { - if (!m_termSupportsColor) return; - const base::type::char_t* resetColor = ELPP_LITERAL("\x1b[0m"); - if (level == Level::Error || level == Level::Fatal) - *logLine = ELPP_LITERAL("\x1b[31m") + *logLine + resetColor; - else if (level == Level::Warning) - *logLine = ELPP_LITERAL("\x1b[33m") + *logLine + resetColor; - else if (level == Level::Debug) - *logLine = ELPP_LITERAL("\x1b[32m") + *logLine + resetColor; - else if (level == Level::Info) - *logLine = ELPP_LITERAL("\x1b[36m") + *logLine + resetColor; - else if (level == Level::Trace) - *logLine = ELPP_LITERAL("\x1b[35m") + *logLine + resetColor; -} - -// Logger - -Logger::Logger(const std::string& id, base::LogStreamsReferenceMap* logStreamsReference) : - m_id(id), - m_typedConfigurations(nullptr), - m_parentApplicationName(std::string()), - m_isConfigured(false), - m_logStreamsReference(logStreamsReference) { - initUnflushedCount(); -} - -Logger::Logger(const std::string& id, const Configurations& configurations, - base::LogStreamsReferenceMap* logStreamsReference) : - m_id(id), - m_typedConfigurations(nullptr), - m_parentApplicationName(std::string()), - m_isConfigured(false), - m_logStreamsReference(logStreamsReference) { - initUnflushedCount(); - configure(configurations); -} - -Logger::Logger(const Logger& logger) { - base::utils::safeDelete(m_typedConfigurations); - m_id = logger.m_id; - m_typedConfigurations = logger.m_typedConfigurations; - m_parentApplicationName = logger.m_parentApplicationName; - m_isConfigured = logger.m_isConfigured; - m_configurations = logger.m_configurations; - m_unflushedCount = logger.m_unflushedCount; - m_logStreamsReference = logger.m_logStreamsReference; -} - -Logger& Logger::operator=(const Logger& logger) { - if (&logger != this) { - base::utils::safeDelete(m_typedConfigurations); - m_id = logger.m_id; - m_typedConfigurations = logger.m_typedConfigurations; - m_parentApplicationName = logger.m_parentApplicationName; - m_isConfigured = logger.m_isConfigured; - m_configurations = logger.m_configurations; - m_unflushedCount = logger.m_unflushedCount; - m_logStreamsReference = logger.m_logStreamsReference; - } - return *this; -} - -void Logger::configure(const Configurations& configurations) { - m_isConfigured = false; // we set it to false in case if we fail - initUnflushedCount(); - if (m_typedConfigurations != nullptr) { - Configurations* c = const_cast(m_typedConfigurations->configurations()); - if (c->hasConfiguration(Level::Global, ConfigurationType::Filename)) { - flush(); - } - } - base::threading::ScopedLock scopedLock(lock()); - if (m_configurations != configurations) { - m_configurations.setFromBase(const_cast(&configurations)); - } - base::utils::safeDelete(m_typedConfigurations); - m_typedConfigurations = new base::TypedConfigurations(&m_configurations, m_logStreamsReference); - resolveLoggerFormatSpec(); - m_isConfigured = true; -} - -void Logger::reconfigure(void) { - ELPP_INTERNAL_INFO(1, "Reconfiguring logger [" << m_id << "]"); - configure(m_configurations); -} - -bool Logger::isValidId(const std::string& id) { - for (std::string::const_iterator it = id.begin(); it != id.end(); ++it) { - if (!base::utils::Str::contains(base::consts::kValidLoggerIdSymbols, *it)) { - return false; - } - } - return true; -} - -void Logger::flush(void) { - ELPP_INTERNAL_INFO(3, "Flushing logger [" << m_id << "] all levels"); - base::threading::ScopedLock scopedLock(lock()); - base::type::EnumType lIndex = LevelHelper::kMinValid; - LevelHelper::forEachLevel(&lIndex, [&](void) -> bool { - flush(LevelHelper::castFromInt(lIndex), nullptr); - return false; - }); -} - -void Logger::flush(Level level, base::type::fstream_t* fs) { - if (fs == nullptr && m_typedConfigurations->toFile(level)) { - fs = m_typedConfigurations->fileStream(level); - } - if (fs != nullptr) { - fs->flush(); - std::unordered_map::iterator iter = m_unflushedCount.find(level); - if (iter != m_unflushedCount.end()) { - iter->second = 0; - } - Helpers::validateFileRolling(this, level); - } -} - -void Logger::initUnflushedCount(void) { - m_unflushedCount.clear(); - base::type::EnumType lIndex = LevelHelper::kMinValid; - LevelHelper::forEachLevel(&lIndex, [&](void) -> bool { - m_unflushedCount.insert(std::make_pair(LevelHelper::castFromInt(lIndex), 0)); - return false; - }); -} - -void Logger::resolveLoggerFormatSpec(void) const { - base::type::EnumType lIndex = LevelHelper::kMinValid; - LevelHelper::forEachLevel(&lIndex, [&](void) -> bool { - base::LogFormat* logFormat = - const_cast(&m_typedConfigurations->logFormat(LevelHelper::castFromInt(lIndex))); - base::utils::Str::replaceFirstWithEscape(logFormat->m_format, base::consts::kLoggerIdFormatSpecifier, m_id); - return false; - }); -} - -// el::base -namespace base { - -// el::base::utils -namespace utils { - -// File - -base::type::fstream_t* File::newFileStream(const std::string& filename) { - base::type::fstream_t *fs = new base::type::fstream_t(filename.c_str(), - base::type::fstream_t::out -#if !defined(ELPP_FRESH_LOG_FILE) - | base::type::fstream_t::app -#endif - ); -#if defined(ELPP_UNICODE) - std::locale elppUnicodeLocale(""); -# if ELPP_OS_WINDOWS - std::locale elppUnicodeLocaleWindows(elppUnicodeLocale, new std::codecvt_utf8_utf16); - elppUnicodeLocale = elppUnicodeLocaleWindows; -# endif // ELPP_OS_WINDOWS - fs->imbue(elppUnicodeLocale); -#endif // defined(ELPP_UNICODE) - if (fs->is_open()) { - fs->flush(); - } else { - base::utils::safeDelete(fs); - ELPP_INTERNAL_ERROR("Bad file [" << filename << "]", true); - } - return fs; -} - -std::size_t File::getSizeOfFile(base::type::fstream_t* fs) { - if (fs == nullptr) { - return 0; - } - // Since the file stream is appended to or truncated, the current - // offset is the file size. - std::size_t size = static_cast(fs->tellg()); - return size; -} - -bool File::pathExists(const char* path, bool considerFile) { - if (path == nullptr) { - return false; - } -#if ELPP_OS_UNIX - ELPP_UNUSED(considerFile); - struct stat st; - return (stat(path, &st) == 0); -#elif ELPP_OS_WINDOWS - DWORD fileType = GetFileAttributesA(path); - if (fileType == INVALID_FILE_ATTRIBUTES) { - return false; - } - return considerFile ? true : ((fileType & FILE_ATTRIBUTE_DIRECTORY) == 0 ? false : true); -#endif // ELPP_OS_UNIX -} - -bool File::createPath(const std::string& path) { - if (path.empty()) { - return false; - } - if (base::utils::File::pathExists(path.c_str())) { - return true; - } - int status = -1; - - char* currPath = const_cast(path.c_str()); - std::string builtPath = std::string(); -#if ELPP_OS_UNIX - if (path[0] == '/') { - builtPath = "/"; - } - currPath = STRTOK(currPath, base::consts::kFilePathSeperator, 0); -#elif ELPP_OS_WINDOWS - // Use secure functions API - char* nextTok_ = nullptr; - currPath = STRTOK(currPath, base::consts::kFilePathSeperator, &nextTok_); - ELPP_UNUSED(nextTok_); -#endif // ELPP_OS_UNIX - while (currPath != nullptr) { - builtPath.append(currPath); - builtPath.append(base::consts::kFilePathSeperator); -#if ELPP_OS_UNIX - status = mkdir(builtPath.c_str(), ELPP_LOG_PERMS); - currPath = STRTOK(nullptr, base::consts::kFilePathSeperator, 0); -#elif ELPP_OS_WINDOWS - status = _mkdir(builtPath.c_str()); - currPath = STRTOK(nullptr, base::consts::kFilePathSeperator, &nextTok_); -#endif // ELPP_OS_UNIX - } - if (status == -1) { - ELPP_INTERNAL_ERROR("Error while creating path [" << path << "]", true); - return false; - } - return true; -} - -std::string File::extractPathFromFilename(const std::string& fullPath, const char* separator) { - if ((fullPath == "") || (fullPath.find(separator) == std::string::npos)) { - return fullPath; - } - std::size_t lastSlashAt = fullPath.find_last_of(separator); - if (lastSlashAt == 0) { - return std::string(separator); - } - return fullPath.substr(0, lastSlashAt + 1); -} - -void File::buildStrippedFilename(const char* filename, char buff[], std::size_t limit) { - std::size_t sizeOfFilename = strlen(filename); - if (sizeOfFilename >= limit) { - filename += (sizeOfFilename - limit); - if (filename[0] != '.' && filename[1] != '.') { // prepend if not already - filename += 3; // 3 = '..' - STRCAT(buff, "..", limit); - } - } - STRCAT(buff, filename, limit); -} - -void File::buildBaseFilename(const std::string& fullPath, char buff[], std::size_t limit, const char* separator) { - const char *filename = fullPath.c_str(); - std::size_t lastSlashAt = fullPath.find_last_of(separator); - filename += lastSlashAt ? lastSlashAt+1 : 0; - std::size_t sizeOfFilename = strlen(filename); - if (sizeOfFilename >= limit) { - filename += (sizeOfFilename - limit); - if (filename[0] != '.' && filename[1] != '.') { // prepend if not already - filename += 3; // 3 = '..' - STRCAT(buff, "..", limit); - } - } - STRCAT(buff, filename, limit); -} - -// Str - -bool Str::wildCardMatch(const char* str, const char* pattern) { - while (*pattern) { - switch (*pattern) { - case '?': - if (!*str) - return false; - ++str; - ++pattern; - break; - case '*': - if (wildCardMatch(str, pattern + 1)) - return true; - if (*str && wildCardMatch(str + 1, pattern)) - return true; - return false; - default: - if (*str++ != *pattern++) - return false; - break; - } - } - return !*str && !*pattern; -} - -std::string& Str::ltrim(std::string& str) { - str.erase(str.begin(), std::find_if(str.begin(), str.end(), [](char c) { - return !std::isspace(c); - } )); - return str; -} - -std::string& Str::rtrim(std::string& str) { - str.erase(std::find_if(str.rbegin(), str.rend(), [](char c) { - return !std::isspace(c); - }).base(), str.end()); - return str; -} - -std::string& Str::trim(std::string& str) { - return ltrim(rtrim(str)); -} - -bool Str::startsWith(const std::string& str, const std::string& start) { - return (str.length() >= start.length()) && (str.compare(0, start.length(), start) == 0); -} - -bool Str::endsWith(const std::string& str, const std::string& end) { - return (str.length() >= end.length()) && (str.compare(str.length() - end.length(), end.length(), end) == 0); -} - -std::string& Str::replaceAll(std::string& str, char replaceWhat, char replaceWith) { - std::replace(str.begin(), str.end(), replaceWhat, replaceWith); - return str; -} - -std::string& Str::replaceAll(std::string& str, const std::string& replaceWhat, - const std::string& replaceWith) { - if (replaceWhat == replaceWith) - return str; - std::size_t foundAt = std::string::npos; - while ((foundAt = str.find(replaceWhat, foundAt + 1)) != std::string::npos) { - str.replace(foundAt, replaceWhat.length(), replaceWith); - } - return str; -} - -void Str::replaceFirstWithEscape(base::type::string_t& str, const base::type::string_t& replaceWhat, - const base::type::string_t& replaceWith) { - std::size_t foundAt = base::type::string_t::npos; - while ((foundAt = str.find(replaceWhat, foundAt + 1)) != base::type::string_t::npos) { - if (foundAt > 0 && str[foundAt - 1] == base::consts::kFormatSpecifierChar) { - str.erase(foundAt - 1, 1); - ++foundAt; - } else { - str.replace(foundAt, replaceWhat.length(), replaceWith); - return; - } - } -} -#if defined(ELPP_UNICODE) -void Str::replaceFirstWithEscape(base::type::string_t& str, const base::type::string_t& replaceWhat, - const std::string& replaceWith) { - replaceFirstWithEscape(str, replaceWhat, base::type::string_t(replaceWith.begin(), replaceWith.end())); -} -#endif // defined(ELPP_UNICODE) - -std::string& Str::toUpper(std::string& str) { - std::transform(str.begin(), str.end(), str.begin(), - [](char c) { - return static_cast(::toupper(c)); - }); - return str; -} - -bool Str::cStringEq(const char* s1, const char* s2) { - if (s1 == nullptr && s2 == nullptr) return true; - if (s1 == nullptr || s2 == nullptr) return false; - return strcmp(s1, s2) == 0; -} - -bool Str::cStringCaseEq(const char* s1, const char* s2) { - if (s1 == nullptr && s2 == nullptr) return true; - if (s1 == nullptr || s2 == nullptr) return false; - - // With thanks to cygwin for this code - int d = 0; - - while (true) { - const int c1 = toupper(*s1++); - const int c2 = toupper(*s2++); - - if (((d = c1 - c2) != 0) || (c2 == '\0')) { - break; - } - } - - return d == 0; -} - -bool Str::contains(const char* str, char c) { - for (; *str; ++str) { - if (*str == c) - return true; - } - return false; -} - -char* Str::convertAndAddToBuff(std::size_t n, int len, char* buf, const char* bufLim, bool zeroPadded) { - char localBuff[10] = ""; - char* p = localBuff + sizeof(localBuff) - 2; - if (n > 0) { - for (; n > 0 && p > localBuff && len > 0; n /= 10, --len) - *--p = static_cast(n % 10 + '0'); - } else { - *--p = '0'; - --len; - } - if (zeroPadded) - while (p > localBuff && len-- > 0) *--p = static_cast('0'); - return addToBuff(p, buf, bufLim); -} - -char* Str::addToBuff(const char* str, char* buf, const char* bufLim) { - while ((buf < bufLim) && ((*buf = *str++) != '\0')) - ++buf; - return buf; -} - -char* Str::clearBuff(char buff[], std::size_t lim) { - STRCPY(buff, "", lim); - ELPP_UNUSED(lim); // For *nix we dont have anything using lim in above STRCPY macro - return buff; -} - -/// @brief Converst wchar* to char* -/// NOTE: Need to free return value after use! -char* Str::wcharPtrToCharPtr(const wchar_t* line) { - std::size_t len_ = wcslen(line) + 1; - char* buff_ = static_cast(malloc(len_ + 1)); -# if ELPP_OS_UNIX || (ELPP_OS_WINDOWS && !ELPP_CRT_DBG_WARNINGS) - std::wcstombs(buff_, line, len_); -# elif ELPP_OS_WINDOWS - std::size_t convCount_ = 0; - mbstate_t mbState_; - ::memset(static_cast(&mbState_), 0, sizeof(mbState_)); - wcsrtombs_s(&convCount_, buff_, len_, &line, len_, &mbState_); -# endif // ELPP_OS_UNIX || (ELPP_OS_WINDOWS && !ELPP_CRT_DBG_WARNINGS) - return buff_; -} - -// OS - -#if ELPP_OS_WINDOWS -/// @brief Gets environment variables for Windows based OS. -/// We are not using getenv(const char*) because of CRT deprecation -/// @param varname Variable name to get environment variable value for -/// @return If variable exist the value of it otherwise nullptr -const char* OS::getWindowsEnvironmentVariable(const char* varname) { - const DWORD bufferLen = 50; - static char buffer[bufferLen]; - if (GetEnvironmentVariableA(varname, buffer, bufferLen)) { - return buffer; - } - return nullptr; -} -#endif // ELPP_OS_WINDOWS -#if ELPP_OS_ANDROID -std::string OS::getProperty(const char* prop) { - char propVal[PROP_VALUE_MAX + 1]; - int ret = __system_property_get(prop, propVal); - return ret == 0 ? std::string() : std::string(propVal); -} - -std::string OS::getDeviceName(void) { - std::stringstream ss; - std::string manufacturer = getProperty("ro.product.manufacturer"); - std::string model = getProperty("ro.product.model"); - if (manufacturer.empty() || model.empty()) { - return std::string(); - } - ss << manufacturer << "-" << model; - return ss.str(); -} -#endif // ELPP_OS_ANDROID - -const std::string OS::getBashOutput(const char* command) { -#if (ELPP_OS_UNIX && !ELPP_OS_ANDROID && !ELPP_CYGWIN) - if (command == nullptr) { - return std::string(); - } - FILE* proc = nullptr; - if ((proc = popen(command, "r")) == nullptr) { - ELPP_INTERNAL_ERROR("\nUnable to run command [" << command << "]", true); - return std::string(); - } - char hBuff[4096]; - if (fgets(hBuff, sizeof(hBuff), proc) != nullptr) { - pclose(proc); - const std::size_t buffLen = strlen(hBuff); - if (buffLen > 0 && hBuff[buffLen - 1] == '\n') { - hBuff[buffLen - 1] = '\0'; - } - return std::string(hBuff); - } else { - pclose(proc); - } - return std::string(); -#else - ELPP_UNUSED(command); - return std::string(); -#endif // (ELPP_OS_UNIX && !ELPP_OS_ANDROID && !ELPP_CYGWIN) -} - -std::string OS::getEnvironmentVariable(const char* variableName, const char* defaultVal, - const char* alternativeBashCommand) { -#if ELPP_OS_UNIX - const char* val = getenv(variableName); -#elif ELPP_OS_WINDOWS - const char* val = getWindowsEnvironmentVariable(variableName); -#endif // ELPP_OS_UNIX - if ((val == nullptr) || ((strcmp(val, "") == 0))) { -#if ELPP_OS_UNIX && defined(ELPP_FORCE_ENV_VAR_FROM_BASH) - // Try harder on unix-based systems - std::string valBash = base::utils::OS::getBashOutput(alternativeBashCommand); - if (valBash.empty()) { - return std::string(defaultVal); - } else { - return valBash; - } -#elif ELPP_OS_WINDOWS || ELPP_OS_UNIX - ELPP_UNUSED(alternativeBashCommand); - return std::string(defaultVal); -#endif // ELPP_OS_UNIX && defined(ELPP_FORCE_ENV_VAR_FROM_BASH) - } - return std::string(val); -} - -std::string OS::currentUser(void) { -#if ELPP_OS_UNIX && !ELPP_OS_ANDROID - return getEnvironmentVariable("USER", base::consts::kUnknownUser, "whoami"); -#elif ELPP_OS_WINDOWS - return getEnvironmentVariable("USERNAME", base::consts::kUnknownUser); -#elif ELPP_OS_ANDROID - ELPP_UNUSED(base::consts::kUnknownUser); - return std::string("android"); -#else - return std::string(); -#endif // ELPP_OS_UNIX && !ELPP_OS_ANDROID -} - -std::string OS::currentHost(void) { -#if ELPP_OS_UNIX && !ELPP_OS_ANDROID - return getEnvironmentVariable("HOSTNAME", base::consts::kUnknownHost, "hostname"); -#elif ELPP_OS_WINDOWS - return getEnvironmentVariable("COMPUTERNAME", base::consts::kUnknownHost); -#elif ELPP_OS_ANDROID - ELPP_UNUSED(base::consts::kUnknownHost); - return getDeviceName(); -#else - return std::string(); -#endif // ELPP_OS_UNIX && !ELPP_OS_ANDROID -} - -bool OS::termSupportsColor(void) { - std::string term = getEnvironmentVariable("TERM", ""); - return term == "xterm" || term == "xterm-color" || term == "xterm-256color" - || term == "screen" || term == "linux" || term == "cygwin" - || term == "screen-256color"; -} - -// DateTime - -void DateTime::gettimeofday(struct timeval* tv) { -#if ELPP_OS_WINDOWS - if (tv != nullptr) { -# if ELPP_COMPILER_MSVC || defined(_MSC_EXTENSIONS) - const unsigned __int64 delta_ = 11644473600000000Ui64; -# else - const unsigned __int64 delta_ = 11644473600000000ULL; -# endif // ELPP_COMPILER_MSVC || defined(_MSC_EXTENSIONS) - const double secOffSet = 0.000001; - const unsigned long usecOffSet = 1000000; - FILETIME fileTime; - GetSystemTimeAsFileTime(&fileTime); - unsigned __int64 present = 0; - present |= fileTime.dwHighDateTime; - present = present << 32; - present |= fileTime.dwLowDateTime; - present /= 10; // mic-sec - // Subtract the difference - present -= delta_; - tv->tv_sec = static_cast(present * secOffSet); - tv->tv_usec = static_cast(present % usecOffSet); - } -#else - ::gettimeofday(tv, nullptr); -#endif // ELPP_OS_WINDOWS -} - -std::string DateTime::getDateTime(const char* format, const base::SubsecondPrecision* ssPrec) { - struct timeval currTime; - gettimeofday(&currTime); - return timevalToString(currTime, format, ssPrec); -} - -std::string DateTime::timevalToString(struct timeval tval, const char* format, - const el::base::SubsecondPrecision* ssPrec) { - struct ::tm timeInfo; - buildTimeInfo(&tval, &timeInfo); - const int kBuffSize = 30; - char buff_[kBuffSize] = ""; - parseFormat(buff_, kBuffSize, format, &timeInfo, static_cast(tval.tv_usec / ssPrec->m_offset), - ssPrec); - return std::string(buff_); -} - -base::type::string_t DateTime::formatTime(unsigned long long time, base::TimestampUnit timestampUnit) { - base::type::EnumType start = static_cast(timestampUnit); - const base::type::char_t* unit = base::consts::kTimeFormats[start].unit; - for (base::type::EnumType i = start; i < base::consts::kTimeFormatsCount - 1; ++i) { - if (time <= base::consts::kTimeFormats[i].value) { - break; - } - if (base::consts::kTimeFormats[i].value == 1000.0f && time / 1000.0f < 1.9f) { - break; - } - time /= static_cast(base::consts::kTimeFormats[i].value); - unit = base::consts::kTimeFormats[i + 1].unit; - } - base::type::stringstream_t ss; - ss << time << " " << unit; - return ss.str(); -} - -unsigned long long DateTime::getTimeDifference(const struct timeval& endTime, const struct timeval& startTime, - base::TimestampUnit timestampUnit) { - if (timestampUnit == base::TimestampUnit::Microsecond) { - return static_cast(static_cast(1000000 * endTime.tv_sec + endTime.tv_usec) - - static_cast(1000000 * startTime.tv_sec + startTime.tv_usec)); - } - // milliseconds - auto conv = [](const struct timeval& tim) { - return static_cast((tim.tv_sec * 1000) + (tim.tv_usec / 1000)); - }; - return static_cast(conv(endTime) - conv(startTime)); -} - -struct ::tm* DateTime::buildTimeInfo(struct timeval* currTime, struct ::tm* timeInfo) { -#if ELPP_OS_UNIX - time_t rawTime = currTime->tv_sec; - ::elpptime_r(&rawTime, timeInfo); - return timeInfo; -#else -# if ELPP_COMPILER_MSVC - ELPP_UNUSED(currTime); - time_t t; -# if defined(_USE_32BIT_TIME_T) - _time32(&t); -# else - _time64(&t); -# endif - elpptime_s(timeInfo, &t); - return timeInfo; -# else - // For any other compilers that don't have CRT warnings issue e.g, MinGW or TDM GCC- we use different method - time_t rawTime = currTime->tv_sec; - struct tm* tmInf = elpptime(&rawTime); - *timeInfo = *tmInf; - return timeInfo; -# endif // ELPP_COMPILER_MSVC -#endif // ELPP_OS_UNIX -} - -char* DateTime::parseFormat(char* buf, std::size_t bufSz, const char* format, const struct tm* tInfo, - std::size_t msec, const base::SubsecondPrecision* ssPrec) { - const char* bufLim = buf + bufSz; - for (; *format; ++format) { - if (*format == base::consts::kFormatSpecifierChar) { - switch (*++format) { - case base::consts::kFormatSpecifierChar: // Escape - break; - case '\0': // End - --format; - break; - case 'd': // Day - buf = base::utils::Str::convertAndAddToBuff(tInfo->tm_mday, 2, buf, bufLim); - continue; - case 'a': // Day of week (short) - buf = base::utils::Str::addToBuff(base::consts::kDaysAbbrev[tInfo->tm_wday], buf, bufLim); - continue; - case 'A': // Day of week (long) - buf = base::utils::Str::addToBuff(base::consts::kDays[tInfo->tm_wday], buf, bufLim); - continue; - case 'M': // month - buf = base::utils::Str::convertAndAddToBuff(tInfo->tm_mon + 1, 2, buf, bufLim); - continue; - case 'b': // month (short) - buf = base::utils::Str::addToBuff(base::consts::kMonthsAbbrev[tInfo->tm_mon], buf, bufLim); - continue; - case 'B': // month (long) - buf = base::utils::Str::addToBuff(base::consts::kMonths[tInfo->tm_mon], buf, bufLim); - continue; - case 'y': // year (two digits) - buf = base::utils::Str::convertAndAddToBuff(tInfo->tm_year + base::consts::kYearBase, 2, buf, bufLim); - continue; - case 'Y': // year (four digits) - buf = base::utils::Str::convertAndAddToBuff(tInfo->tm_year + base::consts::kYearBase, 4, buf, bufLim); - continue; - case 'h': // hour (12-hour) - buf = base::utils::Str::convertAndAddToBuff(tInfo->tm_hour % 12, 2, buf, bufLim); - continue; - case 'H': // hour (24-hour) - buf = base::utils::Str::convertAndAddToBuff(tInfo->tm_hour, 2, buf, bufLim); - continue; - case 'm': // minute - buf = base::utils::Str::convertAndAddToBuff(tInfo->tm_min, 2, buf, bufLim); - continue; - case 's': // second - buf = base::utils::Str::convertAndAddToBuff(tInfo->tm_sec, 2, buf, bufLim); - continue; - case 'z': // subsecond part - case 'g': - buf = base::utils::Str::convertAndAddToBuff(msec, ssPrec->m_width, buf, bufLim); - continue; - case 'F': // AM/PM - buf = base::utils::Str::addToBuff((tInfo->tm_hour >= 12) ? base::consts::kPm : base::consts::kAm, buf, bufLim); - continue; - default: - continue; - } - } - if (buf == bufLim) break; - *buf++ = *format; - } - return buf; -} - -// CommandLineArgs - -void CommandLineArgs::setArgs(int argc, char** argv) { - m_params.clear(); - m_paramsWithValue.clear(); - if (argc == 0 || argv == nullptr) { - return; - } - m_argc = argc; - m_argv = argv; - for (int i = 1; i < m_argc; ++i) { - const char* v = (strstr(m_argv[i], "=")); - if (v != nullptr && strlen(v) > 0) { - std::string key = std::string(m_argv[i]); - key = key.substr(0, key.find_first_of('=')); - if (hasParamWithValue(key.c_str())) { - ELPP_INTERNAL_INFO(1, "Skipping [" << key << "] arg since it already has value [" - << getParamValue(key.c_str()) << "]"); - } else { - m_paramsWithValue.insert(std::make_pair(key, std::string(v + 1))); - } - } - if (v == nullptr) { - if (hasParam(m_argv[i])) { - ELPP_INTERNAL_INFO(1, "Skipping [" << m_argv[i] << "] arg since it already exists"); - } else { - m_params.push_back(std::string(m_argv[i])); - } - } - } -} - -bool CommandLineArgs::hasParamWithValue(const char* paramKey) const { - return m_paramsWithValue.find(std::string(paramKey)) != m_paramsWithValue.end(); -} - -const char* CommandLineArgs::getParamValue(const char* paramKey) const { - std::unordered_map::const_iterator iter = m_paramsWithValue.find(std::string(paramKey)); - return iter != m_paramsWithValue.end() ? iter->second.c_str() : ""; -} - -bool CommandLineArgs::hasParam(const char* paramKey) const { - return std::find(m_params.begin(), m_params.end(), std::string(paramKey)) != m_params.end(); -} - -bool CommandLineArgs::empty(void) const { - return m_params.empty() && m_paramsWithValue.empty(); -} - -std::size_t CommandLineArgs::size(void) const { - return m_params.size() + m_paramsWithValue.size(); -} - -base::type::ostream_t& operator<<(base::type::ostream_t& os, const CommandLineArgs& c) { - for (int i = 1; i < c.m_argc; ++i) { - os << ELPP_LITERAL("[") << c.m_argv[i] << ELPP_LITERAL("]"); - if (i < c.m_argc - 1) { - os << ELPP_LITERAL(" "); - } - } - return os; -} - -} // namespace utils - -// el::base::threading -namespace threading { - -#if ELPP_THREADING_ENABLED -# if ELPP_USE_STD_THREADING -# if ELPP_ASYNC_LOGGING -static void msleep(int ms) { - // Only when async logging enabled - this is because async is strict on compiler -# if defined(ELPP_NO_SLEEP_FOR) - usleep(ms * 1000); -# else - std::this_thread::sleep_for(std::chrono::milliseconds(ms)); -# endif // defined(ELPP_NO_SLEEP_FOR) -} -# endif // ELPP_ASYNC_LOGGING -# endif // !ELPP_USE_STD_THREADING -#endif // ELPP_THREADING_ENABLED - -} // namespace threading - -// el::base - -// SubsecondPrecision - -void SubsecondPrecision::init(int width) { - if (width < 1 || width > 6) { - width = base::consts::kDefaultSubsecondPrecision; - } - m_width = width; - switch (m_width) { - case 3: - m_offset = 1000; - break; - case 4: - m_offset = 100; - break; - case 5: - m_offset = 10; - break; - case 6: - m_offset = 1; - break; - default: - m_offset = 1000; - break; - } -} - -// LogFormat - -LogFormat::LogFormat(void) : - m_level(Level::Unknown), - m_userFormat(base::type::string_t()), - m_format(base::type::string_t()), - m_dateTimeFormat(std::string()), - m_flags(0x0), - m_currentUser(base::utils::OS::currentUser()), - m_currentHost(base::utils::OS::currentHost()) { -} - -LogFormat::LogFormat(Level level, const base::type::string_t& format) - : m_level(level), m_userFormat(format), m_currentUser(base::utils::OS::currentUser()), - m_currentHost(base::utils::OS::currentHost()) { - parseFromFormat(m_userFormat); -} - -LogFormat::LogFormat(const LogFormat& logFormat): - m_level(logFormat.m_level), - m_userFormat(logFormat.m_userFormat), - m_format(logFormat.m_format), - m_dateTimeFormat(logFormat.m_dateTimeFormat), - m_flags(logFormat.m_flags), - m_currentUser(logFormat.m_currentUser), - m_currentHost(logFormat.m_currentHost) { -} - -LogFormat::LogFormat(LogFormat&& logFormat) { - m_level = std::move(logFormat.m_level); - m_userFormat = std::move(logFormat.m_userFormat); - m_format = std::move(logFormat.m_format); - m_dateTimeFormat = std::move(logFormat.m_dateTimeFormat); - m_flags = std::move(logFormat.m_flags); - m_currentUser = std::move(logFormat.m_currentUser); - m_currentHost = std::move(logFormat.m_currentHost); -} - -LogFormat& LogFormat::operator=(const LogFormat& logFormat) { - if (&logFormat != this) { - m_level = logFormat.m_level; - m_userFormat = logFormat.m_userFormat; - m_dateTimeFormat = logFormat.m_dateTimeFormat; - m_flags = logFormat.m_flags; - m_currentUser = logFormat.m_currentUser; - m_currentHost = logFormat.m_currentHost; - } - return *this; -} - -bool LogFormat::operator==(const LogFormat& other) { - return m_level == other.m_level && m_userFormat == other.m_userFormat && m_format == other.m_format && - m_dateTimeFormat == other.m_dateTimeFormat && m_flags == other.m_flags; -} - -/// @brief Updates format to be used while logging. -/// @param userFormat User provided format -void LogFormat::parseFromFormat(const base::type::string_t& userFormat) { - // We make copy because we will be changing the format - // i.e, removing user provided date format from original format - // and then storing it. - base::type::string_t formatCopy = userFormat; - m_flags = 0x0; - auto conditionalAddFlag = [&](const base::type::char_t* specifier, base::FormatFlags flag) { - std::size_t foundAt = base::type::string_t::npos; - while ((foundAt = formatCopy.find(specifier, foundAt + 1)) != base::type::string_t::npos) { - if (foundAt > 0 && formatCopy[foundAt - 1] == base::consts::kFormatSpecifierChar) { - if (hasFlag(flag)) { - // If we already have flag we remove the escape chars so that '%%' is turned to '%' - // even after specifier resolution - this is because we only replaceFirst specifier - formatCopy.erase(foundAt - 1, 1); - ++foundAt; - } - } else { - if (!hasFlag(flag)) addFlag(flag); - } - } - }; - conditionalAddFlag(base::consts::kAppNameFormatSpecifier, base::FormatFlags::AppName); - conditionalAddFlag(base::consts::kSeverityLevelFormatSpecifier, base::FormatFlags::Level); - conditionalAddFlag(base::consts::kSeverityLevelShortFormatSpecifier, base::FormatFlags::LevelShort); - conditionalAddFlag(base::consts::kLoggerIdFormatSpecifier, base::FormatFlags::LoggerId); - conditionalAddFlag(base::consts::kThreadIdFormatSpecifier, base::FormatFlags::ThreadId); - conditionalAddFlag(base::consts::kLogFileFormatSpecifier, base::FormatFlags::File); - conditionalAddFlag(base::consts::kLogFileBaseFormatSpecifier, base::FormatFlags::FileBase); - conditionalAddFlag(base::consts::kLogLineFormatSpecifier, base::FormatFlags::Line); - conditionalAddFlag(base::consts::kLogLocationFormatSpecifier, base::FormatFlags::Location); - conditionalAddFlag(base::consts::kLogFunctionFormatSpecifier, base::FormatFlags::Function); - conditionalAddFlag(base::consts::kCurrentUserFormatSpecifier, base::FormatFlags::User); - conditionalAddFlag(base::consts::kCurrentHostFormatSpecifier, base::FormatFlags::Host); - conditionalAddFlag(base::consts::kMessageFormatSpecifier, base::FormatFlags::LogMessage); - conditionalAddFlag(base::consts::kVerboseLevelFormatSpecifier, base::FormatFlags::VerboseLevel); - // For date/time we need to extract user's date format first - std::size_t dateIndex = std::string::npos; - if ((dateIndex = formatCopy.find(base::consts::kDateTimeFormatSpecifier)) != std::string::npos) { - while (dateIndex > 0 && formatCopy[dateIndex - 1] == base::consts::kFormatSpecifierChar) { - dateIndex = formatCopy.find(base::consts::kDateTimeFormatSpecifier, dateIndex + 1); - } - if (dateIndex != std::string::npos) { - addFlag(base::FormatFlags::DateTime); - updateDateFormat(dateIndex, formatCopy); - } - } - m_format = formatCopy; - updateFormatSpec(); -} - -void LogFormat::updateDateFormat(std::size_t index, base::type::string_t& currFormat) { - if (hasFlag(base::FormatFlags::DateTime)) { - index += ELPP_STRLEN(base::consts::kDateTimeFormatSpecifier); - } - const base::type::char_t* ptr = currFormat.c_str() + index; - if ((currFormat.size() > index) && (ptr[0] == '{')) { - // User has provided format for date/time - ++ptr; - int count = 1; // Start by 1 in order to remove starting brace - std::stringstream ss; - for (; *ptr; ++ptr, ++count) { - if (*ptr == '}') { - ++count; // In order to remove ending brace - break; - } - ss << static_cast(*ptr); - } - currFormat.erase(index, count); - m_dateTimeFormat = ss.str(); - } else { - // No format provided, use default - if (hasFlag(base::FormatFlags::DateTime)) { - m_dateTimeFormat = std::string(base::consts::kDefaultDateTimeFormat); - } - } -} - -void LogFormat::updateFormatSpec(void) { - // Do not use switch over strongly typed enums because Intel C++ compilers dont support them yet. - if (m_level == Level::Debug) { - base::utils::Str::replaceFirstWithEscape(m_format, base::consts::kSeverityLevelFormatSpecifier, - base::consts::kDebugLevelLogValue); - base::utils::Str::replaceFirstWithEscape(m_format, base::consts::kSeverityLevelShortFormatSpecifier, - base::consts::kDebugLevelShortLogValue); - } else if (m_level == Level::Info) { - base::utils::Str::replaceFirstWithEscape(m_format, base::consts::kSeverityLevelFormatSpecifier, - base::consts::kInfoLevelLogValue); - base::utils::Str::replaceFirstWithEscape(m_format, base::consts::kSeverityLevelShortFormatSpecifier, - base::consts::kInfoLevelShortLogValue); - } else if (m_level == Level::Warning) { - base::utils::Str::replaceFirstWithEscape(m_format, base::consts::kSeverityLevelFormatSpecifier, - base::consts::kWarningLevelLogValue); - base::utils::Str::replaceFirstWithEscape(m_format, base::consts::kSeverityLevelShortFormatSpecifier, - base::consts::kWarningLevelShortLogValue); - } else if (m_level == Level::Error) { - base::utils::Str::replaceFirstWithEscape(m_format, base::consts::kSeverityLevelFormatSpecifier, - base::consts::kErrorLevelLogValue); - base::utils::Str::replaceFirstWithEscape(m_format, base::consts::kSeverityLevelShortFormatSpecifier, - base::consts::kErrorLevelShortLogValue); - } else if (m_level == Level::Fatal) { - base::utils::Str::replaceFirstWithEscape(m_format, base::consts::kSeverityLevelFormatSpecifier, - base::consts::kFatalLevelLogValue); - base::utils::Str::replaceFirstWithEscape(m_format, base::consts::kSeverityLevelShortFormatSpecifier, - base::consts::kFatalLevelShortLogValue); - } else if (m_level == Level::Verbose) { - base::utils::Str::replaceFirstWithEscape(m_format, base::consts::kSeverityLevelFormatSpecifier, - base::consts::kVerboseLevelLogValue); - base::utils::Str::replaceFirstWithEscape(m_format, base::consts::kSeverityLevelShortFormatSpecifier, - base::consts::kVerboseLevelShortLogValue); - } else if (m_level == Level::Trace) { - base::utils::Str::replaceFirstWithEscape(m_format, base::consts::kSeverityLevelFormatSpecifier, - base::consts::kTraceLevelLogValue); - base::utils::Str::replaceFirstWithEscape(m_format, base::consts::kSeverityLevelShortFormatSpecifier, - base::consts::kTraceLevelShortLogValue); - } - if (hasFlag(base::FormatFlags::User)) { - base::utils::Str::replaceFirstWithEscape(m_format, base::consts::kCurrentUserFormatSpecifier, - m_currentUser); - } - if (hasFlag(base::FormatFlags::Host)) { - base::utils::Str::replaceFirstWithEscape(m_format, base::consts::kCurrentHostFormatSpecifier, - m_currentHost); - } - // Ignore Level::Global and Level::Unknown -} - -// TypedConfigurations - -TypedConfigurations::TypedConfigurations(Configurations* configurations, - base::LogStreamsReferenceMap* logStreamsReference) { - m_configurations = configurations; - m_logStreamsReference = logStreamsReference; - build(m_configurations); -} - -TypedConfigurations::TypedConfigurations(const TypedConfigurations& other) { - this->m_configurations = other.m_configurations; - this->m_logStreamsReference = other.m_logStreamsReference; - build(m_configurations); -} - -bool TypedConfigurations::enabled(Level level) { - return getConfigByVal(level, &m_enabledMap, "enabled"); -} - -bool TypedConfigurations::toFile(Level level) { - return getConfigByVal(level, &m_toFileMap, "toFile"); -} - -const std::string& TypedConfigurations::filename(Level level) { - return getConfigByRef(level, &m_filenameMap, "filename"); -} - -bool TypedConfigurations::toStandardOutput(Level level) { - return getConfigByVal(level, &m_toStandardOutputMap, "toStandardOutput"); -} - -const base::LogFormat& TypedConfigurations::logFormat(Level level) { - return getConfigByRef(level, &m_logFormatMap, "logFormat"); -} - -const base::SubsecondPrecision& TypedConfigurations::subsecondPrecision(Level level) { - return getConfigByRef(level, &m_subsecondPrecisionMap, "subsecondPrecision"); -} - -const base::MillisecondsWidth& TypedConfigurations::millisecondsWidth(Level level) { - return getConfigByRef(level, &m_subsecondPrecisionMap, "millisecondsWidth"); -} - -bool TypedConfigurations::performanceTracking(Level level) { - return getConfigByVal(level, &m_performanceTrackingMap, "performanceTracking"); -} - -base::type::fstream_t* TypedConfigurations::fileStream(Level level) { - return getConfigByRef(level, &m_fileStreamMap, "fileStream").get(); -} - -std::size_t TypedConfigurations::maxLogFileSize(Level level) { - return getConfigByVal(level, &m_maxLogFileSizeMap, "maxLogFileSize"); -} - -std::size_t TypedConfigurations::logFlushThreshold(Level level) { - return getConfigByVal(level, &m_logFlushThresholdMap, "logFlushThreshold"); -} - -void TypedConfigurations::build(Configurations* configurations) { - base::threading::ScopedLock scopedLock(lock()); - auto getBool = [] (std::string boolStr) -> bool { // Pass by value for trimming - base::utils::Str::trim(boolStr); - return (boolStr == "TRUE" || boolStr == "true" || boolStr == "1"); - }; - std::vector withFileSizeLimit; - for (Configurations::const_iterator it = configurations->begin(); it != configurations->end(); ++it) { - Configuration* conf = *it; - // We cannot use switch on strong enums because Intel C++ dont support them yet - if (conf->configurationType() == ConfigurationType::Enabled) { - setValue(conf->level(), getBool(conf->value()), &m_enabledMap); - } else if (conf->configurationType() == ConfigurationType::ToFile) { - setValue(conf->level(), getBool(conf->value()), &m_toFileMap); - } else if (conf->configurationType() == ConfigurationType::ToStandardOutput) { - setValue(conf->level(), getBool(conf->value()), &m_toStandardOutputMap); - } else if (conf->configurationType() == ConfigurationType::Filename) { - // We do not yet configure filename but we will configure in another - // loop. This is because if file cannot be created, we will force ToFile - // to be false. Because configuring logger is not necessarily performance - // sensative operation, we can live with another loop; (by the way this loop - // is not very heavy either) - } else if (conf->configurationType() == ConfigurationType::Format) { - setValue(conf->level(), base::LogFormat(conf->level(), - base::type::string_t(conf->value().begin(), conf->value().end())), &m_logFormatMap); - } else if (conf->configurationType() == ConfigurationType::SubsecondPrecision) { - setValue(Level::Global, - base::SubsecondPrecision(static_cast(getULong(conf->value()))), &m_subsecondPrecisionMap); - } else if (conf->configurationType() == ConfigurationType::PerformanceTracking) { - setValue(Level::Global, getBool(conf->value()), &m_performanceTrackingMap); - } else if (conf->configurationType() == ConfigurationType::MaxLogFileSize) { - auto v = getULong(conf->value()); - setValue(conf->level(), static_cast(v), &m_maxLogFileSizeMap); - if (v != 0) { - withFileSizeLimit.push_back(conf); - } - } else if (conf->configurationType() == ConfigurationType::LogFlushThreshold) { - setValue(conf->level(), static_cast(getULong(conf->value())), &m_logFlushThresholdMap); - } - } - // As mentioned earlier, we will now set filename configuration in separate loop to deal with non-existent files - for (Configurations::const_iterator it = configurations->begin(); it != configurations->end(); ++it) { - Configuration* conf = *it; - if (conf->configurationType() == ConfigurationType::Filename) { - insertFile(conf->level(), conf->value()); - } - } - for (std::vector::iterator conf = withFileSizeLimit.begin(); - conf != withFileSizeLimit.end(); ++conf) { - // This is not unsafe as mutex is locked in currect scope - unsafeValidateFileRolling((*conf)->level(), base::defaultPreRollOutCallback); - } -} - -unsigned long TypedConfigurations::getULong(std::string confVal) { - bool valid = true; - base::utils::Str::trim(confVal); - valid = !confVal.empty() && std::find_if(confVal.begin(), confVal.end(), - [](char c) { - return !base::utils::Str::isDigit(c); - }) == confVal.end(); - if (!valid) { - valid = false; - ELPP_ASSERT(valid, "Configuration value not a valid integer [" << confVal << "]"); - return 0; - } - return atol(confVal.c_str()); -} - -std::string TypedConfigurations::resolveFilename(const std::string& filename) { - std::string resultingFilename = filename; - std::size_t dateIndex = std::string::npos; - std::string dateTimeFormatSpecifierStr = std::string(base::consts::kDateTimeFormatSpecifierForFilename); - if ((dateIndex = resultingFilename.find(dateTimeFormatSpecifierStr.c_str())) != std::string::npos) { - while (dateIndex > 0 && resultingFilename[dateIndex - 1] == base::consts::kFormatSpecifierChar) { - dateIndex = resultingFilename.find(dateTimeFormatSpecifierStr.c_str(), dateIndex + 1); - } - if (dateIndex != std::string::npos) { - const char* ptr = resultingFilename.c_str() + dateIndex; - // Goto end of specifier - ptr += dateTimeFormatSpecifierStr.size(); - std::string fmt; - if ((resultingFilename.size() > dateIndex) && (ptr[0] == '{')) { - // User has provided format for date/time - ++ptr; - int count = 1; // Start by 1 in order to remove starting brace - std::stringstream ss; - for (; *ptr; ++ptr, ++count) { - if (*ptr == '}') { - ++count; // In order to remove ending brace - break; - } - ss << *ptr; - } - resultingFilename.erase(dateIndex + dateTimeFormatSpecifierStr.size(), count); - fmt = ss.str(); - } else { - fmt = std::string(base::consts::kDefaultDateTimeFormatInFilename); - } - base::SubsecondPrecision ssPrec(3); - std::string now = base::utils::DateTime::getDateTime(fmt.c_str(), &ssPrec); - base::utils::Str::replaceAll(now, '/', '-'); // Replace path element since we are dealing with filename - base::utils::Str::replaceAll(resultingFilename, dateTimeFormatSpecifierStr, now); - } - } - return resultingFilename; -} - -void TypedConfigurations::insertFile(Level level, const std::string& fullFilename) { - std::string resolvedFilename = resolveFilename(fullFilename); - if (resolvedFilename.empty()) { - std::cerr << "Could not load empty file for logging, please re-check your configurations for level [" - << LevelHelper::convertToString(level) << "]"; - } - std::string filePath = base::utils::File::extractPathFromFilename(resolvedFilename, base::consts::kFilePathSeperator); - if (filePath.size() < resolvedFilename.size()) { - base::utils::File::createPath(filePath); - } - auto create = [&](Level level) { - base::LogStreamsReferenceMap::iterator filestreamIter = m_logStreamsReference->find(resolvedFilename); - base::type::fstream_t* fs = nullptr; - if (filestreamIter == m_logStreamsReference->end()) { - // We need a completely new stream, nothing to share with - fs = base::utils::File::newFileStream(resolvedFilename); - m_filenameMap.insert(std::make_pair(level, resolvedFilename)); - m_fileStreamMap.insert(std::make_pair(level, base::FileStreamPtr(fs))); - m_logStreamsReference->insert(std::make_pair(resolvedFilename, base::FileStreamPtr(m_fileStreamMap.at(level)))); - } else { - // Woops! we have an existing one, share it! - m_filenameMap.insert(std::make_pair(level, filestreamIter->first)); - m_fileStreamMap.insert(std::make_pair(level, base::FileStreamPtr(filestreamIter->second))); - fs = filestreamIter->second.get(); - } - if (fs == nullptr) { - // We display bad file error from newFileStream() - ELPP_INTERNAL_ERROR("Setting [TO_FILE] of [" - << LevelHelper::convertToString(level) << "] to FALSE", false); - setValue(level, false, &m_toFileMap); - } - }; - // If we dont have file conf for any level, create it for Level::Global first - // otherwise create for specified level - create(m_filenameMap.empty() && m_fileStreamMap.empty() ? Level::Global : level); -} - -bool TypedConfigurations::unsafeValidateFileRolling(Level level, const PreRollOutCallback& preRollOutCallback) { - base::type::fstream_t* fs = unsafeGetConfigByRef(level, &m_fileStreamMap, "fileStream").get(); - if (fs == nullptr) { - return true; - } - std::size_t maxLogFileSize = unsafeGetConfigByVal(level, &m_maxLogFileSizeMap, "maxLogFileSize"); - std::size_t currFileSize = base::utils::File::getSizeOfFile(fs); - if (maxLogFileSize != 0 && currFileSize >= maxLogFileSize) { - std::string fname = unsafeGetConfigByRef(level, &m_filenameMap, "filename"); - ELPP_INTERNAL_INFO(1, "Truncating log file [" << fname << "] as a result of configurations for level [" - << LevelHelper::convertToString(level) << "]"); - fs->close(); - preRollOutCallback(fname.c_str(), currFileSize, level); - fs->open(fname, std::fstream::out | std::fstream::trunc); - return true; - } - return false; -} - -// RegisteredHitCounters - -bool RegisteredHitCounters::validateEveryN(const char* filename, base::type::LineNumber lineNumber, std::size_t n) { - base::threading::ScopedLock scopedLock(lock()); - base::HitCounter* counter = get(filename, lineNumber); - if (counter == nullptr) { - registerNew(counter = new base::HitCounter(filename, lineNumber)); - } - counter->validateHitCounts(n); - bool result = (n >= 1 && counter->hitCounts() != 0 && counter->hitCounts() % n == 0); - return result; -} - -/// @brief Validates counter for hits >= N, i.e, registers new if does not exist otherwise updates original one -/// @return True if validation resulted in triggering hit. Meaning logs should be written everytime true is returned -bool RegisteredHitCounters::validateAfterN(const char* filename, base::type::LineNumber lineNumber, std::size_t n) { - base::threading::ScopedLock scopedLock(lock()); - base::HitCounter* counter = get(filename, lineNumber); - if (counter == nullptr) { - registerNew(counter = new base::HitCounter(filename, lineNumber)); - } - // Do not use validateHitCounts here since we do not want to reset counter here - // Note the >= instead of > because we are incrementing - // after this check - if (counter->hitCounts() >= n) - return true; - counter->increment(); - return false; -} - -/// @brief Validates counter for hits are <= n, i.e, registers new if does not exist otherwise updates original one -/// @return True if validation resulted in triggering hit. Meaning logs should be written everytime true is returned -bool RegisteredHitCounters::validateNTimes(const char* filename, base::type::LineNumber lineNumber, std::size_t n) { - base::threading::ScopedLock scopedLock(lock()); - base::HitCounter* counter = get(filename, lineNumber); - if (counter == nullptr) { - registerNew(counter = new base::HitCounter(filename, lineNumber)); - } - counter->increment(); - // Do not use validateHitCounts here since we do not want to reset counter here - if (counter->hitCounts() <= n) - return true; - return false; -} - -// RegisteredLoggers - -RegisteredLoggers::RegisteredLoggers(const LogBuilderPtr& defaultLogBuilder) : - m_defaultLogBuilder(defaultLogBuilder) { - m_defaultConfigurations.setToDefault(); -} - -Logger* RegisteredLoggers::get(const std::string& id, bool forceCreation) { - base::threading::ScopedLock scopedLock(lock()); - Logger* logger_ = base::utils::Registry::get(id); - if (logger_ == nullptr && forceCreation) { - bool validId = Logger::isValidId(id); - if (!validId) { - ELPP_ASSERT(validId, "Invalid logger ID [" << id << "]. Not registering this logger."); - return nullptr; - } - logger_ = new Logger(id, m_defaultConfigurations, &m_logStreamsReference); - logger_->m_logBuilder = m_defaultLogBuilder; - registerNew(id, logger_); - LoggerRegistrationCallback* callback = nullptr; - for (const std::pair& h - : m_loggerRegistrationCallbacks) { - callback = h.second.get(); - if (callback != nullptr && callback->enabled()) { - callback->handle(logger_); - } - } - } - return logger_; -} - -bool RegisteredLoggers::remove(const std::string& id) { - if (id == base::consts::kDefaultLoggerId) { - return false; - } - // get has internal lock - Logger* logger = base::utils::Registry::get(id); - if (logger != nullptr) { - // unregister has internal lock - unregister(logger); - } - return true; -} - -void RegisteredLoggers::unsafeFlushAll(void) { - ELPP_INTERNAL_INFO(1, "Flushing all log files"); - for (base::LogStreamsReferenceMap::iterator it = m_logStreamsReference.begin(); - it != m_logStreamsReference.end(); ++it) { - if (it->second.get() == nullptr) continue; - it->second->flush(); - } -} - -// VRegistry - -VRegistry::VRegistry(base::type::VerboseLevel level, base::type::EnumType* pFlags) : m_level(level), m_pFlags(pFlags) { -} - -/// @brief Sets verbose level. Accepted range is 0-9 -void VRegistry::setLevel(base::type::VerboseLevel level) { - base::threading::ScopedLock scopedLock(lock()); - if (level > 9) - m_level = base::consts::kMaxVerboseLevel; - else - m_level = level; -} - -void VRegistry::setModules(const char* modules) { - base::threading::ScopedLock scopedLock(lock()); - auto addSuffix = [](std::stringstream& ss, const char* sfx, const char* prev) { - if (prev != nullptr && base::utils::Str::endsWith(ss.str(), std::string(prev))) { - std::string chr(ss.str().substr(0, ss.str().size() - strlen(prev))); - ss.str(std::string("")); - ss << chr; - } - if (base::utils::Str::endsWith(ss.str(), std::string(sfx))) { - std::string chr(ss.str().substr(0, ss.str().size() - strlen(sfx))); - ss.str(std::string("")); - ss << chr; - } - ss << sfx; - }; - auto insert = [&](std::stringstream& ss, base::type::VerboseLevel level) { - if (!base::utils::hasFlag(LoggingFlag::DisableVModulesExtensions, *m_pFlags)) { - addSuffix(ss, ".h", nullptr); - m_modules.insert(std::make_pair(ss.str(), level)); - addSuffix(ss, ".c", ".h"); - m_modules.insert(std::make_pair(ss.str(), level)); - addSuffix(ss, ".cpp", ".c"); - m_modules.insert(std::make_pair(ss.str(), level)); - addSuffix(ss, ".cc", ".cpp"); - m_modules.insert(std::make_pair(ss.str(), level)); - addSuffix(ss, ".cxx", ".cc"); - m_modules.insert(std::make_pair(ss.str(), level)); - addSuffix(ss, ".-inl.h", ".cxx"); - m_modules.insert(std::make_pair(ss.str(), level)); - addSuffix(ss, ".hxx", ".-inl.h"); - m_modules.insert(std::make_pair(ss.str(), level)); - addSuffix(ss, ".hpp", ".hxx"); - m_modules.insert(std::make_pair(ss.str(), level)); - addSuffix(ss, ".hh", ".hpp"); - } - m_modules.insert(std::make_pair(ss.str(), level)); - }; - bool isMod = true; - bool isLevel = false; - std::stringstream ss; - int level = -1; - for (; *modules; ++modules) { - switch (*modules) { - case '=': - isLevel = true; - isMod = false; - break; - case ',': - isLevel = false; - isMod = true; - if (!ss.str().empty() && level != -1) { - insert(ss, static_cast(level)); - ss.str(std::string("")); - level = -1; - } - break; - default: - if (isMod) { - ss << *modules; - } else if (isLevel) { - if (isdigit(*modules)) { - level = static_cast(*modules) - 48; - } - } - break; - } - } - if (!ss.str().empty() && level != -1) { - insert(ss, static_cast(level)); - } -} - -bool VRegistry::allowed(base::type::VerboseLevel vlevel, const char* file) { - base::threading::ScopedLock scopedLock(lock()); - if (m_modules.empty() || file == nullptr) { - return vlevel <= m_level; - } else { - char baseFilename[base::consts::kSourceFilenameMaxLength] = ""; - base::utils::File::buildBaseFilename(file, baseFilename); - std::unordered_map::iterator it = m_modules.begin(); - for (; it != m_modules.end(); ++it) { - if (base::utils::Str::wildCardMatch(baseFilename, it->first.c_str())) { - return vlevel <= it->second; - } - } - if (base::utils::hasFlag(LoggingFlag::AllowVerboseIfModuleNotSpecified, *m_pFlags)) { - return true; - } - return false; - } -} - -void VRegistry::setFromArgs(const base::utils::CommandLineArgs* commandLineArgs) { - if (commandLineArgs->hasParam("-v") || commandLineArgs->hasParam("--verbose") || - commandLineArgs->hasParam("-V") || commandLineArgs->hasParam("--VERBOSE")) { - setLevel(base::consts::kMaxVerboseLevel); - } else if (commandLineArgs->hasParamWithValue("--v")) { - setLevel(static_cast(atoi(commandLineArgs->getParamValue("--v")))); - } else if (commandLineArgs->hasParamWithValue("--V")) { - setLevel(static_cast(atoi(commandLineArgs->getParamValue("--V")))); - } else if ((commandLineArgs->hasParamWithValue("-vmodule")) && vModulesEnabled()) { - setModules(commandLineArgs->getParamValue("-vmodule")); - } else if (commandLineArgs->hasParamWithValue("-VMODULE") && vModulesEnabled()) { - setModules(commandLineArgs->getParamValue("-VMODULE")); - } -} - -#if !defined(ELPP_DEFAULT_LOGGING_FLAGS) -# define ELPP_DEFAULT_LOGGING_FLAGS 0x0 -#endif // !defined(ELPP_DEFAULT_LOGGING_FLAGS) -// Storage -#if ELPP_ASYNC_LOGGING -Storage::Storage(const LogBuilderPtr& defaultLogBuilder, base::IWorker* asyncDispatchWorker) : -#else -Storage::Storage(const LogBuilderPtr& defaultLogBuilder) : -#endif // ELPP_ASYNC_LOGGING - m_registeredHitCounters(new base::RegisteredHitCounters()), - m_registeredLoggers(new base::RegisteredLoggers(defaultLogBuilder)), - m_flags(ELPP_DEFAULT_LOGGING_FLAGS), - m_vRegistry(new base::VRegistry(0, &m_flags)), - -#if ELPP_ASYNC_LOGGING - m_asyncLogQueue(new base::AsyncLogQueue()), - m_asyncDispatchWorker(asyncDispatchWorker), -#endif // ELPP_ASYNC_LOGGING - - m_preRollOutCallback(base::defaultPreRollOutCallback) { - // Register default logger - m_registeredLoggers->get(std::string(base::consts::kDefaultLoggerId)); - // We register default logger anyway (worse case it's not going to register) just in case - m_registeredLoggers->get("default"); - -#if defined(ELPP_FEATURE_ALL) || defined(ELPP_FEATURE_PERFORMANCE_TRACKING) - // Register performance logger and reconfigure format - Logger* performanceLogger = m_registeredLoggers->get(std::string(base::consts::kPerformanceLoggerId)); - m_registeredLoggers->get("performance"); - performanceLogger->configurations()->setGlobally(ConfigurationType::Format, std::string("%datetime %level %msg")); - performanceLogger->reconfigure(); -#endif // defined(ELPP_FEATURE_ALL) || defined(ELPP_FEATURE_PERFORMANCE_TRACKING) - -#if defined(ELPP_SYSLOG) - // Register syslog logger and reconfigure format - Logger* sysLogLogger = m_registeredLoggers->get(std::string(base::consts::kSysLogLoggerId)); - sysLogLogger->configurations()->setGlobally(ConfigurationType::Format, std::string("%level: %msg")); - sysLogLogger->reconfigure(); -#endif // defined(ELPP_SYSLOG) - addFlag(LoggingFlag::AllowVerboseIfModuleNotSpecified); -#if ELPP_ASYNC_LOGGING - installLogDispatchCallback(std::string("AsyncLogDispatchCallback")); -#else - installLogDispatchCallback(std::string("DefaultLogDispatchCallback")); -#endif // ELPP_ASYNC_LOGGING -#if defined(ELPP_FEATURE_ALL) || defined(ELPP_FEATURE_PERFORMANCE_TRACKING) - installPerformanceTrackingCallback - (std::string("DefaultPerformanceTrackingCallback")); -#endif // defined(ELPP_FEATURE_ALL) || defined(ELPP_FEATURE_PERFORMANCE_TRACKING) - ELPP_INTERNAL_INFO(1, "Easylogging++ has been initialized"); -#if ELPP_ASYNC_LOGGING - m_asyncDispatchWorker->start(); -#endif // ELPP_ASYNC_LOGGING -} - -Storage::~Storage(void) { - ELPP_INTERNAL_INFO(4, "Destroying storage"); -#if ELPP_ASYNC_LOGGING - ELPP_INTERNAL_INFO(5, "Replacing log dispatch callback to synchronous"); - uninstallLogDispatchCallback(std::string("AsyncLogDispatchCallback")); - installLogDispatchCallback(std::string("DefaultLogDispatchCallback")); - ELPP_INTERNAL_INFO(5, "Destroying asyncDispatchWorker"); - base::utils::safeDelete(m_asyncDispatchWorker); - ELPP_INTERNAL_INFO(5, "Destroying asyncLogQueue"); - base::utils::safeDelete(m_asyncLogQueue); -#endif // ELPP_ASYNC_LOGGING - ELPP_INTERNAL_INFO(5, "Destroying registeredHitCounters"); - base::utils::safeDelete(m_registeredHitCounters); - ELPP_INTERNAL_INFO(5, "Destroying registeredLoggers"); - base::utils::safeDelete(m_registeredLoggers); - ELPP_INTERNAL_INFO(5, "Destroying vRegistry"); - base::utils::safeDelete(m_vRegistry); -} - -bool Storage::hasCustomFormatSpecifier(const char* formatSpecifier) { - base::threading::ScopedLock scopedLock(customFormatSpecifiersLock()); - return std::find(m_customFormatSpecifiers.begin(), m_customFormatSpecifiers.end(), - formatSpecifier) != m_customFormatSpecifiers.end(); -} - -void Storage::installCustomFormatSpecifier(const CustomFormatSpecifier& customFormatSpecifier) { - if (hasCustomFormatSpecifier(customFormatSpecifier.formatSpecifier())) { - return; - } - base::threading::ScopedLock scopedLock(customFormatSpecifiersLock()); - m_customFormatSpecifiers.push_back(customFormatSpecifier); -} - -bool Storage::uninstallCustomFormatSpecifier(const char* formatSpecifier) { - base::threading::ScopedLock scopedLock(customFormatSpecifiersLock()); - std::vector::iterator it = std::find(m_customFormatSpecifiers.begin(), - m_customFormatSpecifiers.end(), formatSpecifier); - if (it != m_customFormatSpecifiers.end() && strcmp(formatSpecifier, it->formatSpecifier()) == 0) { - m_customFormatSpecifiers.erase(it); - return true; - } - return false; -} - -void Storage::setApplicationArguments(int argc, char** argv) { - m_commandLineArgs.setArgs(argc, argv); - m_vRegistry->setFromArgs(commandLineArgs()); - // default log file -#if !defined(ELPP_DISABLE_LOG_FILE_FROM_ARG) - if (m_commandLineArgs.hasParamWithValue(base::consts::kDefaultLogFileParam)) { - Configurations c; - c.setGlobally(ConfigurationType::Filename, - std::string(m_commandLineArgs.getParamValue(base::consts::kDefaultLogFileParam))); - registeredLoggers()->setDefaultConfigurations(c); - for (base::RegisteredLoggers::iterator it = registeredLoggers()->begin(); - it != registeredLoggers()->end(); ++it) { - it->second->configure(c); - } - } -#endif // !defined(ELPP_DISABLE_LOG_FILE_FROM_ARG) -#if defined(ELPP_LOGGING_FLAGS_FROM_ARG) - if (m_commandLineArgs.hasParamWithValue(base::consts::kLoggingFlagsParam)) { - int userInput = atoi(m_commandLineArgs.getParamValue(base::consts::kLoggingFlagsParam)); - if (ELPP_DEFAULT_LOGGING_FLAGS == 0x0) { - m_flags = userInput; - } else { - base::utils::addFlag(userInput, &m_flags); - } - } -#endif // defined(ELPP_LOGGING_FLAGS_FROM_ARG) -} - -} // namespace base - -// LogDispatchCallback -void LogDispatchCallback::handle(const LogDispatchData* data) { -#if defined(ELPP_THREAD_SAFE) - base::threading::ScopedLock scopedLock(m_fileLocksMapLock); - std::string filename = data->logMessage()->logger()->typedConfigurations()->filename(data->logMessage()->level()); - auto lock = m_fileLocks.find(filename); - if (lock == m_fileLocks.end()) { - m_fileLocks.emplace(std::make_pair(filename, std::unique_ptr(new base::threading::Mutex))); - } -#endif -} - -base::threading::Mutex& LogDispatchCallback::fileHandle(const LogDispatchData* data) { - auto it = m_fileLocks.find(data->logMessage()->logger()->typedConfigurations()->filename(data->logMessage()->level())); - return *(it->second.get()); -} - -namespace base { -// DefaultLogDispatchCallback - -void DefaultLogDispatchCallback::handle(const LogDispatchData* data) { -#if defined(ELPP_THREAD_SAFE) - LogDispatchCallback::handle(data); - base::threading::ScopedLock scopedLock(fileHandle(data)); -#endif - m_data = data; - dispatch(m_data->logMessage()->logger()->logBuilder()->build(m_data->logMessage(), - m_data->dispatchAction() == base::DispatchAction::NormalLog)); -} - -void DefaultLogDispatchCallback::dispatch(base::type::string_t&& logLine) { - if (m_data->dispatchAction() == base::DispatchAction::NormalLog) { - if (m_data->logMessage()->logger()->m_typedConfigurations->toFile(m_data->logMessage()->level())) { - base::type::fstream_t* fs = m_data->logMessage()->logger()->m_typedConfigurations->fileStream( - m_data->logMessage()->level()); - if (fs != nullptr) { - fs->write(logLine.c_str(), logLine.size()); - if (fs->fail()) { - ELPP_INTERNAL_ERROR("Unable to write log to file [" - << m_data->logMessage()->logger()->m_typedConfigurations->filename(m_data->logMessage()->level()) << "].\n" - << "Few possible reasons (could be something else):\n" << " * Permission denied\n" - << " * Disk full\n" << " * Disk is not writable", true); - } else { - if (ELPP->hasFlag(LoggingFlag::ImmediateFlush) - || (m_data->logMessage()->logger()->isFlushNeeded(m_data->logMessage()->level()))) { - m_data->logMessage()->logger()->flush(m_data->logMessage()->level(), fs); - } - } - } else { - ELPP_INTERNAL_ERROR("Log file for [" << LevelHelper::convertToString(m_data->logMessage()->level()) << "] " - << "has not been configured but [TO_FILE] is configured to TRUE. [Logger ID: " - << m_data->logMessage()->logger()->id() << "]", false); - } - } - if (m_data->logMessage()->logger()->m_typedConfigurations->toStandardOutput(m_data->logMessage()->level())) { - if (ELPP->hasFlag(LoggingFlag::ColoredTerminalOutput)) - m_data->logMessage()->logger()->logBuilder()->convertToColoredOutput(&logLine, m_data->logMessage()->level()); - ELPP_COUT << ELPP_COUT_LINE(logLine); - } - } -#if defined(ELPP_SYSLOG) - else if (m_data->dispatchAction() == base::DispatchAction::SysLog) { - // Determine syslog priority - int sysLogPriority = 0; - if (m_data->logMessage()->level() == Level::Fatal) - sysLogPriority = LOG_EMERG; - else if (m_data->logMessage()->level() == Level::Error) - sysLogPriority = LOG_ERR; - else if (m_data->logMessage()->level() == Level::Warning) - sysLogPriority = LOG_WARNING; - else if (m_data->logMessage()->level() == Level::Info) - sysLogPriority = LOG_INFO; - else if (m_data->logMessage()->level() == Level::Debug) - sysLogPriority = LOG_DEBUG; - else - sysLogPriority = LOG_NOTICE; -# if defined(ELPP_UNICODE) - char* line = base::utils::Str::wcharPtrToCharPtr(logLine.c_str()); - syslog(sysLogPriority, "%s", line); - free(line); -# else - syslog(sysLogPriority, "%s", logLine.c_str()); -# endif - } -#endif // defined(ELPP_SYSLOG) -} - -#if ELPP_ASYNC_LOGGING - -// AsyncLogDispatchCallback - -void AsyncLogDispatchCallback::handle(const LogDispatchData* data) { - base::type::string_t logLine = data->logMessage()->logger()->logBuilder()->build(data->logMessage(), - data->dispatchAction() == base::DispatchAction::NormalLog); - if (data->dispatchAction() == base::DispatchAction::NormalLog - && data->logMessage()->logger()->typedConfigurations()->toStandardOutput(data->logMessage()->level())) { - if (ELPP->hasFlag(LoggingFlag::ColoredTerminalOutput)) - data->logMessage()->logger()->logBuilder()->convertToColoredOutput(&logLine, data->logMessage()->level()); - ELPP_COUT << ELPP_COUT_LINE(logLine); - } - // Save resources and only queue if we want to write to file otherwise just ignore handler - if (data->logMessage()->logger()->typedConfigurations()->toFile(data->logMessage()->level())) { - ELPP->asyncLogQueue()->push(AsyncLogItem(*(data->logMessage()), *data, logLine)); - } -} - -// AsyncDispatchWorker -AsyncDispatchWorker::AsyncDispatchWorker() { - setContinueRunning(false); -} - -AsyncDispatchWorker::~AsyncDispatchWorker() { - setContinueRunning(false); - ELPP_INTERNAL_INFO(6, "Stopping dispatch worker - Cleaning log queue"); - clean(); - ELPP_INTERNAL_INFO(6, "Log queue cleaned"); -} - -bool AsyncDispatchWorker::clean(void) { - std::mutex m; - std::unique_lock lk(m); - cv.wait(lk, [] { return !ELPP->asyncLogQueue()->empty(); }); - emptyQueue(); - lk.unlock(); - cv.notify_one(); - return ELPP->asyncLogQueue()->empty(); -} - -void AsyncDispatchWorker::emptyQueue(void) { - while (!ELPP->asyncLogQueue()->empty()) { - AsyncLogItem data = ELPP->asyncLogQueue()->next(); - handle(&data); - base::threading::msleep(100); - } -} - -void AsyncDispatchWorker::start(void) { - base::threading::msleep(5000); // 5s (why?) - setContinueRunning(true); - std::thread t1(&AsyncDispatchWorker::run, this); - t1.join(); -} - -void AsyncDispatchWorker::handle(AsyncLogItem* logItem) { - LogDispatchData* data = logItem->data(); - LogMessage* logMessage = logItem->logMessage(); - Logger* logger = logMessage->logger(); - base::TypedConfigurations* conf = logger->typedConfigurations(); - base::type::string_t logLine = logItem->logLine(); - if (data->dispatchAction() == base::DispatchAction::NormalLog) { - if (conf->toFile(logMessage->level())) { - base::type::fstream_t* fs = conf->fileStream(logMessage->level()); - if (fs != nullptr) { - fs->write(logLine.c_str(), logLine.size()); - if (fs->fail()) { - ELPP_INTERNAL_ERROR("Unable to write log to file [" - << conf->filename(logMessage->level()) << "].\n" - << "Few possible reasons (could be something else):\n" << " * Permission denied\n" - << " * Disk full\n" << " * Disk is not writable", true); - } else { - if (ELPP->hasFlag(LoggingFlag::ImmediateFlush) || (logger->isFlushNeeded(logMessage->level()))) { - logger->flush(logMessage->level(), fs); - } - } - } else { - ELPP_INTERNAL_ERROR("Log file for [" << LevelHelper::convertToString(logMessage->level()) << "] " - << "has not been configured but [TO_FILE] is configured to TRUE. [Logger ID: " << logger->id() << "]", false); - } - } - } -# if defined(ELPP_SYSLOG) - else if (data->dispatchAction() == base::DispatchAction::SysLog) { - // Determine syslog priority - int sysLogPriority = 0; - if (logMessage->level() == Level::Fatal) - sysLogPriority = LOG_EMERG; - else if (logMessage->level() == Level::Error) - sysLogPriority = LOG_ERR; - else if (logMessage->level() == Level::Warning) - sysLogPriority = LOG_WARNING; - else if (logMessage->level() == Level::Info) - sysLogPriority = LOG_INFO; - else if (logMessage->level() == Level::Debug) - sysLogPriority = LOG_DEBUG; - else - sysLogPriority = LOG_NOTICE; -# if defined(ELPP_UNICODE) - char* line = base::utils::Str::wcharPtrToCharPtr(logLine.c_str()); - syslog(sysLogPriority, "%s", line); - free(line); -# else - syslog(sysLogPriority, "%s", logLine.c_str()); -# endif - } -# endif // defined(ELPP_SYSLOG) -} - -void AsyncDispatchWorker::run(void) { - while (continueRunning()) { - emptyQueue(); - base::threading::msleep(10); // 10ms - } -} -#endif // ELPP_ASYNC_LOGGING - -// DefaultLogBuilder - -base::type::string_t DefaultLogBuilder::build(const LogMessage* logMessage, bool appendNewLine) const { - base::TypedConfigurations* tc = logMessage->logger()->typedConfigurations(); - const base::LogFormat* logFormat = &tc->logFormat(logMessage->level()); - base::type::string_t logLine = logFormat->format(); - char buff[base::consts::kSourceFilenameMaxLength + base::consts::kSourceLineMaxLength] = ""; - const char* bufLim = buff + sizeof(buff); - if (logFormat->hasFlag(base::FormatFlags::AppName)) { - // App name - base::utils::Str::replaceFirstWithEscape(logLine, base::consts::kAppNameFormatSpecifier, - logMessage->logger()->parentApplicationName()); - } - if (logFormat->hasFlag(base::FormatFlags::ThreadId)) { - // Thread ID - base::utils::Str::replaceFirstWithEscape(logLine, base::consts::kThreadIdFormatSpecifier, - ELPP->getThreadName(base::threading::getCurrentThreadId())); - } - if (logFormat->hasFlag(base::FormatFlags::DateTime)) { - // DateTime - base::utils::Str::replaceFirstWithEscape(logLine, base::consts::kDateTimeFormatSpecifier, - base::utils::DateTime::getDateTime(logFormat->dateTimeFormat().c_str(), - &tc->subsecondPrecision(logMessage->level()))); - } - if (logFormat->hasFlag(base::FormatFlags::Function)) { - // Function - base::utils::Str::replaceFirstWithEscape(logLine, base::consts::kLogFunctionFormatSpecifier, logMessage->func()); - } - if (logFormat->hasFlag(base::FormatFlags::File)) { - // File - base::utils::Str::clearBuff(buff, base::consts::kSourceFilenameMaxLength); - base::utils::File::buildStrippedFilename(logMessage->file().c_str(), buff); - base::utils::Str::replaceFirstWithEscape(logLine, base::consts::kLogFileFormatSpecifier, std::string(buff)); - } - if (logFormat->hasFlag(base::FormatFlags::FileBase)) { - // FileBase - base::utils::Str::clearBuff(buff, base::consts::kSourceFilenameMaxLength); - base::utils::File::buildBaseFilename(logMessage->file(), buff); - base::utils::Str::replaceFirstWithEscape(logLine, base::consts::kLogFileBaseFormatSpecifier, std::string(buff)); - } - if (logFormat->hasFlag(base::FormatFlags::Line)) { - // Line - char* buf = base::utils::Str::clearBuff(buff, base::consts::kSourceLineMaxLength); - buf = base::utils::Str::convertAndAddToBuff(logMessage->line(), base::consts::kSourceLineMaxLength, buf, bufLim, false); - base::utils::Str::replaceFirstWithEscape(logLine, base::consts::kLogLineFormatSpecifier, std::string(buff)); - } - if (logFormat->hasFlag(base::FormatFlags::Location)) { - // Location - char* buf = base::utils::Str::clearBuff(buff, - base::consts::kSourceFilenameMaxLength + base::consts::kSourceLineMaxLength); - base::utils::File::buildStrippedFilename(logMessage->file().c_str(), buff); - buf = base::utils::Str::addToBuff(buff, buf, bufLim); - buf = base::utils::Str::addToBuff(":", buf, bufLim); - buf = base::utils::Str::convertAndAddToBuff(logMessage->line(), base::consts::kSourceLineMaxLength, buf, bufLim, - false); - base::utils::Str::replaceFirstWithEscape(logLine, base::consts::kLogLocationFormatSpecifier, std::string(buff)); - } - if (logMessage->level() == Level::Verbose && logFormat->hasFlag(base::FormatFlags::VerboseLevel)) { - // Verbose level - char* buf = base::utils::Str::clearBuff(buff, 1); - buf = base::utils::Str::convertAndAddToBuff(logMessage->verboseLevel(), 1, buf, bufLim, false); - base::utils::Str::replaceFirstWithEscape(logLine, base::consts::kVerboseLevelFormatSpecifier, std::string(buff)); - } - if (logFormat->hasFlag(base::FormatFlags::LogMessage)) { - // Log message - base::utils::Str::replaceFirstWithEscape(logLine, base::consts::kMessageFormatSpecifier, logMessage->message()); - } -#if !defined(ELPP_DISABLE_CUSTOM_FORMAT_SPECIFIERS) - el::base::threading::ScopedLock lock_(ELPP->customFormatSpecifiersLock()); - ELPP_UNUSED(lock_); - for (std::vector::const_iterator it = ELPP->customFormatSpecifiers()->begin(); - it != ELPP->customFormatSpecifiers()->end(); ++it) { - std::string fs(it->formatSpecifier()); - base::type::string_t wcsFormatSpecifier(fs.begin(), fs.end()); - base::utils::Str::replaceFirstWithEscape(logLine, wcsFormatSpecifier, it->resolver()(logMessage)); - } -#endif // !defined(ELPP_DISABLE_CUSTOM_FORMAT_SPECIFIERS) - if (appendNewLine) logLine += ELPP_LITERAL("\n"); - return logLine; -} - -// LogDispatcher - -void LogDispatcher::dispatch(void) { - if (m_proceed && m_dispatchAction == base::DispatchAction::None) { - m_proceed = false; - } - if (!m_proceed) { - return; - } -#ifndef ELPP_NO_GLOBAL_LOCK - // see https://github.com/muflihun/easyloggingpp/issues/580 - // global lock is turned off by default unless - // ELPP_NO_GLOBAL_LOCK is defined - base::threading::ScopedLock scopedLock(ELPP->lock()); -#endif - base::TypedConfigurations* tc = m_logMessage->logger()->m_typedConfigurations; - if (ELPP->hasFlag(LoggingFlag::StrictLogFileSizeCheck)) { - tc->validateFileRolling(m_logMessage->level(), ELPP->preRollOutCallback()); - } - LogDispatchCallback* callback = nullptr; - LogDispatchData data; - for (const std::pair& h - : ELPP->m_logDispatchCallbacks) { - callback = h.second.get(); - if (callback != nullptr && callback->enabled()) { - data.setLogMessage(m_logMessage); - data.setDispatchAction(m_dispatchAction); - callback->handle(&data); - } - } -} - -// MessageBuilder - -void MessageBuilder::initialize(Logger* logger) { - m_logger = logger; - m_containerLogSeperator = ELPP->hasFlag(LoggingFlag::NewLineForContainer) ? - ELPP_LITERAL("\n ") : ELPP_LITERAL(", "); -} - -MessageBuilder& MessageBuilder::operator<<(const wchar_t* msg) { - if (msg == nullptr) { - m_logger->stream() << base::consts::kNullPointer; - return *this; - } -# if defined(ELPP_UNICODE) - m_logger->stream() << msg; -# else - char* buff_ = base::utils::Str::wcharPtrToCharPtr(msg); - m_logger->stream() << buff_; - free(buff_); -# endif - if (ELPP->hasFlag(LoggingFlag::AutoSpacing)) { - m_logger->stream() << " "; - } - return *this; -} - -// Writer - -Writer& Writer::construct(Logger* logger, bool needLock) { - m_logger = logger; - initializeLogger(logger->id(), false, needLock); - m_messageBuilder.initialize(m_logger); - return *this; -} - -Writer& Writer::construct(int count, const char* loggerIds, ...) { - if (ELPP->hasFlag(LoggingFlag::MultiLoggerSupport)) { - va_list loggersList; - va_start(loggersList, loggerIds); - const char* id = loggerIds; - m_loggerIds.reserve(count); - for (int i = 0; i < count; ++i) { - m_loggerIds.push_back(std::string(id)); - id = va_arg(loggersList, const char*); - } - va_end(loggersList); - initializeLogger(m_loggerIds.at(0)); - } else { - initializeLogger(std::string(loggerIds)); - } - m_messageBuilder.initialize(m_logger); - return *this; -} - -void Writer::initializeLogger(const std::string& loggerId, bool lookup, bool needLock) { - if (lookup) { - m_logger = ELPP->registeredLoggers()->get(loggerId, ELPP->hasFlag(LoggingFlag::CreateLoggerAutomatically)); - } - if (m_logger == nullptr) { - { - if (!ELPP->registeredLoggers()->has(std::string(base::consts::kDefaultLoggerId))) { - // Somehow default logger has been unregistered. Not good! Register again - ELPP->registeredLoggers()->get(std::string(base::consts::kDefaultLoggerId)); - } - } - Writer(Level::Debug, m_file, m_line, m_func).construct(1, base::consts::kDefaultLoggerId) - << "Logger [" << loggerId << "] is not registered yet!"; - m_proceed = false; - } else { - if (needLock) { - m_logger->acquireLock(); // This should not be unlocked by checking m_proceed because - // m_proceed can be changed by lines below - } - if (ELPP->hasFlag(LoggingFlag::HierarchicalLogging)) { - m_proceed = m_level == Level::Verbose ? m_logger->enabled(m_level) : - LevelHelper::castToInt(m_level) >= LevelHelper::castToInt(ELPP->m_loggingLevel); - } else { - m_proceed = m_logger->enabled(m_level); - } - } -} - -void Writer::processDispatch() { -#if ELPP_LOGGING_ENABLED - if (ELPP->hasFlag(LoggingFlag::MultiLoggerSupport)) { - bool firstDispatched = false; - base::type::string_t logMessage; - std::size_t i = 0; - do { - if (m_proceed) { - if (firstDispatched) { - m_logger->stream() << logMessage; - } else { - firstDispatched = true; - if (m_loggerIds.size() > 1) { - logMessage = m_logger->stream().str(); - } - } - triggerDispatch(); - } else if (m_logger != nullptr) { - m_logger->stream().str(ELPP_LITERAL("")); - m_logger->releaseLock(); - } - if (i + 1 < m_loggerIds.size()) { - initializeLogger(m_loggerIds.at(i + 1)); - } - } while (++i < m_loggerIds.size()); - } else { - if (m_proceed) { - triggerDispatch(); - } else if (m_logger != nullptr) { - m_logger->stream().str(ELPP_LITERAL("")); - m_logger->releaseLock(); - } - } -#else - if (m_logger != nullptr) { - m_logger->stream().str(ELPP_LITERAL("")); - m_logger->releaseLock(); - } -#endif // ELPP_LOGGING_ENABLED -} - -void Writer::triggerDispatch(void) { - if (m_proceed) { - if (m_msg == nullptr) { - LogMessage msg(m_level, m_file, m_line, m_func, m_verboseLevel, - m_logger); - base::LogDispatcher(m_proceed, &msg, m_dispatchAction).dispatch(); - } else { - base::LogDispatcher(m_proceed, m_msg, m_dispatchAction).dispatch(); - } - } - if (m_logger != nullptr) { - m_logger->stream().str(ELPP_LITERAL("")); - m_logger->releaseLock(); - } - if (m_proceed && m_level == Level::Fatal - && !ELPP->hasFlag(LoggingFlag::DisableApplicationAbortOnFatalLog)) { - base::Writer(Level::Warning, m_file, m_line, m_func).construct(1, base::consts::kDefaultLoggerId) - << "Aborting application. Reason: Fatal log at [" << m_file << ":" << m_line << "]"; - std::stringstream reasonStream; - reasonStream << "Fatal log at [" << m_file << ":" << m_line << "]" - << " If you wish to disable 'abort on fatal log' please use " - << "el::Loggers::addFlag(el::LoggingFlag::DisableApplicationAbortOnFatalLog)"; - base::utils::abort(1, reasonStream.str()); - } - m_proceed = false; -} - -// PErrorWriter - -PErrorWriter::~PErrorWriter(void) { - if (m_proceed) { -#if ELPP_COMPILER_MSVC - char buff[256]; - strerror_s(buff, 256, errno); - m_logger->stream() << ": " << buff << " [" << errno << "]"; -#else - m_logger->stream() << ": " << strerror(errno) << " [" << errno << "]"; -#endif - } -} - -// PerformanceTracker - -#if defined(ELPP_FEATURE_ALL) || defined(ELPP_FEATURE_PERFORMANCE_TRACKING) - -PerformanceTracker::PerformanceTracker(const std::string& blockName, - base::TimestampUnit timestampUnit, - const std::string& loggerId, - bool scopedLog, Level level) : - m_blockName(blockName), m_timestampUnit(timestampUnit), m_loggerId(loggerId), m_scopedLog(scopedLog), - m_level(level), m_hasChecked(false), m_lastCheckpointId(std::string()), m_enabled(false) { -#if !defined(ELPP_DISABLE_PERFORMANCE_TRACKING) && ELPP_LOGGING_ENABLED - // We store it locally so that if user happen to change configuration by the end of scope - // or before calling checkpoint, we still depend on state of configuraton at time of construction - el::Logger* loggerPtr = ELPP->registeredLoggers()->get(loggerId, false); - m_enabled = loggerPtr != nullptr && loggerPtr->m_typedConfigurations->performanceTracking(m_level); - if (m_enabled) { - base::utils::DateTime::gettimeofday(&m_startTime); - } -#endif // !defined(ELPP_DISABLE_PERFORMANCE_TRACKING) && ELPP_LOGGING_ENABLED -} - -PerformanceTracker::~PerformanceTracker(void) { -#if !defined(ELPP_DISABLE_PERFORMANCE_TRACKING) && ELPP_LOGGING_ENABLED - if (m_enabled) { - base::threading::ScopedLock scopedLock(lock()); - if (m_scopedLog) { - base::utils::DateTime::gettimeofday(&m_endTime); - base::type::string_t formattedTime = getFormattedTimeTaken(); - PerformanceTrackingData data(PerformanceTrackingData::DataType::Complete); - data.init(this); - data.m_formattedTimeTaken = formattedTime; - PerformanceTrackingCallback* callback = nullptr; - for (const std::pair& h - : ELPP->m_performanceTrackingCallbacks) { - callback = h.second.get(); - if (callback != nullptr && callback->enabled()) { - callback->handle(&data); - } - } - } - } -#endif // !defined(ELPP_DISABLE_PERFORMANCE_TRACKING) -} - -void PerformanceTracker::checkpoint(const std::string& id, const char* file, base::type::LineNumber line, - const char* func) { -#if !defined(ELPP_DISABLE_PERFORMANCE_TRACKING) && ELPP_LOGGING_ENABLED - if (m_enabled) { - base::threading::ScopedLock scopedLock(lock()); - base::utils::DateTime::gettimeofday(&m_endTime); - base::type::string_t formattedTime = m_hasChecked ? getFormattedTimeTaken(m_lastCheckpointTime) : ELPP_LITERAL(""); - PerformanceTrackingData data(PerformanceTrackingData::DataType::Checkpoint); - data.init(this); - data.m_checkpointId = id; - data.m_file = file; - data.m_line = line; - data.m_func = func; - data.m_formattedTimeTaken = formattedTime; - PerformanceTrackingCallback* callback = nullptr; - for (const std::pair& h - : ELPP->m_performanceTrackingCallbacks) { - callback = h.second.get(); - if (callback != nullptr && callback->enabled()) { - callback->handle(&data); - } - } - base::utils::DateTime::gettimeofday(&m_lastCheckpointTime); - m_hasChecked = true; - m_lastCheckpointId = id; - } -#endif // !defined(ELPP_DISABLE_PERFORMANCE_TRACKING) && ELPP_LOGGING_ENABLED - ELPP_UNUSED(id); - ELPP_UNUSED(file); - ELPP_UNUSED(line); - ELPP_UNUSED(func); -} - -const base::type::string_t PerformanceTracker::getFormattedTimeTaken(struct timeval startTime) const { - if (ELPP->hasFlag(LoggingFlag::FixedTimeFormat)) { - base::type::stringstream_t ss; - ss << base::utils::DateTime::getTimeDifference(m_endTime, - startTime, m_timestampUnit) << " " << base::consts::kTimeFormats[static_cast - (m_timestampUnit)].unit; - return ss.str(); - } - return base::utils::DateTime::formatTime(base::utils::DateTime::getTimeDifference(m_endTime, - startTime, m_timestampUnit), m_timestampUnit); -} - -#endif // defined(ELPP_FEATURE_ALL) || defined(ELPP_FEATURE_PERFORMANCE_TRACKING) - -namespace debug { -#if defined(ELPP_FEATURE_ALL) || defined(ELPP_FEATURE_CRASH_LOG) - -// StackTrace - -StackTrace::StackTraceEntry::StackTraceEntry(std::size_t index, const std::string& loc, const std::string& demang, - const std::string& hex, - const std::string& addr) : - m_index(index), - m_location(loc), - m_demangled(demang), - m_hex(hex), - m_addr(addr) { -} - -std::ostream& operator<<(std::ostream& ss, const StackTrace::StackTraceEntry& si) { - ss << "[" << si.m_index << "] " << si.m_location << (si.m_hex.empty() ? "" : "+") << si.m_hex << " " << si.m_addr << - (si.m_demangled.empty() ? "" : ":") << si.m_demangled; - return ss; -} - -std::ostream& operator<<(std::ostream& os, const StackTrace& st) { - std::vector::const_iterator it = st.m_stack.begin(); - while (it != st.m_stack.end()) { - os << " " << *it++ << "\n"; - } - return os; -} - -void StackTrace::generateNew(void) { -#if ELPP_STACKTRACE - m_stack.clear(); - void* stack[kMaxStack]; - unsigned int size = backtrace(stack, kMaxStack); - char** strings = backtrace_symbols(stack, size); - if (size > kStackStart) { // Skip StackTrace c'tor and generateNew - for (std::size_t i = kStackStart; i < size; ++i) { - std::string mangName; - std::string location; - std::string hex; - std::string addr; - - // entry: 2 crash.cpp.bin 0x0000000101552be5 _ZN2el4base5debug10StackTraceC1Ev + 21 - const std::string line(strings[i]); - auto p = line.find("_"); - if (p != std::string::npos) { - mangName = line.substr(p); - mangName = mangName.substr(0, mangName.find(" +")); - } - p = line.find("0x"); - if (p != std::string::npos) { - addr = line.substr(p); - addr = addr.substr(0, addr.find("_")); - } - // Perform demangling if parsed properly - if (!mangName.empty()) { - int status = 0; - char* demangName = abi::__cxa_demangle(mangName.data(), 0, 0, &status); - // if demangling is successful, output the demangled function name - if (status == 0) { - // Success (see http://gcc.gnu.org/onlinedocs/libstdc++/libstdc++-html-USERS-4.3/a01696.html) - StackTraceEntry entry(i - 1, location, demangName, hex, addr); - m_stack.push_back(entry); - } else { - // Not successful - we will use mangled name - StackTraceEntry entry(i - 1, location, mangName, hex, addr); - m_stack.push_back(entry); - } - free(demangName); - } else { - StackTraceEntry entry(i - 1, line); - m_stack.push_back(entry); - } - } - } - free(strings); -#else - ELPP_INTERNAL_INFO(1, "Stacktrace generation not supported for selected compiler"); -#endif // ELPP_STACKTRACE -} - -// Static helper functions - -static std::string crashReason(int sig) { - std::stringstream ss; - bool foundReason = false; - for (int i = 0; i < base::consts::kCrashSignalsCount; ++i) { - if (base::consts::kCrashSignals[i].numb == sig) { - ss << "Application has crashed due to [" << base::consts::kCrashSignals[i].name << "] signal"; - if (ELPP->hasFlag(el::LoggingFlag::LogDetailedCrashReason)) { - ss << std::endl << - " " << base::consts::kCrashSignals[i].brief << std::endl << - " " << base::consts::kCrashSignals[i].detail; - } - foundReason = true; - } - } - if (!foundReason) { - ss << "Application has crashed due to unknown signal [" << sig << "]"; - } - return ss.str(); -} -/// @brief Logs reason of crash from sig -static void logCrashReason(int sig, bool stackTraceIfAvailable, Level level, const char* logger) { - if (sig == SIGINT && ELPP->hasFlag(el::LoggingFlag::IgnoreSigInt)) { - return; - } - std::stringstream ss; - ss << "CRASH HANDLED; "; - ss << crashReason(sig); -#if ELPP_STACKTRACE - if (stackTraceIfAvailable) { - ss << std::endl << " ======= Backtrace: =========" << std::endl << base::debug::StackTrace(); - } -#else - ELPP_UNUSED(stackTraceIfAvailable); -#endif // ELPP_STACKTRACE - ELPP_WRITE_LOG(el::base::Writer, level, base::DispatchAction::NormalLog, logger) << ss.str(); -} - -static inline void crashAbort(int sig) { - base::utils::abort(sig, std::string()); -} - -/// @brief Default application crash handler -/// -/// @detail This function writes log using 'default' logger, prints stack trace for GCC based compilers and aborts program. -static inline void defaultCrashHandler(int sig) { - base::debug::logCrashReason(sig, true, Level::Fatal, base::consts::kDefaultLoggerId); - base::debug::crashAbort(sig); -} - -// CrashHandler - -CrashHandler::CrashHandler(bool useDefault) { - if (useDefault) { - setHandler(defaultCrashHandler); - } -} - -void CrashHandler::setHandler(const Handler& cHandler) { - m_handler = cHandler; -#if defined(ELPP_HANDLE_SIGABRT) - int i = 0; // SIGABRT is at base::consts::kCrashSignals[0] -#else - int i = 1; -#endif // defined(ELPP_HANDLE_SIGABRT) - for (; i < base::consts::kCrashSignalsCount; ++i) { - m_handler = signal(base::consts::kCrashSignals[i].numb, cHandler); - } -} - -#endif // defined(ELPP_FEATURE_ALL) || defined(ELPP_FEATURE_CRASH_LOG) -} // namespace debug -} // namespace base - -// el - -// Helpers - -#if defined(ELPP_FEATURE_ALL) || defined(ELPP_FEATURE_CRASH_LOG) - -void Helpers::crashAbort(int sig, const char* sourceFile, unsigned int long line) { - std::stringstream ss; - ss << base::debug::crashReason(sig).c_str(); - ss << " - [Called el::Helpers::crashAbort(" << sig << ")]"; - if (sourceFile != nullptr && strlen(sourceFile) > 0) { - ss << " - Source: " << sourceFile; - if (line > 0) - ss << ":" << line; - else - ss << " (line number not specified)"; - } - base::utils::abort(sig, ss.str()); -} - -void Helpers::logCrashReason(int sig, bool stackTraceIfAvailable, Level level, const char* logger) { - el::base::debug::logCrashReason(sig, stackTraceIfAvailable, level, logger); -} - -#endif // defined(ELPP_FEATURE_ALL) || defined(ELPP_FEATURE_CRASH_LOG) - -// Loggers - -Logger* Loggers::getLogger(const std::string& identity, bool registerIfNotAvailable) { - return ELPP->registeredLoggers()->get(identity, registerIfNotAvailable); -} - -void Loggers::setDefaultLogBuilder(el::LogBuilderPtr& logBuilderPtr) { - ELPP->registeredLoggers()->setDefaultLogBuilder(logBuilderPtr); -} - -bool Loggers::unregisterLogger(const std::string& identity) { - return ELPP->registeredLoggers()->remove(identity); -} - -bool Loggers::hasLogger(const std::string& identity) { - return ELPP->registeredLoggers()->has(identity); -} - -Logger* Loggers::reconfigureLogger(Logger* logger, const Configurations& configurations) { - if (!logger) return nullptr; - logger->configure(configurations); - return logger; -} - -Logger* Loggers::reconfigureLogger(const std::string& identity, const Configurations& configurations) { - return Loggers::reconfigureLogger(Loggers::getLogger(identity), configurations); -} - -Logger* Loggers::reconfigureLogger(const std::string& identity, ConfigurationType configurationType, - const std::string& value) { - Logger* logger = Loggers::getLogger(identity); - if (logger == nullptr) { - return nullptr; - } - logger->configurations()->set(Level::Global, configurationType, value); - logger->reconfigure(); - return logger; -} - -void Loggers::reconfigureAllLoggers(const Configurations& configurations) { - for (base::RegisteredLoggers::iterator it = ELPP->registeredLoggers()->begin(); - it != ELPP->registeredLoggers()->end(); ++it) { - Loggers::reconfigureLogger(it->second, configurations); - } -} - -void Loggers::reconfigureAllLoggers(Level level, ConfigurationType configurationType, - const std::string& value) { - for (base::RegisteredLoggers::iterator it = ELPP->registeredLoggers()->begin(); - it != ELPP->registeredLoggers()->end(); ++it) { - Logger* logger = it->second; - logger->configurations()->set(level, configurationType, value); - logger->reconfigure(); - } -} - -void Loggers::setDefaultConfigurations(const Configurations& configurations, bool reconfigureExistingLoggers) { - ELPP->registeredLoggers()->setDefaultConfigurations(configurations); - if (reconfigureExistingLoggers) { - Loggers::reconfigureAllLoggers(configurations); - } -} - -const Configurations* Loggers::defaultConfigurations(void) { - return ELPP->registeredLoggers()->defaultConfigurations(); -} - -const base::LogStreamsReferenceMap* Loggers::logStreamsReference(void) { - return ELPP->registeredLoggers()->logStreamsReference(); -} - -base::TypedConfigurations Loggers::defaultTypedConfigurations(void) { - return base::TypedConfigurations( - ELPP->registeredLoggers()->defaultConfigurations(), - ELPP->registeredLoggers()->logStreamsReference()); -} - -std::vector* Loggers::populateAllLoggerIds(std::vector* targetList) { - targetList->clear(); - for (base::RegisteredLoggers::iterator it = ELPP->registeredLoggers()->list().begin(); - it != ELPP->registeredLoggers()->list().end(); ++it) { - targetList->push_back(it->first); - } - return targetList; -} - -void Loggers::configureFromGlobal(const char* globalConfigurationFilePath) { - std::ifstream gcfStream(globalConfigurationFilePath, std::ifstream::in); - ELPP_ASSERT(gcfStream.is_open(), "Unable to open global configuration file [" << globalConfigurationFilePath - << "] for parsing."); - std::string line = std::string(); - std::stringstream ss; - Logger* logger = nullptr; - auto configure = [&](void) { - ELPP_INTERNAL_INFO(8, "Configuring logger: '" << logger->id() << "' with configurations \n" << ss.str() - << "\n--------------"); - Configurations c; - c.parseFromText(ss.str()); - logger->configure(c); - }; - while (gcfStream.good()) { - std::getline(gcfStream, line); - ELPP_INTERNAL_INFO(1, "Parsing line: " << line); - base::utils::Str::trim(line); - if (Configurations::Parser::isComment(line)) continue; - Configurations::Parser::ignoreComments(&line); - base::utils::Str::trim(line); - if (line.size() > 2 && base::utils::Str::startsWith(line, std::string(base::consts::kConfigurationLoggerId))) { - if (!ss.str().empty() && logger != nullptr) { - configure(); - } - ss.str(std::string("")); - line = line.substr(2); - base::utils::Str::trim(line); - if (line.size() > 1) { - ELPP_INTERNAL_INFO(1, "Getting logger: '" << line << "'"); - logger = getLogger(line); - } - } else { - ss << line << "\n"; - } - } - if (!ss.str().empty() && logger != nullptr) { - configure(); - } -} - -bool Loggers::configureFromArg(const char* argKey) { -#if defined(ELPP_DISABLE_CONFIGURATION_FROM_PROGRAM_ARGS) - ELPP_UNUSED(argKey); -#else - if (!Helpers::commandLineArgs()->hasParamWithValue(argKey)) { - return false; - } - configureFromGlobal(Helpers::commandLineArgs()->getParamValue(argKey)); -#endif // defined(ELPP_DISABLE_CONFIGURATION_FROM_PROGRAM_ARGS) - return true; -} - -void Loggers::flushAll(void) { - ELPP->registeredLoggers()->flushAll(); -} - -void Loggers::setVerboseLevel(base::type::VerboseLevel level) { - ELPP->vRegistry()->setLevel(level); -} - -base::type::VerboseLevel Loggers::verboseLevel(void) { - return ELPP->vRegistry()->level(); -} - -void Loggers::setVModules(const char* modules) { - if (ELPP->vRegistry()->vModulesEnabled()) { - ELPP->vRegistry()->setModules(modules); - } -} - -void Loggers::clearVModules(void) { - ELPP->vRegistry()->clearModules(); -} - -// VersionInfo - -const std::string VersionInfo::version(void) { - return std::string("9.96.7"); -} -/// @brief Release date of current version -const std::string VersionInfo::releaseDate(void) { - return std::string("24-11-2018 0728hrs"); -} - -} // namespace el diff --git a/core/src/external/easyloggingpp/easylogging++.h b/core/src/external/easyloggingpp/easylogging++.h deleted file mode 100644 index 62a7c5a423..0000000000 --- a/core/src/external/easyloggingpp/easylogging++.h +++ /dev/null @@ -1,4569 +0,0 @@ -// -// Bismillah ar-Rahmaan ar-Raheem -// -// Easylogging++ v9.96.7 -// Single-header only, cross-platform logging library for C++ applications -// -// Copyright (c) 2012-2018 Zuhd Web Services -// Copyright (c) 2012-2018 @abumusamq -// -// This library is released under the MIT Licence. -// https://github.com/zuhd-org/easyloggingpp/blob/master/LICENSE -// -// https://zuhd.org -// http://muflihun.com -// - -#ifndef EASYLOGGINGPP_H -#define EASYLOGGINGPP_H -// Compilers and C++0x/C++11 Evaluation -#if __cplusplus >= 201103L -# define ELPP_CXX11 1 -#endif // __cplusplus >= 201103L -#if (defined(__GNUC__)) -# define ELPP_COMPILER_GCC 1 -#else -# define ELPP_COMPILER_GCC 0 -#endif -#if ELPP_COMPILER_GCC -# define ELPP_GCC_VERSION (__GNUC__ * 10000 \ -+ __GNUC_MINOR__ * 100 \ -+ __GNUC_PATCHLEVEL__) -# if defined(__GXX_EXPERIMENTAL_CXX0X__) -# define ELPP_CXX0X 1 -# endif -#endif -// Visual C++ -#if defined(_MSC_VER) -# define ELPP_COMPILER_MSVC 1 -#else -# define ELPP_COMPILER_MSVC 0 -#endif -#define ELPP_CRT_DBG_WARNINGS ELPP_COMPILER_MSVC -#if ELPP_COMPILER_MSVC -# if (_MSC_VER == 1600) -# define ELPP_CXX0X 1 -# elif(_MSC_VER >= 1700) -# define ELPP_CXX11 1 -# endif -#endif -// Clang++ -#if (defined(__clang__) && (__clang__ == 1)) -# define ELPP_COMPILER_CLANG 1 -#else -# define ELPP_COMPILER_CLANG 0 -#endif -#if ELPP_COMPILER_CLANG -# if __has_include() -# include // Make __GLIBCXX__ defined when using libstdc++ -# if !defined(__GLIBCXX__) || __GLIBCXX__ >= 20150426 -# define ELPP_CLANG_SUPPORTS_THREAD -# endif // !defined(__GLIBCXX__) || __GLIBCXX__ >= 20150426 -# endif // __has_include() -#endif -#if (defined(__MINGW32__) || defined(__MINGW64__)) -# define ELPP_MINGW 1 -#else -# define ELPP_MINGW 0 -#endif -#if (defined(__CYGWIN__) && (__CYGWIN__ == 1)) -# define ELPP_CYGWIN 1 -#else -# define ELPP_CYGWIN 0 -#endif -#if (defined(__INTEL_COMPILER)) -# define ELPP_COMPILER_INTEL 1 -#else -# define ELPP_COMPILER_INTEL 0 -#endif -// Operating System Evaluation -// Windows -#if (defined(_WIN32) || defined(_WIN64)) -# define ELPP_OS_WINDOWS 1 -#else -# define ELPP_OS_WINDOWS 0 -#endif -// Linux -#if (defined(__linux) || defined(__linux__)) -# define ELPP_OS_LINUX 1 -#else -# define ELPP_OS_LINUX 0 -#endif -#if (defined(__APPLE__)) -# define ELPP_OS_MAC 1 -#else -# define ELPP_OS_MAC 0 -#endif -#if (defined(__FreeBSD__) || defined(__FreeBSD_kernel__)) -# define ELPP_OS_FREEBSD 1 -#else -# define ELPP_OS_FREEBSD 0 -#endif -#if (defined(__sun)) -# define ELPP_OS_SOLARIS 1 -#else -# define ELPP_OS_SOLARIS 0 -#endif -#if (defined(_AIX)) -# define ELPP_OS_AIX 1 -#else -# define ELPP_OS_AIX 0 -#endif -#if (defined(__NetBSD__)) -# define ELPP_OS_NETBSD 1 -#else -# define ELPP_OS_NETBSD 0 -#endif -#if defined(__EMSCRIPTEN__) -# define ELPP_OS_EMSCRIPTEN 1 -#else -# define ELPP_OS_EMSCRIPTEN 0 -#endif -// Unix -#if ((ELPP_OS_LINUX || ELPP_OS_MAC || ELPP_OS_FREEBSD || ELPP_OS_NETBSD || ELPP_OS_SOLARIS || ELPP_OS_AIX || ELPP_OS_EMSCRIPTEN) && (!ELPP_OS_WINDOWS)) -# define ELPP_OS_UNIX 1 -#else -# define ELPP_OS_UNIX 0 -#endif -#if (defined(__ANDROID__)) -# define ELPP_OS_ANDROID 1 -#else -# define ELPP_OS_ANDROID 0 -#endif -// Evaluating Cygwin as *nix OS -#if !ELPP_OS_UNIX && !ELPP_OS_WINDOWS && ELPP_CYGWIN -# undef ELPP_OS_UNIX -# undef ELPP_OS_LINUX -# define ELPP_OS_UNIX 1 -# define ELPP_OS_LINUX 1 -#endif // !ELPP_OS_UNIX && !ELPP_OS_WINDOWS && ELPP_CYGWIN -#if !defined(ELPP_INTERNAL_DEBUGGING_OUT_INFO) -# define ELPP_INTERNAL_DEBUGGING_OUT_INFO std::cout -#endif // !defined(ELPP_INTERNAL_DEBUGGING_OUT) -#if !defined(ELPP_INTERNAL_DEBUGGING_OUT_ERROR) -# define ELPP_INTERNAL_DEBUGGING_OUT_ERROR std::cerr -#endif // !defined(ELPP_INTERNAL_DEBUGGING_OUT) -#if !defined(ELPP_INTERNAL_DEBUGGING_ENDL) -# define ELPP_INTERNAL_DEBUGGING_ENDL std::endl -#endif // !defined(ELPP_INTERNAL_DEBUGGING_OUT) -#if !defined(ELPP_INTERNAL_DEBUGGING_MSG) -# define ELPP_INTERNAL_DEBUGGING_MSG(msg) msg -#endif // !defined(ELPP_INTERNAL_DEBUGGING_OUT) -// Internal Assertions and errors -#if !defined(ELPP_DISABLE_ASSERT) -# if (defined(ELPP_DEBUG_ASSERT_FAILURE)) -# define ELPP_ASSERT(expr, msg) if (!(expr)) { \ -std::stringstream internalInfoStream; internalInfoStream << msg; \ -ELPP_INTERNAL_DEBUGGING_OUT_ERROR \ -<< "EASYLOGGING++ ASSERTION FAILED (LINE: " << __LINE__ << ") [" #expr << "] WITH MESSAGE \"" \ -<< ELPP_INTERNAL_DEBUGGING_MSG(internalInfoStream.str()) << "\"" << ELPP_INTERNAL_DEBUGGING_ENDL; base::utils::abort(1, \ -"ELPP Assertion failure, please define ELPP_DEBUG_ASSERT_FAILURE"); } -# else -# define ELPP_ASSERT(expr, msg) if (!(expr)) { \ -std::stringstream internalInfoStream; internalInfoStream << msg; \ -ELPP_INTERNAL_DEBUGGING_OUT_ERROR\ -<< "ASSERTION FAILURE FROM EASYLOGGING++ (LINE: " \ -<< __LINE__ << ") [" #expr << "] WITH MESSAGE \"" << ELPP_INTERNAL_DEBUGGING_MSG(internalInfoStream.str()) << "\"" \ -<< ELPP_INTERNAL_DEBUGGING_ENDL; } -# endif // (defined(ELPP_DEBUG_ASSERT_FAILURE)) -#else -# define ELPP_ASSERT(x, y) -#endif //(!defined(ELPP_DISABLE_ASSERT) -#if ELPP_COMPILER_MSVC -# define ELPP_INTERNAL_DEBUGGING_WRITE_PERROR \ -{ char buff[256]; strerror_s(buff, 256, errno); \ -ELPP_INTERNAL_DEBUGGING_OUT_ERROR << ": " << buff << " [" << errno << "]";} (void)0 -#else -# define ELPP_INTERNAL_DEBUGGING_WRITE_PERROR \ -ELPP_INTERNAL_DEBUGGING_OUT_ERROR << ": " << strerror(errno) << " [" << errno << "]"; (void)0 -#endif // ELPP_COMPILER_MSVC -#if defined(ELPP_DEBUG_ERRORS) -# if !defined(ELPP_INTERNAL_ERROR) -# define ELPP_INTERNAL_ERROR(msg, pe) { \ -std::stringstream internalInfoStream; internalInfoStream << " " << msg; \ -ELPP_INTERNAL_DEBUGGING_OUT_ERROR \ -<< "ERROR FROM EASYLOGGING++ (LINE: " << __LINE__ << ") " \ -<< ELPP_INTERNAL_DEBUGGING_MSG(internalInfoStream.str()) << ELPP_INTERNAL_DEBUGGING_ENDL; \ -if (pe) { ELPP_INTERNAL_DEBUGGING_OUT_ERROR << " "; ELPP_INTERNAL_DEBUGGING_WRITE_PERROR; }} (void)0 -# endif -#else -# undef ELPP_INTERNAL_INFO -# define ELPP_INTERNAL_ERROR(msg, pe) -#endif // defined(ELPP_DEBUG_ERRORS) -#if (defined(ELPP_DEBUG_INFO)) -# if !(defined(ELPP_INTERNAL_INFO_LEVEL)) -# define ELPP_INTERNAL_INFO_LEVEL 9 -# endif // !(defined(ELPP_INTERNAL_INFO_LEVEL)) -# if !defined(ELPP_INTERNAL_INFO) -# define ELPP_INTERNAL_INFO(lvl, msg) { if (lvl <= ELPP_INTERNAL_INFO_LEVEL) { \ -std::stringstream internalInfoStream; internalInfoStream << " " << msg; \ -ELPP_INTERNAL_DEBUGGING_OUT_INFO << ELPP_INTERNAL_DEBUGGING_MSG(internalInfoStream.str()) \ -<< ELPP_INTERNAL_DEBUGGING_ENDL; }} -# endif -#else -# undef ELPP_INTERNAL_INFO -# define ELPP_INTERNAL_INFO(lvl, msg) -#endif // (defined(ELPP_DEBUG_INFO)) -#if (defined(ELPP_FEATURE_ALL)) || (defined(ELPP_FEATURE_CRASH_LOG)) -# if (ELPP_COMPILER_GCC && !ELPP_MINGW && !ELPP_OS_ANDROID && !ELPP_OS_EMSCRIPTEN) -# define ELPP_STACKTRACE 1 -# else -# if ELPP_COMPILER_MSVC -# pragma message("Stack trace not available for this compiler") -# else -# warning "Stack trace not available for this compiler"; -# endif // ELPP_COMPILER_MSVC -# define ELPP_STACKTRACE 0 -# endif // ELPP_COMPILER_GCC -#else -# define ELPP_STACKTRACE 0 -#endif // (defined(ELPP_FEATURE_ALL)) || (defined(ELPP_FEATURE_CRASH_LOG)) -// Miscellaneous macros -#define ELPP_UNUSED(x) (void)x -#if ELPP_OS_UNIX -// Log file permissions for unix-based systems -# define ELPP_LOG_PERMS S_IRUSR | S_IWUSR | S_IXUSR | S_IWGRP | S_IRGRP | S_IXGRP | S_IWOTH | S_IXOTH -#endif // ELPP_OS_UNIX -#if defined(ELPP_AS_DLL) && ELPP_COMPILER_MSVC -# if defined(ELPP_EXPORT_SYMBOLS) -# define ELPP_EXPORT __declspec(dllexport) -# else -# define ELPP_EXPORT __declspec(dllimport) -# endif // defined(ELPP_EXPORT_SYMBOLS) -#else -# define ELPP_EXPORT -#endif // defined(ELPP_AS_DLL) && ELPP_COMPILER_MSVC -// Some special functions that are VC++ specific -#undef STRTOK -#undef STRERROR -#undef STRCAT -#undef STRCPY -#if ELPP_CRT_DBG_WARNINGS -# define STRTOK(a, b, c) strtok_s(a, b, c) -# define STRERROR(a, b, c) strerror_s(a, b, c) -# define STRCAT(a, b, len) strcat_s(a, len, b) -# define STRCPY(a, b, len) strcpy_s(a, len, b) -#else -# define STRTOK(a, b, c) strtok(a, b) -# define STRERROR(a, b, c) strerror(c) -# define STRCAT(a, b, len) strcat(a, b) -# define STRCPY(a, b, len) strcpy(a, b) -#endif -// Compiler specific support evaluations -#if (ELPP_MINGW && !defined(ELPP_FORCE_USE_STD_THREAD)) -# define ELPP_USE_STD_THREADING 0 -#else -# if ((ELPP_COMPILER_CLANG && defined(ELPP_CLANG_SUPPORTS_THREAD)) || \ - (!ELPP_COMPILER_CLANG && defined(ELPP_CXX11)) || \ - defined(ELPP_FORCE_USE_STD_THREAD)) -# define ELPP_USE_STD_THREADING 1 -# else -# define ELPP_USE_STD_THREADING 0 -# endif -#endif -#undef ELPP_FINAL -#if ELPP_COMPILER_INTEL || (ELPP_GCC_VERSION < 40702) -# define ELPP_FINAL -#else -# define ELPP_FINAL final -#endif // ELPP_COMPILER_INTEL || (ELPP_GCC_VERSION < 40702) -#if defined(ELPP_EXPERIMENTAL_ASYNC) -# define ELPP_ASYNC_LOGGING 1 -#else -# define ELPP_ASYNC_LOGGING 0 -#endif // defined(ELPP_EXPERIMENTAL_ASYNC) -#if defined(ELPP_THREAD_SAFE) || ELPP_ASYNC_LOGGING -# define ELPP_THREADING_ENABLED 1 -#else -# define ELPP_THREADING_ENABLED 0 -#endif // defined(ELPP_THREAD_SAFE) || ELPP_ASYNC_LOGGING -// Function macro ELPP_FUNC -#undef ELPP_FUNC -#if ELPP_COMPILER_MSVC // Visual C++ -# define ELPP_FUNC __FUNCSIG__ -#elif ELPP_COMPILER_GCC // GCC -# define ELPP_FUNC __PRETTY_FUNCTION__ -#elif ELPP_COMPILER_INTEL // Intel C++ -# define ELPP_FUNC __PRETTY_FUNCTION__ -#elif ELPP_COMPILER_CLANG // Clang++ -# define ELPP_FUNC __PRETTY_FUNCTION__ -#else -# if defined(__func__) -# define ELPP_FUNC __func__ -# else -# define ELPP_FUNC "" -# endif // defined(__func__) -#endif // defined(_MSC_VER) -#undef ELPP_VARIADIC_TEMPLATES_SUPPORTED -// Keep following line commented until features are fixed -#define ELPP_VARIADIC_TEMPLATES_SUPPORTED \ -(ELPP_COMPILER_GCC || ELPP_COMPILER_CLANG || ELPP_COMPILER_INTEL || (ELPP_COMPILER_MSVC && _MSC_VER >= 1800)) -// Logging Enable/Disable macros -#if defined(ELPP_DISABLE_LOGS) -#define ELPP_LOGGING_ENABLED 0 -#else -#define ELPP_LOGGING_ENABLED 1 -#endif -#if (!defined(ELPP_DISABLE_DEBUG_LOGS) && (ELPP_LOGGING_ENABLED)) -# define ELPP_DEBUG_LOG 1 -#else -# define ELPP_DEBUG_LOG 0 -#endif // (!defined(ELPP_DISABLE_DEBUG_LOGS) && (ELPP_LOGGING_ENABLED)) -#if (!defined(ELPP_DISABLE_INFO_LOGS) && (ELPP_LOGGING_ENABLED)) -# define ELPP_INFO_LOG 1 -#else -# define ELPP_INFO_LOG 0 -#endif // (!defined(ELPP_DISABLE_INFO_LOGS) && (ELPP_LOGGING_ENABLED)) -#if (!defined(ELPP_DISABLE_WARNING_LOGS) && (ELPP_LOGGING_ENABLED)) -# define ELPP_WARNING_LOG 1 -#else -# define ELPP_WARNING_LOG 0 -#endif // (!defined(ELPP_DISABLE_WARNING_LOGS) && (ELPP_LOGGING_ENABLED)) -#if (!defined(ELPP_DISABLE_ERROR_LOGS) && (ELPP_LOGGING_ENABLED)) -# define ELPP_ERROR_LOG 1 -#else -# define ELPP_ERROR_LOG 0 -#endif // (!defined(ELPP_DISABLE_ERROR_LOGS) && (ELPP_LOGGING_ENABLED)) -#if (!defined(ELPP_DISABLE_FATAL_LOGS) && (ELPP_LOGGING_ENABLED)) -# define ELPP_FATAL_LOG 1 -#else -# define ELPP_FATAL_LOG 0 -#endif // (!defined(ELPP_DISABLE_FATAL_LOGS) && (ELPP_LOGGING_ENABLED)) -#if (!defined(ELPP_DISABLE_TRACE_LOGS) && (ELPP_LOGGING_ENABLED)) -# define ELPP_TRACE_LOG 1 -#else -# define ELPP_TRACE_LOG 0 -#endif // (!defined(ELPP_DISABLE_TRACE_LOGS) && (ELPP_LOGGING_ENABLED)) -#if (!defined(ELPP_DISABLE_VERBOSE_LOGS) && (ELPP_LOGGING_ENABLED)) -# define ELPP_VERBOSE_LOG 1 -#else -# define ELPP_VERBOSE_LOG 0 -#endif // (!defined(ELPP_DISABLE_VERBOSE_LOGS) && (ELPP_LOGGING_ENABLED)) -#if (!(ELPP_CXX0X || ELPP_CXX11)) -# error "C++0x (or higher) support not detected! (Is `-std=c++11' missing?)" -#endif // (!(ELPP_CXX0X || ELPP_CXX11)) -// Headers -#if defined(ELPP_SYSLOG) -# include -#endif // defined(ELPP_SYSLOG) -#include -#include -#include -#include -#include -#include -#include -#include -#if defined(ELPP_UNICODE) -# include -# if ELPP_OS_WINDOWS -# include -# endif // ELPP_OS_WINDOWS -#endif // defined(ELPP_UNICODE) -#if ELPP_STACKTRACE -# include -# include -#endif // ELPP_STACKTRACE -#if ELPP_OS_ANDROID -# include -#endif // ELPP_OS_ANDROID -#if ELPP_OS_UNIX -# include -# include -#elif ELPP_OS_WINDOWS -# include -# include -# if defined(WIN32_LEAN_AND_MEAN) -# if defined(ELPP_WINSOCK2) -# include -# else -# include -# endif // defined(ELPP_WINSOCK2) -# endif // defined(WIN32_LEAN_AND_MEAN) -#endif // ELPP_OS_UNIX -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#if ELPP_THREADING_ENABLED -# if ELPP_USE_STD_THREADING -# include -# include -# else -# if ELPP_OS_UNIX -# include -# endif // ELPP_OS_UNIX -# endif // ELPP_USE_STD_THREADING -#endif // ELPP_THREADING_ENABLED -#if ELPP_ASYNC_LOGGING -# if defined(ELPP_NO_SLEEP_FOR) -# include -# endif // defined(ELPP_NO_SLEEP_FOR) -# include -# include -# include -#endif // ELPP_ASYNC_LOGGING -#if defined(ELPP_STL_LOGGING) -// For logging STL based templates -# include -# include -# include -# include -# include -# include -# if defined(ELPP_LOG_STD_ARRAY) -# include -# endif // defined(ELPP_LOG_STD_ARRAY) -# if defined(ELPP_LOG_UNORDERED_SET) -# include -# endif // defined(ELPP_UNORDERED_SET) -#endif // defined(ELPP_STL_LOGGING) -#if defined(ELPP_QT_LOGGING) -// For logging Qt based classes & templates -# include -# include -# include -# include -# include -# include -# include -# include -# include -# include -# include -# include -#endif // defined(ELPP_QT_LOGGING) -#if defined(ELPP_BOOST_LOGGING) -// For logging boost based classes & templates -# include -# include -# include -# include -# include -# include -# include -# include -#endif // defined(ELPP_BOOST_LOGGING) -#if defined(ELPP_WXWIDGETS_LOGGING) -// For logging wxWidgets based classes & templates -# include -#endif // defined(ELPP_WXWIDGETS_LOGGING) -#if defined(ELPP_UTC_DATETIME) -# define elpptime_r gmtime_r -# define elpptime_s gmtime_s -# define elpptime gmtime -#else -# define elpptime_r localtime_r -# define elpptime_s localtime_s -# define elpptime localtime -#endif // defined(ELPP_UTC_DATETIME) -// Forward declarations -namespace el { -class Logger; -class LogMessage; -class PerformanceTrackingData; -class Loggers; -class Helpers; -template class Callback; -class LogDispatchCallback; -class PerformanceTrackingCallback; -class LoggerRegistrationCallback; -class LogDispatchData; -namespace base { -class Storage; -class RegisteredLoggers; -class PerformanceTracker; -class MessageBuilder; -class Writer; -class PErrorWriter; -class LogDispatcher; -class DefaultLogBuilder; -class DefaultLogDispatchCallback; -#if ELPP_ASYNC_LOGGING -class AsyncLogDispatchCallback; -class AsyncDispatchWorker; -#endif // ELPP_ASYNC_LOGGING -class DefaultPerformanceTrackingCallback; -} // namespace base -} // namespace el -/// @brief Easylogging++ entry namespace -namespace el { -/// @brief Namespace containing base/internal functionality used by Easylogging++ -namespace base { -/// @brief Data types used by Easylogging++ -namespace type { -#undef ELPP_LITERAL -#undef ELPP_STRLEN -#undef ELPP_COUT -#if defined(ELPP_UNICODE) -# define ELPP_LITERAL(txt) L##txt -# define ELPP_STRLEN wcslen -# if defined ELPP_CUSTOM_COUT -# define ELPP_COUT ELPP_CUSTOM_COUT -# else -# define ELPP_COUT std::wcout -# endif // defined ELPP_CUSTOM_COUT -typedef wchar_t char_t; -typedef std::wstring string_t; -typedef std::wstringstream stringstream_t; -typedef std::wfstream fstream_t; -typedef std::wostream ostream_t; -#else -# define ELPP_LITERAL(txt) txt -# define ELPP_STRLEN strlen -# if defined ELPP_CUSTOM_COUT -# define ELPP_COUT ELPP_CUSTOM_COUT -# else -# define ELPP_COUT std::cout -# endif // defined ELPP_CUSTOM_COUT -typedef char char_t; -typedef std::string string_t; -typedef std::stringstream stringstream_t; -typedef std::fstream fstream_t; -typedef std::ostream ostream_t; -#endif // defined(ELPP_UNICODE) -#if defined(ELPP_CUSTOM_COUT_LINE) -# define ELPP_COUT_LINE(logLine) ELPP_CUSTOM_COUT_LINE(logLine) -#else -# define ELPP_COUT_LINE(logLine) logLine << std::flush -#endif // defined(ELPP_CUSTOM_COUT_LINE) -typedef unsigned int EnumType; -typedef unsigned short VerboseLevel; -typedef unsigned long int LineNumber; -typedef std::shared_ptr StoragePointer; -typedef std::shared_ptr LogDispatchCallbackPtr; -typedef std::shared_ptr PerformanceTrackingCallbackPtr; -typedef std::shared_ptr LoggerRegistrationCallbackPtr; -typedef std::unique_ptr PerformanceTrackerPtr; -} // namespace type -/// @brief Internal helper class that prevent copy constructor for class -/// -/// @detail When using this class simply inherit it privately -class NoCopy { - protected: - NoCopy(void) {} - private: - NoCopy(const NoCopy&); - NoCopy& operator=(const NoCopy&); -}; -/// @brief Internal helper class that makes all default constructors private. -/// -/// @detail This prevents initializing class making it static unless an explicit constructor is declared. -/// When using this class simply inherit it privately -class StaticClass { - private: - StaticClass(void); - StaticClass(const StaticClass&); - StaticClass& operator=(const StaticClass&); -}; -} // namespace base -/// @brief Represents enumeration for severity level used to determine level of logging -/// -/// @detail With Easylogging++, developers may disable or enable any level regardless of -/// what the severity is. Or they can choose to log using hierarchical logging flag -enum class Level : base::type::EnumType { - /// @brief Generic level that represents all the levels. Useful when setting global configuration for all levels - Global = 1, - /// @brief Information that can be useful to back-trace certain events - mostly useful than debug logs. - Trace = 2, - /// @brief Informational events most useful for developers to debug application - Debug = 4, - /// @brief Severe error information that will presumably abort application - Fatal = 8, - /// @brief Information representing errors in application but application will keep running - Error = 16, - /// @brief Useful when application has potentially harmful situtaions - Warning = 32, - /// @brief Information that can be highly useful and vary with verbose logging level. - Verbose = 64, - /// @brief Mainly useful to represent current progress of application - Info = 128, - /// @brief Represents unknown level - Unknown = 1010 -}; -} // namespace el -namespace std { -template<> struct hash { - public: - std::size_t operator()(const el::Level& l) const { - return hash {}(static_cast(l)); - } -}; -} -namespace el { -/// @brief Static class that contains helper functions for el::Level -class LevelHelper : base::StaticClass { - public: - /// @brief Represents minimum valid level. Useful when iterating through enum. - static const base::type::EnumType kMinValid = static_cast(Level::Trace); - /// @brief Represents maximum valid level. This is used internally and you should not need it. - static const base::type::EnumType kMaxValid = static_cast(Level::Info); - /// @brief Casts level to int, useful for iterating through enum. - static base::type::EnumType castToInt(Level level) { - return static_cast(level); - } - /// @brief Casts int(ushort) to level, useful for iterating through enum. - static Level castFromInt(base::type::EnumType l) { - return static_cast(l); - } - /// @brief Converts level to associated const char* - /// @return Upper case string based level. - static const char* convertToString(Level level); - /// @brief Converts from levelStr to Level - /// @param levelStr Upper case string based level. - /// Lower case is also valid but providing upper case is recommended. - static Level convertFromString(const char* levelStr); - /// @brief Applies specified function to each level starting from startIndex - /// @param startIndex initial value to start the iteration from. This is passed as pointer and - /// is left-shifted so this can be used inside function (fn) to represent current level. - /// @param fn function to apply with each level. This bool represent whether or not to stop iterating through levels. - static void forEachLevel(base::type::EnumType* startIndex, const std::function& fn); -}; -/// @brief Represents enumeration of ConfigurationType used to configure or access certain aspect -/// of logging -enum class ConfigurationType : base::type::EnumType { - /// @brief Determines whether or not corresponding level and logger of logging is enabled - /// You may disable all logs by using el::Level::Global - Enabled = 1, - /// @brief Whether or not to write corresponding log to log file - ToFile = 2, - /// @brief Whether or not to write corresponding level and logger log to standard output. - /// By standard output meaning termnal, command prompt etc - ToStandardOutput = 4, - /// @brief Determines format of logging corresponding level and logger. - Format = 8, - /// @brief Determines log file (full path) to write logs to for correponding level and logger - Filename = 16, - /// @brief Specifies precision of the subsecond part. It should be within range (1-6). - SubsecondPrecision = 32, - /// @brief Alias of SubsecondPrecision (for backward compatibility) - MillisecondsWidth = SubsecondPrecision, - /// @brief Determines whether or not performance tracking is enabled. - /// - /// @detail This does not depend on logger or level. Performance tracking always uses 'performance' logger - PerformanceTracking = 64, - /// @brief Specifies log file max size. - /// - /// @detail If file size of corresponding log file (for corresponding level) is >= specified size, log file will - /// be truncated and re-initiated. - MaxLogFileSize = 128, - /// @brief Specifies number of log entries to hold until we flush pending log data - LogFlushThreshold = 256, - /// @brief Represents unknown configuration - Unknown = 1010 -}; -/// @brief Static class that contains helper functions for el::ConfigurationType -class ConfigurationTypeHelper : base::StaticClass { - public: - /// @brief Represents minimum valid configuration type. Useful when iterating through enum. - static const base::type::EnumType kMinValid = static_cast(ConfigurationType::Enabled); - /// @brief Represents maximum valid configuration type. This is used internally and you should not need it. - static const base::type::EnumType kMaxValid = static_cast(ConfigurationType::MaxLogFileSize); - /// @brief Casts configuration type to int, useful for iterating through enum. - static base::type::EnumType castToInt(ConfigurationType configurationType) { - return static_cast(configurationType); - } - /// @brief Casts int(ushort) to configurationt type, useful for iterating through enum. - static ConfigurationType castFromInt(base::type::EnumType c) { - return static_cast(c); - } - /// @brief Converts configuration type to associated const char* - /// @returns Upper case string based configuration type. - static const char* convertToString(ConfigurationType configurationType); - /// @brief Converts from configStr to ConfigurationType - /// @param configStr Upper case string based configuration type. - /// Lower case is also valid but providing upper case is recommended. - static ConfigurationType convertFromString(const char* configStr); - /// @brief Applies specified function to each configuration type starting from startIndex - /// @param startIndex initial value to start the iteration from. This is passed by pointer and is left-shifted - /// so this can be used inside function (fn) to represent current configuration type. - /// @param fn function to apply with each configuration type. - /// This bool represent whether or not to stop iterating through configurations. - static inline void forEachConfigType(base::type::EnumType* startIndex, const std::function& fn); -}; -/// @brief Flags used while writing logs. This flags are set by user -enum class LoggingFlag : base::type::EnumType { - /// @brief Makes sure we have new line for each container log entry - NewLineForContainer = 1, - /// @brief Makes sure if -vmodule is used and does not specifies a module, then verbose - /// logging is allowed via that module. - AllowVerboseIfModuleNotSpecified = 2, - /// @brief When handling crashes by default, detailed crash reason will be logged as well - LogDetailedCrashReason = 4, - /// @brief Allows to disable application abortion when logged using FATAL level - DisableApplicationAbortOnFatalLog = 8, - /// @brief Flushes log with every log-entry (performance sensative) - Disabled by default - ImmediateFlush = 16, - /// @brief Enables strict file rolling - StrictLogFileSizeCheck = 32, - /// @brief Make terminal output colorful for supported terminals - ColoredTerminalOutput = 64, - /// @brief Supports use of multiple logging in same macro, e.g, CLOG(INFO, "default", "network") - MultiLoggerSupport = 128, - /// @brief Disables comparing performance tracker's checkpoints - DisablePerformanceTrackingCheckpointComparison = 256, - /// @brief Disable VModules - DisableVModules = 512, - /// @brief Disable VModules extensions - DisableVModulesExtensions = 1024, - /// @brief Enables hierarchical logging - HierarchicalLogging = 2048, - /// @brief Creates logger automatically when not available - CreateLoggerAutomatically = 4096, - /// @brief Adds spaces b/w logs that separated by left-shift operator - AutoSpacing = 8192, - /// @brief Preserves time format and does not convert it to sec, hour etc (performance tracking only) - FixedTimeFormat = 16384, - // @brief Ignore SIGINT or crash - IgnoreSigInt = 32768, -}; -namespace base { -/// @brief Namespace containing constants used internally. -namespace consts { -static const char kFormatSpecifierCharValue = 'v'; -static const char kFormatSpecifierChar = '%'; -static const unsigned int kMaxLogPerCounter = 100000; -static const unsigned int kMaxLogPerContainer = 100; -static const unsigned int kDefaultSubsecondPrecision = 3; - -#ifdef ELPP_DEFAULT_LOGGER -static const char* kDefaultLoggerId = ELPP_DEFAULT_LOGGER; -#else -static const char* kDefaultLoggerId = "default"; -#endif - -#if defined(ELPP_FEATURE_ALL) || defined(ELPP_FEATURE_PERFORMANCE_TRACKING) -#ifdef ELPP_DEFAULT_PERFORMANCE_LOGGER -static const char* kPerformanceLoggerId = ELPP_DEFAULT_PERFORMANCE_LOGGER; -#else -static const char* kPerformanceLoggerId = "performance"; -#endif // ELPP_DEFAULT_PERFORMANCE_LOGGER -#endif - -#if defined(ELPP_SYSLOG) -static const char* kSysLogLoggerId = "syslog"; -#endif // defined(ELPP_SYSLOG) - -#if ELPP_OS_WINDOWS -static const char* kFilePathSeperator = "\\"; -#else -static const char* kFilePathSeperator = "/"; -#endif // ELPP_OS_WINDOWS - -static const std::size_t kSourceFilenameMaxLength = 100; -static const std::size_t kSourceLineMaxLength = 10; -static const Level kPerformanceTrackerDefaultLevel = Level::Info; -const struct { - double value; - const base::type::char_t* unit; -} kTimeFormats[] = { - { 1000.0f, ELPP_LITERAL("us") }, - { 1000.0f, ELPP_LITERAL("ms") }, - { 60.0f, ELPP_LITERAL("seconds") }, - { 60.0f, ELPP_LITERAL("minutes") }, - { 24.0f, ELPP_LITERAL("hours") }, - { 7.0f, ELPP_LITERAL("days") } -}; -static const int kTimeFormatsCount = sizeof(kTimeFormats) / sizeof(kTimeFormats[0]); -const struct { - int numb; - const char* name; - const char* brief; - const char* detail; -} kCrashSignals[] = { - // NOTE: Do not re-order, if you do please check CrashHandler(bool) constructor and CrashHandler::setHandler(..) - { - SIGABRT, "SIGABRT", "Abnormal termination", - "Program was abnormally terminated." - }, - { - SIGFPE, "SIGFPE", "Erroneous arithmetic operation", - "Arithemetic operation issue such as division by zero or operation resulting in overflow." - }, - { - SIGILL, "SIGILL", "Illegal instruction", - "Generally due to a corruption in the code or to an attempt to execute data." - }, - { - SIGSEGV, "SIGSEGV", "Invalid access to memory", - "Program is trying to read an invalid (unallocated, deleted or corrupted) or inaccessible memory." - }, - { - SIGINT, "SIGINT", "Interactive attention signal", - "Interruption generated (generally) by user or operating system." - }, -}; -static const int kCrashSignalsCount = sizeof(kCrashSignals) / sizeof(kCrashSignals[0]); -} // namespace consts -} // namespace base -typedef std::function PreRollOutCallback; -namespace base { -static inline void defaultPreRollOutCallback(const char*, std::size_t, Level level) {} -/// @brief Enum to represent timestamp unit -enum class TimestampUnit : base::type::EnumType { - Microsecond = 0, Millisecond = 1, Second = 2, Minute = 3, Hour = 4, Day = 5 -}; -/// @brief Format flags used to determine specifiers that are active for performance improvements. -enum class FormatFlags : base::type::EnumType { - DateTime = 1 << 1, - LoggerId = 1 << 2, - File = 1 << 3, - Line = 1 << 4, - Location = 1 << 5, - Function = 1 << 6, - User = 1 << 7, - Host = 1 << 8, - LogMessage = 1 << 9, - VerboseLevel = 1 << 10, - AppName = 1 << 11, - ThreadId = 1 << 12, - Level = 1 << 13, - FileBase = 1 << 14, - LevelShort = 1 << 15 -}; -/// @brief A subsecond precision class containing actual width and offset of the subsecond part -class SubsecondPrecision { - public: - SubsecondPrecision(void) { - init(base::consts::kDefaultSubsecondPrecision); - } - explicit SubsecondPrecision(int width) { - init(width); - } - bool operator==(const SubsecondPrecision& ssPrec) { - return m_width == ssPrec.m_width && m_offset == ssPrec.m_offset; - } - int m_width; - unsigned int m_offset; - private: - void init(int width); -}; -/// @brief Type alias of SubsecondPrecision -typedef SubsecondPrecision MillisecondsWidth; -/// @brief Namespace containing utility functions/static classes used internally -namespace utils { -/// @brief Deletes memory safely and points to null -template -static -typename std::enable_if::value, void>::type -safeDelete(T*& pointer) { - if (pointer == nullptr) - return; - delete pointer; - pointer = nullptr; -} -/// @brief Bitwise operations for C++11 strong enum class. This casts e into Flag_T and returns value after bitwise operation -/// Use these function as 
flag = bitwise::Or(MyEnum::val1, flag);
-namespace bitwise { -template -static inline base::type::EnumType And(Enum e, base::type::EnumType flag) { - return static_cast(flag) & static_cast(e); -} -template -static inline base::type::EnumType Not(Enum e, base::type::EnumType flag) { - return static_cast(flag) & ~(static_cast(e)); -} -template -static inline base::type::EnumType Or(Enum e, base::type::EnumType flag) { - return static_cast(flag) | static_cast(e); -} -} // namespace bitwise -template -static inline void addFlag(Enum e, base::type::EnumType* flag) { - *flag = base::utils::bitwise::Or(e, *flag); -} -template -static inline void removeFlag(Enum e, base::type::EnumType* flag) { - *flag = base::utils::bitwise::Not(e, *flag); -} -template -static inline bool hasFlag(Enum e, base::type::EnumType flag) { - return base::utils::bitwise::And(e, flag) > 0x0; -} -} // namespace utils -namespace threading { -#if ELPP_THREADING_ENABLED -# if !ELPP_USE_STD_THREADING -namespace internal { -/// @brief A mutex wrapper for compiler that dont yet support std::recursive_mutex -class Mutex : base::NoCopy { - public: - Mutex(void) { -# if ELPP_OS_UNIX - pthread_mutexattr_t attr; - pthread_mutexattr_init(&attr); - pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_RECURSIVE); - pthread_mutex_init(&m_underlyingMutex, &attr); - pthread_mutexattr_destroy(&attr); -# elif ELPP_OS_WINDOWS - InitializeCriticalSection(&m_underlyingMutex); -# endif // ELPP_OS_UNIX - } - - virtual ~Mutex(void) { -# if ELPP_OS_UNIX - pthread_mutex_destroy(&m_underlyingMutex); -# elif ELPP_OS_WINDOWS - DeleteCriticalSection(&m_underlyingMutex); -# endif // ELPP_OS_UNIX - } - - inline void lock(void) { -# if ELPP_OS_UNIX - pthread_mutex_lock(&m_underlyingMutex); -# elif ELPP_OS_WINDOWS - EnterCriticalSection(&m_underlyingMutex); -# endif // ELPP_OS_UNIX - } - - inline bool try_lock(void) { -# if ELPP_OS_UNIX - return (pthread_mutex_trylock(&m_underlyingMutex) == 0); -# elif ELPP_OS_WINDOWS - return TryEnterCriticalSection(&m_underlyingMutex); -# endif // ELPP_OS_UNIX - } - - inline void unlock(void) { -# if ELPP_OS_UNIX - pthread_mutex_unlock(&m_underlyingMutex); -# elif ELPP_OS_WINDOWS - LeaveCriticalSection(&m_underlyingMutex); -# endif // ELPP_OS_UNIX - } - - private: -# if ELPP_OS_UNIX - pthread_mutex_t m_underlyingMutex; -# elif ELPP_OS_WINDOWS - CRITICAL_SECTION m_underlyingMutex; -# endif // ELPP_OS_UNIX -}; -/// @brief Scoped lock for compiler that dont yet support std::lock_guard -template -class ScopedLock : base::NoCopy { - public: - explicit ScopedLock(M& mutex) { - m_mutex = &mutex; - m_mutex->lock(); - } - - virtual ~ScopedLock(void) { - m_mutex->unlock(); - } - private: - M* m_mutex; - ScopedLock(void); -}; -} // namespace internal -typedef base::threading::internal::Mutex Mutex; -typedef base::threading::internal::ScopedLock ScopedLock; -# else -typedef std::recursive_mutex Mutex; -typedef std::lock_guard ScopedLock; -# endif // !ELPP_USE_STD_THREADING -#else -namespace internal { -/// @brief Mutex wrapper used when multi-threading is disabled. -class NoMutex : base::NoCopy { - public: - NoMutex(void) {} - inline void lock(void) {} - inline bool try_lock(void) { - return true; - } - inline void unlock(void) {} -}; -/// @brief Lock guard wrapper used when multi-threading is disabled. -template -class NoScopedLock : base::NoCopy { - public: - explicit NoScopedLock(Mutex&) { - } - virtual ~NoScopedLock(void) { - } - private: - NoScopedLock(void); -}; -} // namespace internal -typedef base::threading::internal::NoMutex Mutex; -typedef base::threading::internal::NoScopedLock ScopedLock; -#endif // ELPP_THREADING_ENABLED -/// @brief Base of thread safe class, this class is inheritable-only -class ThreadSafe { - public: - virtual inline void acquireLock(void) ELPP_FINAL { m_mutex.lock(); } - virtual inline void releaseLock(void) ELPP_FINAL { m_mutex.unlock(); } - virtual inline base::threading::Mutex& lock(void) ELPP_FINAL { return m_mutex; } - protected: - ThreadSafe(void) {} - virtual ~ThreadSafe(void) {} - private: - base::threading::Mutex m_mutex; -}; - -#if ELPP_THREADING_ENABLED -# if !ELPP_USE_STD_THREADING -/// @brief Gets ID of currently running threading in windows systems. On unix, nothing is returned. -static std::string getCurrentThreadId(void) { - std::stringstream ss; -# if (ELPP_OS_WINDOWS) - ss << GetCurrentThreadId(); -# endif // (ELPP_OS_WINDOWS) - return ss.str(); -} -# else -/// @brief Gets ID of currently running threading using std::this_thread::get_id() -static std::string getCurrentThreadId(void) { - std::stringstream ss; - ss << std::this_thread::get_id(); - return ss.str(); -} -# endif // !ELPP_USE_STD_THREADING -#else -static inline std::string getCurrentThreadId(void) { - return std::string(); -} -#endif // ELPP_THREADING_ENABLED -} // namespace threading -namespace utils { -class File : base::StaticClass { - public: - /// @brief Creates new out file stream for specified filename. - /// @return Pointer to newly created fstream or nullptr - static base::type::fstream_t* newFileStream(const std::string& filename); - - /// @brief Gets size of file provided in stream - static std::size_t getSizeOfFile(base::type::fstream_t* fs); - - /// @brief Determines whether or not provided path exist in current file system - static bool pathExists(const char* path, bool considerFile = false); - - /// @brief Creates specified path on file system - /// @param path Path to create. - static bool createPath(const std::string& path); - /// @brief Extracts path of filename with leading slash - static std::string extractPathFromFilename(const std::string& fullPath, - const char* seperator = base::consts::kFilePathSeperator); - /// @brief builds stripped filename and puts it in buff - static void buildStrippedFilename(const char* filename, char buff[], - std::size_t limit = base::consts::kSourceFilenameMaxLength); - /// @brief builds base filename and puts it in buff - static void buildBaseFilename(const std::string& fullPath, char buff[], - std::size_t limit = base::consts::kSourceFilenameMaxLength, - const char* seperator = base::consts::kFilePathSeperator); -}; -/// @brief String utilities helper class used internally. You should not use it. -class Str : base::StaticClass { - public: - /// @brief Checks if character is digit. Dont use libc implementation of it to prevent locale issues. - static inline bool isDigit(char c) { - return c >= '0' && c <= '9'; - } - - /// @brief Matches wildcards, '*' and '?' only supported. - static bool wildCardMatch(const char* str, const char* pattern); - - static std::string& ltrim(std::string& str); - static std::string& rtrim(std::string& str); - static std::string& trim(std::string& str); - - /// @brief Determines whether or not str starts with specified string - /// @param str String to check - /// @param start String to check against - /// @return Returns true if starts with specified string, false otherwise - static bool startsWith(const std::string& str, const std::string& start); - - /// @brief Determines whether or not str ends with specified string - /// @param str String to check - /// @param end String to check against - /// @return Returns true if ends with specified string, false otherwise - static bool endsWith(const std::string& str, const std::string& end); - - /// @brief Replaces all instances of replaceWhat with 'replaceWith'. Original variable is changed for performance. - /// @param [in,out] str String to replace from - /// @param replaceWhat Character to replace - /// @param replaceWith Character to replace with - /// @return Modified version of str - static std::string& replaceAll(std::string& str, char replaceWhat, char replaceWith); - - /// @brief Replaces all instances of 'replaceWhat' with 'replaceWith'. (String version) Replaces in place - /// @param str String to replace from - /// @param replaceWhat Character to replace - /// @param replaceWith Character to replace with - /// @return Modified (original) str - static std::string& replaceAll(std::string& str, const std::string& replaceWhat, - const std::string& replaceWith); - - static void replaceFirstWithEscape(base::type::string_t& str, const base::type::string_t& replaceWhat, - const base::type::string_t& replaceWith); -#if defined(ELPP_UNICODE) - static void replaceFirstWithEscape(base::type::string_t& str, const base::type::string_t& replaceWhat, - const std::string& replaceWith); -#endif // defined(ELPP_UNICODE) - /// @brief Converts string to uppercase - /// @param str String to convert - /// @return Uppercase string - static std::string& toUpper(std::string& str); - - /// @brief Compares cstring equality - uses strcmp - static bool cStringEq(const char* s1, const char* s2); - - /// @brief Compares cstring equality (case-insensitive) - uses toupper(char) - /// Dont use strcasecmp because of CRT (VC++) - static bool cStringCaseEq(const char* s1, const char* s2); - - /// @brief Returns true if c exist in str - static bool contains(const char* str, char c); - - static char* convertAndAddToBuff(std::size_t n, int len, char* buf, const char* bufLim, bool zeroPadded = true); - static char* addToBuff(const char* str, char* buf, const char* bufLim); - static char* clearBuff(char buff[], std::size_t lim); - - /// @brief Converst wchar* to char* - /// NOTE: Need to free return value after use! - static char* wcharPtrToCharPtr(const wchar_t* line); -}; -/// @brief Operating System helper static class used internally. You should not use it. -class OS : base::StaticClass { - public: -#if ELPP_OS_WINDOWS - /// @brief Gets environment variables for Windows based OS. - /// We are not using getenv(const char*) because of CRT deprecation - /// @param varname Variable name to get environment variable value for - /// @return If variable exist the value of it otherwise nullptr - static const char* getWindowsEnvironmentVariable(const char* varname); -#endif // ELPP_OS_WINDOWS -#if ELPP_OS_ANDROID - /// @brief Reads android property value - static std::string getProperty(const char* prop); - - /// @brief Reads android device name - static std::string getDeviceName(void); -#endif // ELPP_OS_ANDROID - - /// @brief Runs command on terminal and returns the output. - /// - /// @detail This is applicable only on unix based systems, for all other OS, an empty string is returned. - /// @param command Bash command - /// @return Result of bash output or empty string if no result found. - static const std::string getBashOutput(const char* command); - - /// @brief Gets environment variable. This is cross-platform and CRT safe (for VC++) - /// @param variableName Environment variable name - /// @param defaultVal If no environment variable or value found the value to return by default - /// @param alternativeBashCommand If environment variable not found what would be alternative bash command - /// in order to look for value user is looking for. E.g, for 'user' alternative command will 'whoami' - static std::string getEnvironmentVariable(const char* variableName, const char* defaultVal, - const char* alternativeBashCommand = nullptr); - /// @brief Gets current username. - static std::string currentUser(void); - - /// @brief Gets current host name or computer name. - /// - /// @detail For android systems this is device name with its manufacturer and model seperated by hyphen - static std::string currentHost(void); - /// @brief Whether or not terminal supports colors - static bool termSupportsColor(void); -}; -/// @brief Contains utilities for cross-platform date/time. This class make use of el::base::utils::Str -class DateTime : base::StaticClass { - public: - /// @brief Cross platform gettimeofday for Windows and unix platform. This can be used to determine current microsecond. - /// - /// @detail For unix system it uses gettimeofday(timeval*, timezone*) and for Windows, a seperate implementation is provided - /// @param [in,out] tv Pointer that gets updated - static void gettimeofday(struct timeval* tv); - - /// @brief Gets current date and time with a subsecond part. - /// @param format User provided date/time format - /// @param ssPrec A pointer to base::SubsecondPrecision from configuration (non-null) - /// @returns string based date time in specified format. - static std::string getDateTime(const char* format, const base::SubsecondPrecision* ssPrec); - - /// @brief Converts timeval (struct from ctime) to string using specified format and subsecond precision - static std::string timevalToString(struct timeval tval, const char* format, - const el::base::SubsecondPrecision* ssPrec); - - /// @brief Formats time to get unit accordingly, units like second if > 1000 or minutes if > 60000 etc - static base::type::string_t formatTime(unsigned long long time, base::TimestampUnit timestampUnit); - - /// @brief Gets time difference in milli/micro second depending on timestampUnit - static unsigned long long getTimeDifference(const struct timeval& endTime, const struct timeval& startTime, - base::TimestampUnit timestampUnit); - - - static struct ::tm* buildTimeInfo(struct timeval* currTime, struct ::tm* timeInfo); - private: - static char* parseFormat(char* buf, std::size_t bufSz, const char* format, const struct tm* tInfo, - std::size_t msec, const base::SubsecondPrecision* ssPrec); -}; -/// @brief Command line arguments for application if specified using el::Helpers::setArgs(..) or START_EASYLOGGINGPP(..) -class CommandLineArgs { - public: - CommandLineArgs(void) { - setArgs(0, static_cast(nullptr)); - } - CommandLineArgs(int argc, const char** argv) { - setArgs(argc, argv); - } - CommandLineArgs(int argc, char** argv) { - setArgs(argc, argv); - } - virtual ~CommandLineArgs(void) {} - /// @brief Sets arguments and parses them - inline void setArgs(int argc, const char** argv) { - setArgs(argc, const_cast(argv)); - } - /// @brief Sets arguments and parses them - void setArgs(int argc, char** argv); - /// @brief Returns true if arguments contain paramKey with a value (seperated by '=') - bool hasParamWithValue(const char* paramKey) const; - /// @brief Returns value of arguments - /// @see hasParamWithValue(const char*) - const char* getParamValue(const char* paramKey) const; - /// @brief Return true if arguments has a param (not having a value) i,e without '=' - bool hasParam(const char* paramKey) const; - /// @brief Returns true if no params available. This exclude argv[0] - bool empty(void) const; - /// @brief Returns total number of arguments. This exclude argv[0] - std::size_t size(void) const; - friend base::type::ostream_t& operator<<(base::type::ostream_t& os, const CommandLineArgs& c); - - private: - int m_argc; - char** m_argv; - std::unordered_map m_paramsWithValue; - std::vector m_params; -}; -/// @brief Abstract registry (aka repository) that provides basic interface for pointer repository specified by T_Ptr type. -/// -/// @detail Most of the functions are virtual final methods but anything implementing this abstract class should implement -/// unregisterAll() and deepCopy(const AbstractRegistry&) and write registerNew() method according to container -/// and few more methods; get() to find element, unregister() to unregister single entry. -/// Please note that this is thread-unsafe and should also implement thread-safety mechanisms in implementation. -template -class AbstractRegistry : public base::threading::ThreadSafe { - public: - typedef typename Container::iterator iterator; - typedef typename Container::const_iterator const_iterator; - - /// @brief Default constructor - AbstractRegistry(void) {} - - /// @brief Move constructor that is useful for base classes - AbstractRegistry(AbstractRegistry&& sr) { - if (this == &sr) { - return; - } - unregisterAll(); - m_list = std::move(sr.m_list); - } - - bool operator==(const AbstractRegistry& other) { - if (size() != other.size()) { - return false; - } - for (std::size_t i = 0; i < m_list.size(); ++i) { - if (m_list.at(i) != other.m_list.at(i)) { - return false; - } - } - return true; - } - - bool operator!=(const AbstractRegistry& other) { - if (size() != other.size()) { - return true; - } - for (std::size_t i = 0; i < m_list.size(); ++i) { - if (m_list.at(i) != other.m_list.at(i)) { - return true; - } - } - return false; - } - - /// @brief Assignment move operator - AbstractRegistry& operator=(AbstractRegistry&& sr) { - if (this == &sr) { - return *this; - } - unregisterAll(); - m_list = std::move(sr.m_list); - return *this; - } - - virtual ~AbstractRegistry(void) { - } - - /// @return Iterator pointer from start of repository - virtual inline iterator begin(void) ELPP_FINAL { - return m_list.begin(); - } - - /// @return Iterator pointer from end of repository - virtual inline iterator end(void) ELPP_FINAL { - return m_list.end(); - } - - - /// @return Constant iterator pointer from start of repository - virtual inline const_iterator cbegin(void) const ELPP_FINAL { - return m_list.cbegin(); - } - - /// @return End of repository - virtual inline const_iterator cend(void) const ELPP_FINAL { - return m_list.cend(); - } - - /// @return Whether or not repository is empty - virtual inline bool empty(void) const ELPP_FINAL { - return m_list.empty(); - } - - /// @return Size of repository - virtual inline std::size_t size(void) const ELPP_FINAL { - return m_list.size(); - } - - /// @brief Returns underlying container by reference - virtual inline Container& list(void) ELPP_FINAL { - return m_list; - } - - /// @brief Returns underlying container by constant reference. - virtual inline const Container& list(void) const ELPP_FINAL { - return m_list; - } - - /// @brief Unregisters all the pointers from current repository. - virtual void unregisterAll(void) = 0; - - protected: - virtual void deepCopy(const AbstractRegistry&) = 0; - void reinitDeepCopy(const AbstractRegistry& sr) { - unregisterAll(); - deepCopy(sr); - } - - private: - Container m_list; -}; - -/// @brief A pointer registry mechanism to manage memory and provide search functionalities. (non-predicate version) -/// -/// @detail NOTE: This is thread-unsafe implementation (although it contains lock function, it does not use these functions) -/// of AbstractRegistry. Any implementation of this class should be -/// explicitly (by using lock functions) -template -class Registry : public AbstractRegistry> { - public: - typedef typename Registry::iterator iterator; - typedef typename Registry::const_iterator const_iterator; - - Registry(void) {} - - /// @brief Copy constructor that is useful for base classes. Try to avoid this constructor, use move constructor. - Registry(const Registry& sr) : AbstractRegistry>() { - if (this == &sr) { - return; - } - this->reinitDeepCopy(sr); - } - - /// @brief Assignment operator that unregisters all the existing registeries and deeply copies each of repo element - /// @see unregisterAll() - /// @see deepCopy(const AbstractRegistry&) - Registry& operator=(const Registry& sr) { - if (this == &sr) { - return *this; - } - this->reinitDeepCopy(sr); - return *this; - } - - virtual ~Registry(void) { - unregisterAll(); - } - - protected: - virtual void unregisterAll(void) ELPP_FINAL { - if (!this->empty()) { - for (auto&& curr : this->list()) { - base::utils::safeDelete(curr.second); - } - this->list().clear(); - } - } - -/// @brief Registers new registry to repository. - virtual void registerNew(const T_Key& uniqKey, T_Ptr* ptr) ELPP_FINAL { - unregister(uniqKey); - this->list().insert(std::make_pair(uniqKey, ptr)); - } - -/// @brief Unregisters single entry mapped to specified unique key - void unregister(const T_Key& uniqKey) { - T_Ptr* existing = get(uniqKey); - if (existing != nullptr) { - this->list().erase(uniqKey); - base::utils::safeDelete(existing); - } - } - -/// @brief Gets pointer from repository. If none found, nullptr is returned. - T_Ptr* get(const T_Key& uniqKey) { - iterator it = this->list().find(uniqKey); - return it == this->list().end() - ? nullptr - : it->second; - } - - private: - virtual void deepCopy(const AbstractRegistry>& sr) ELPP_FINAL { - for (const_iterator it = sr.cbegin(); it != sr.cend(); ++it) { - registerNew(it->first, new T_Ptr(*it->second)); - } - } -}; - -/// @brief A pointer registry mechanism to manage memory and provide search functionalities. (predicate version) -/// -/// @detail NOTE: This is thread-unsafe implementation of AbstractRegistry. Any implementation of this class -/// should be made thread-safe explicitly -template -class RegistryWithPred : public AbstractRegistry> { - public: - typedef typename RegistryWithPred::iterator iterator; - typedef typename RegistryWithPred::const_iterator const_iterator; - - RegistryWithPred(void) { - } - - virtual ~RegistryWithPred(void) { - unregisterAll(); - } - - /// @brief Copy constructor that is useful for base classes. Try to avoid this constructor, use move constructor. - RegistryWithPred(const RegistryWithPred& sr) : AbstractRegistry>() { - if (this == &sr) { - return; - } - this->reinitDeepCopy(sr); - } - - /// @brief Assignment operator that unregisters all the existing registeries and deeply copies each of repo element - /// @see unregisterAll() - /// @see deepCopy(const AbstractRegistry&) - RegistryWithPred& operator=(const RegistryWithPred& sr) { - if (this == &sr) { - return *this; - } - this->reinitDeepCopy(sr); - return *this; - } - - friend base::type::ostream_t& operator<<(base::type::ostream_t& os, const RegistryWithPred& sr) { - for (const_iterator it = sr.list().begin(); it != sr.list().end(); ++it) { - os << ELPP_LITERAL(" ") << **it << ELPP_LITERAL("\n"); - } - return os; - } - - protected: - virtual void unregisterAll(void) ELPP_FINAL { - if (!this->empty()) { - for (auto&& curr : this->list()) { - base::utils::safeDelete(curr); - } - this->list().clear(); - } - } - - virtual void unregister(T_Ptr*& ptr) ELPP_FINAL { - if (ptr) { - iterator iter = this->begin(); - for (; iter != this->end(); ++iter) { - if (ptr == *iter) { - break; - } - } - if (iter != this->end() && *iter != nullptr) { - this->list().erase(iter); - base::utils::safeDelete(*iter); - } - } - } - - virtual inline void registerNew(T_Ptr* ptr) ELPP_FINAL { - this->list().push_back(ptr); - } - -/// @brief Gets pointer from repository with speicifed arguments. Arguments are passed to predicate -/// in order to validate pointer. - template - T_Ptr* get(const T& arg1, const T2 arg2) { - iterator iter = std::find_if(this->list().begin(), this->list().end(), Pred(arg1, arg2)); - if (iter != this->list().end() && *iter != nullptr) { - return *iter; - } - return nullptr; - } - - private: - virtual void deepCopy(const AbstractRegistry>& sr) { - for (const_iterator it = sr.list().begin(); it != sr.list().end(); ++it) { - registerNew(new T_Ptr(**it)); - } - } -}; -class Utils { - public: - template - static bool installCallback(const std::string& id, std::unordered_map* mapT) { - if (mapT->find(id) == mapT->end()) { - mapT->insert(std::make_pair(id, TPtr(new T()))); - return true; - } - return false; - } - - template - static void uninstallCallback(const std::string& id, std::unordered_map* mapT) { - if (mapT->find(id) != mapT->end()) { - mapT->erase(id); - } - } - - template - static T* callback(const std::string& id, std::unordered_map* mapT) { - typename std::unordered_map::iterator iter = mapT->find(id); - if (iter != mapT->end()) { - return static_cast(iter->second.get()); - } - return nullptr; - } -}; -} // namespace utils -} // namespace base -/// @brief Base of Easylogging++ friendly class -/// -/// @detail After inheriting this class publicly, implement pure-virtual function `void log(std::ostream&) const` -class Loggable { - public: - virtual ~Loggable(void) {} - virtual void log(el::base::type::ostream_t&) const = 0; - private: - friend inline el::base::type::ostream_t& operator<<(el::base::type::ostream_t& os, const Loggable& loggable) { - loggable.log(os); - return os; - } -}; -namespace base { -/// @brief Represents log format containing flags and date format. This is used internally to start initial log -class LogFormat : public Loggable { - public: - LogFormat(void); - LogFormat(Level level, const base::type::string_t& format); - LogFormat(const LogFormat& logFormat); - LogFormat(LogFormat&& logFormat); - LogFormat& operator=(const LogFormat& logFormat); - virtual ~LogFormat(void) {} - bool operator==(const LogFormat& other); - - /// @brief Updates format to be used while logging. - /// @param userFormat User provided format - void parseFromFormat(const base::type::string_t& userFormat); - - inline Level level(void) const { - return m_level; - } - - inline const base::type::string_t& userFormat(void) const { - return m_userFormat; - } - - inline const base::type::string_t& format(void) const { - return m_format; - } - - inline const std::string& dateTimeFormat(void) const { - return m_dateTimeFormat; - } - - inline base::type::EnumType flags(void) const { - return m_flags; - } - - inline bool hasFlag(base::FormatFlags flag) const { - return base::utils::hasFlag(flag, m_flags); - } - - virtual void log(el::base::type::ostream_t& os) const { - os << m_format; - } - - protected: - /// @brief Updates date time format if available in currFormat. - /// @param index Index where %datetime, %date or %time was found - /// @param [in,out] currFormat current format that is being used to format - virtual void updateDateFormat(std::size_t index, base::type::string_t& currFormat) ELPP_FINAL; - - /// @brief Updates %level from format. This is so that we dont have to do it at log-writing-time. It uses m_format and m_level - virtual void updateFormatSpec(void) ELPP_FINAL; - - inline void addFlag(base::FormatFlags flag) { - base::utils::addFlag(flag, &m_flags); - } - - private: - Level m_level; - base::type::string_t m_userFormat; - base::type::string_t m_format; - std::string m_dateTimeFormat; - base::type::EnumType m_flags; - std::string m_currentUser; - std::string m_currentHost; - friend class el::Logger; // To resolve loggerId format specifier easily -}; -} // namespace base -/// @brief Resolving function for format specifier -typedef std::function FormatSpecifierValueResolver; -/// @brief User-provided custom format specifier -/// @see el::Helpers::installCustomFormatSpecifier -/// @see FormatSpecifierValueResolver -class CustomFormatSpecifier { - public: - CustomFormatSpecifier(const char* formatSpecifier, const FormatSpecifierValueResolver& resolver) : - m_formatSpecifier(formatSpecifier), m_resolver(resolver) {} - inline const char* formatSpecifier(void) const { - return m_formatSpecifier; - } - inline const FormatSpecifierValueResolver& resolver(void) const { - return m_resolver; - } - inline bool operator==(const char* formatSpecifier) { - return strcmp(m_formatSpecifier, formatSpecifier) == 0; - } - - private: - const char* m_formatSpecifier; - FormatSpecifierValueResolver m_resolver; -}; -/// @brief Represents single configuration that has representing level, configuration type and a string based value. -/// -/// @detail String based value means any value either its boolean, integer or string itself, it will be embedded inside quotes -/// and will be parsed later. -/// -/// Consider some examples below: -/// * el::Configuration confEnabledInfo(el::Level::Info, el::ConfigurationType::Enabled, "true"); -/// * el::Configuration confMaxLogFileSizeInfo(el::Level::Info, el::ConfigurationType::MaxLogFileSize, "2048"); -/// * el::Configuration confFilenameInfo(el::Level::Info, el::ConfigurationType::Filename, "/var/log/my.log"); -class Configuration : public Loggable { - public: - Configuration(const Configuration& c); - Configuration& operator=(const Configuration& c); - - virtual ~Configuration(void) { - } - - /// @brief Full constructor used to sets value of configuration - Configuration(Level level, ConfigurationType configurationType, const std::string& value); - - /// @brief Gets level of current configuration - inline Level level(void) const { - return m_level; - } - - /// @brief Gets configuration type of current configuration - inline ConfigurationType configurationType(void) const { - return m_configurationType; - } - - /// @brief Gets string based configuration value - inline const std::string& value(void) const { - return m_value; - } - - /// @brief Set string based configuration value - /// @param value Value to set. Values have to be std::string; For boolean values use "true", "false", for any integral values - /// use them in quotes. They will be parsed when configuring - inline void setValue(const std::string& value) { - m_value = value; - } - - virtual void log(el::base::type::ostream_t& os) const; - - /// @brief Used to find configuration from configuration (pointers) repository. Avoid using it. - class Predicate { - public: - Predicate(Level level, ConfigurationType configurationType); - - bool operator()(const Configuration* conf) const; - - private: - Level m_level; - ConfigurationType m_configurationType; - }; - - private: - Level m_level; - ConfigurationType m_configurationType; - std::string m_value; -}; - -/// @brief Thread-safe Configuration repository -/// -/// @detail This repository represents configurations for all the levels and configuration type mapped to a value. -class Configurations : public base::utils::RegistryWithPred { - public: - /// @brief Default constructor with empty repository - Configurations(void); - - /// @brief Constructor used to set configurations using configuration file. - /// @param configurationFile Full path to configuration file - /// @param useDefaultsForRemaining Lets you set the remaining configurations to default. - /// @param base If provided, this configuration will be based off existing repository that this argument is pointing to. - /// @see parseFromFile(const std::string&, Configurations* base) - /// @see setRemainingToDefault() - Configurations(const std::string& configurationFile, bool useDefaultsForRemaining = true, - Configurations* base = nullptr); - - virtual ~Configurations(void) { - } - - /// @brief Parses configuration from file. - /// @param configurationFile Full path to configuration file - /// @param base Configurations to base new configuration repository off. This value is used when you want to use - /// existing Configurations to base all the values and then set rest of configuration via configuration file. - /// @return True if successfully parsed, false otherwise. You may define 'ELPP_DEBUG_ASSERT_FAILURE' to make sure you - /// do not proceed without successful parse. - bool parseFromFile(const std::string& configurationFile, Configurations* base = nullptr); - - /// @brief Parse configurations from configuration string. - /// - /// @detail This configuration string has same syntax as configuration file contents. Make sure all the necessary - /// new line characters are provided. - /// @param base Configurations to base new configuration repository off. This value is used when you want to use - /// existing Configurations to base all the values and then set rest of configuration via configuration text. - /// @return True if successfully parsed, false otherwise. You may define 'ELPP_DEBUG_ASSERT_FAILURE' to make sure you - /// do not proceed without successful parse. - bool parseFromText(const std::string& configurationsString, Configurations* base = nullptr); - - /// @brief Sets configuration based-off an existing configurations. - /// @param base Pointer to existing configurations. - void setFromBase(Configurations* base); - - /// @brief Determines whether or not specified configuration type exists in the repository. - /// - /// @detail Returns as soon as first level is found. - /// @param configurationType Type of configuration to check existence for. - bool hasConfiguration(ConfigurationType configurationType); - - /// @brief Determines whether or not specified configuration type exists for specified level - /// @param level Level to check - /// @param configurationType Type of configuration to check existence for. - bool hasConfiguration(Level level, ConfigurationType configurationType); - - /// @brief Sets value of configuration for specified level. - /// - /// @detail Any existing configuration for specified level will be replaced. Also note that configuration types - /// ConfigurationType::SubsecondPrecision and ConfigurationType::PerformanceTracking will be ignored if not set for - /// Level::Global because these configurations are not dependant on level. - /// @param level Level to set configuration for (el::Level). - /// @param configurationType Type of configuration (el::ConfigurationType) - /// @param value A string based value. Regardless of what the data type of configuration is, it will always be string - /// from users' point of view. This is then parsed later to be used internally. - /// @see Configuration::setValue(const std::string& value) - /// @see el::Level - /// @see el::ConfigurationType - void set(Level level, ConfigurationType configurationType, const std::string& value); - - /// @brief Sets single configuration based on other single configuration. - /// @see set(Level level, ConfigurationType configurationType, const std::string& value) - void set(Configuration* conf); - - inline Configuration* get(Level level, ConfigurationType configurationType) { - base::threading::ScopedLock scopedLock(lock()); - return RegistryWithPred::get(level, configurationType); - } - - /// @brief Sets configuration for all levels. - /// @param configurationType Type of configuration - /// @param value String based value - /// @see Configurations::set(Level level, ConfigurationType configurationType, const std::string& value) - inline void setGlobally(ConfigurationType configurationType, const std::string& value) { - setGlobally(configurationType, value, false); - } - - /// @brief Clears repository so that all the configurations are unset - inline void clear(void) { - base::threading::ScopedLock scopedLock(lock()); - unregisterAll(); - } - - /// @brief Gets configuration file used in parsing this configurations. - /// - /// @detail If this repository was set manually or by text this returns empty string. - inline const std::string& configurationFile(void) const { - return m_configurationFile; - } - - /// @brief Sets configurations to "factory based" configurations. - void setToDefault(void); - - /// @brief Lets you set the remaining configurations to default. - /// - /// @detail By remaining, it means that the level/type a configuration does not exist for. - /// This function is useful when you want to minimize chances of failures, e.g, if you have a configuration file that sets - /// configuration for all the configurations except for Enabled or not, we use this so that ENABLED is set to default i.e, - /// true. If you dont do this explicitly (either by calling this function or by using second param in Constructor - /// and try to access a value, an error is thrown - void setRemainingToDefault(void); - - /// @brief Parser used internally to parse configurations from file or text. - /// - /// @detail This class makes use of base::utils::Str. - /// You should not need this unless you are working on some tool for Easylogging++ - class Parser : base::StaticClass { - public: - /// @brief Parses configuration from file. - /// @param configurationFile Full path to configuration file - /// @param sender Sender configurations pointer. Usually 'this' is used from calling class - /// @param base Configurations to base new configuration repository off. This value is used when you want to use - /// existing Configurations to base all the values and then set rest of configuration via configuration file. - /// @return True if successfully parsed, false otherwise. You may define '_STOP_ON_FIRSTELPP_ASSERTION' to make sure you - /// do not proceed without successful parse. - static bool parseFromFile(const std::string& configurationFile, Configurations* sender, - Configurations* base = nullptr); - - /// @brief Parse configurations from configuration string. - /// - /// @detail This configuration string has same syntax as configuration file contents. Make sure all the necessary - /// new line characters are provided. You may define '_STOP_ON_FIRSTELPP_ASSERTION' to make sure you - /// do not proceed without successful parse (This is recommended) - /// @param configurationsString the configuration in plain text format - /// @param sender Sender configurations pointer. Usually 'this' is used from calling class - /// @param base Configurations to base new configuration repository off. This value is used when you want to use - /// existing Configurations to base all the values and then set rest of configuration via configuration text. - /// @return True if successfully parsed, false otherwise. - static bool parseFromText(const std::string& configurationsString, Configurations* sender, - Configurations* base = nullptr); - - private: - friend class el::Loggers; - static void ignoreComments(std::string* line); - static bool isLevel(const std::string& line); - static bool isComment(const std::string& line); - static inline bool isConfig(const std::string& line); - static bool parseLine(std::string* line, std::string* currConfigStr, std::string* currLevelStr, Level* currLevel, - Configurations* conf); - }; - - private: - std::string m_configurationFile; - bool m_isFromFile; - friend class el::Loggers; - - /// @brief Unsafely sets configuration if does not already exist - void unsafeSetIfNotExist(Level level, ConfigurationType configurationType, const std::string& value); - - /// @brief Thread unsafe set - void unsafeSet(Level level, ConfigurationType configurationType, const std::string& value); - - /// @brief Sets configurations for all levels including Level::Global if includeGlobalLevel is true - /// @see Configurations::setGlobally(ConfigurationType configurationType, const std::string& value) - void setGlobally(ConfigurationType configurationType, const std::string& value, bool includeGlobalLevel); - - /// @brief Sets configurations (Unsafely) for all levels including Level::Global if includeGlobalLevel is true - /// @see Configurations::setGlobally(ConfigurationType configurationType, const std::string& value) - void unsafeSetGlobally(ConfigurationType configurationType, const std::string& value, bool includeGlobalLevel); -}; - -namespace base { -typedef std::shared_ptr FileStreamPtr; -typedef std::unordered_map LogStreamsReferenceMap; -/// @brief Configurations with data types. -/// -/// @detail el::Configurations have string based values. This is whats used internally in order to read correct configurations. -/// This is to perform faster while writing logs using correct configurations. -/// -/// This is thread safe and final class containing non-virtual destructor (means nothing should inherit this class) -class TypedConfigurations : public base::threading::ThreadSafe { - public: - /// @brief Constructor to initialize (construct) the object off el::Configurations - /// @param configurations Configurations pointer/reference to base this typed configurations off. - /// @param logStreamsReference Use ELPP->registeredLoggers()->logStreamsReference() - TypedConfigurations(Configurations* configurations, base::LogStreamsReferenceMap* logStreamsReference); - - TypedConfigurations(const TypedConfigurations& other); - - virtual ~TypedConfigurations(void) { - } - - const Configurations* configurations(void) const { - return m_configurations; - } - - bool enabled(Level level); - bool toFile(Level level); - const std::string& filename(Level level); - bool toStandardOutput(Level level); - const base::LogFormat& logFormat(Level level); - const base::SubsecondPrecision& subsecondPrecision(Level level = Level::Global); - const base::MillisecondsWidth& millisecondsWidth(Level level = Level::Global); - bool performanceTracking(Level level = Level::Global); - base::type::fstream_t* fileStream(Level level); - std::size_t maxLogFileSize(Level level); - std::size_t logFlushThreshold(Level level); - - private: - Configurations* m_configurations; - std::unordered_map m_enabledMap; - std::unordered_map m_toFileMap; - std::unordered_map m_filenameMap; - std::unordered_map m_toStandardOutputMap; - std::unordered_map m_logFormatMap; - std::unordered_map m_subsecondPrecisionMap; - std::unordered_map m_performanceTrackingMap; - std::unordered_map m_fileStreamMap; - std::unordered_map m_maxLogFileSizeMap; - std::unordered_map m_logFlushThresholdMap; - base::LogStreamsReferenceMap* m_logStreamsReference; - - friend class el::Helpers; - friend class el::base::MessageBuilder; - friend class el::base::Writer; - friend class el::base::DefaultLogDispatchCallback; - friend class el::base::LogDispatcher; - - template - inline Conf_T getConfigByVal(Level level, const std::unordered_map* confMap, const char* confName) { - base::threading::ScopedLock scopedLock(lock()); - return unsafeGetConfigByVal(level, confMap, confName); // This is not unsafe anymore - mutex locked in scope - } - - template - inline Conf_T& getConfigByRef(Level level, std::unordered_map* confMap, const char* confName) { - base::threading::ScopedLock scopedLock(lock()); - return unsafeGetConfigByRef(level, confMap, confName); // This is not unsafe anymore - mutex locked in scope - } - - template - Conf_T unsafeGetConfigByVal(Level level, const std::unordered_map* confMap, const char* confName) { - ELPP_UNUSED(confName); - typename std::unordered_map::const_iterator it = confMap->find(level); - if (it == confMap->end()) { - try { - return confMap->at(Level::Global); - } catch (...) { - ELPP_INTERNAL_ERROR("Unable to get configuration [" << confName << "] for level [" - << LevelHelper::convertToString(level) << "]" - << std::endl << "Please ensure you have properly configured logger.", false); - return Conf_T(); - } - } - return it->second; - } - - template - Conf_T& unsafeGetConfigByRef(Level level, std::unordered_map* confMap, const char* confName) { - ELPP_UNUSED(confName); - typename std::unordered_map::iterator it = confMap->find(level); - if (it == confMap->end()) { - try { - return confMap->at(Level::Global); - } catch (...) { - ELPP_INTERNAL_ERROR("Unable to get configuration [" << confName << "] for level [" - << LevelHelper::convertToString(level) << "]" - << std::endl << "Please ensure you have properly configured logger.", false); - } - } - return it->second; - } - - template - void setValue(Level level, const Conf_T& value, std::unordered_map* confMap, - bool includeGlobalLevel = true) { - // If map is empty and we are allowed to add into generic level (Level::Global), do it! - if (confMap->empty() && includeGlobalLevel) { - confMap->insert(std::make_pair(Level::Global, value)); - return; - } - // If same value exist in generic level already, dont add it to explicit level - typename std::unordered_map::iterator it = confMap->find(Level::Global); - if (it != confMap->end() && it->second == value) { - return; - } - // Now make sure we dont double up values if we really need to add it to explicit level - it = confMap->find(level); - if (it == confMap->end()) { - // Value not found for level, add new - confMap->insert(std::make_pair(level, value)); - } else { - // Value found, just update value - confMap->at(level) = value; - } - } - - void build(Configurations* configurations); - unsigned long getULong(std::string confVal); - std::string resolveFilename(const std::string& filename); - void insertFile(Level level, const std::string& fullFilename); - bool unsafeValidateFileRolling(Level level, const PreRollOutCallback& preRollOutCallback); - - inline bool validateFileRolling(Level level, const PreRollOutCallback& preRollOutCallback) { - base::threading::ScopedLock scopedLock(lock()); - return unsafeValidateFileRolling(level, preRollOutCallback); - } -}; -/// @brief Class that keeps record of current line hit for occasional logging -class HitCounter { - public: - HitCounter(void) : - m_filename(""), - m_lineNumber(0), - m_hitCounts(0) { - } - - HitCounter(const char* filename, base::type::LineNumber lineNumber) : - m_filename(filename), - m_lineNumber(lineNumber), - m_hitCounts(0) { - } - - HitCounter(const HitCounter& hitCounter) : - m_filename(hitCounter.m_filename), - m_lineNumber(hitCounter.m_lineNumber), - m_hitCounts(hitCounter.m_hitCounts) { - } - - HitCounter& operator=(const HitCounter& hitCounter) { - if (&hitCounter != this) { - m_filename = hitCounter.m_filename; - m_lineNumber = hitCounter.m_lineNumber; - m_hitCounts = hitCounter.m_hitCounts; - } - return *this; - } - - virtual ~HitCounter(void) { - } - - /// @brief Resets location of current hit counter - inline void resetLocation(const char* filename, base::type::LineNumber lineNumber) { - m_filename = filename; - m_lineNumber = lineNumber; - } - - /// @brief Validates hit counts and resets it if necessary - inline void validateHitCounts(std::size_t n) { - if (m_hitCounts >= base::consts::kMaxLogPerCounter) { - m_hitCounts = (n >= 1 ? base::consts::kMaxLogPerCounter % n : 0); - } - ++m_hitCounts; - } - - inline const char* filename(void) const { - return m_filename; - } - - inline base::type::LineNumber lineNumber(void) const { - return m_lineNumber; - } - - inline std::size_t hitCounts(void) const { - return m_hitCounts; - } - - inline void increment(void) { - ++m_hitCounts; - } - - class Predicate { - public: - Predicate(const char* filename, base::type::LineNumber lineNumber) - : m_filename(filename), - m_lineNumber(lineNumber) { - } - inline bool operator()(const HitCounter* counter) { - return ((counter != nullptr) && - (strcmp(counter->m_filename, m_filename) == 0) && - (counter->m_lineNumber == m_lineNumber)); - } - - private: - const char* m_filename; - base::type::LineNumber m_lineNumber; - }; - - private: - const char* m_filename; - base::type::LineNumber m_lineNumber; - std::size_t m_hitCounts; -}; -/// @brief Repository for hit counters used across the application -class RegisteredHitCounters : public base::utils::RegistryWithPred { - public: - /// @brief Validates counter for every N, i.e, registers new if does not exist otherwise updates original one - /// @return True if validation resulted in triggering hit. Meaning logs should be written everytime true is returned - bool validateEveryN(const char* filename, base::type::LineNumber lineNumber, std::size_t n); - - /// @brief Validates counter for hits >= N, i.e, registers new if does not exist otherwise updates original one - /// @return True if validation resulted in triggering hit. Meaning logs should be written everytime true is returned - bool validateAfterN(const char* filename, base::type::LineNumber lineNumber, std::size_t n); - - /// @brief Validates counter for hits are <= n, i.e, registers new if does not exist otherwise updates original one - /// @return True if validation resulted in triggering hit. Meaning logs should be written everytime true is returned - bool validateNTimes(const char* filename, base::type::LineNumber lineNumber, std::size_t n); - - /// @brief Gets hit counter registered at specified position - inline const base::HitCounter* getCounter(const char* filename, base::type::LineNumber lineNumber) { - base::threading::ScopedLock scopedLock(lock()); - return get(filename, lineNumber); - } -}; -/// @brief Action to be taken for dispatching -enum class DispatchAction : base::type::EnumType { - None = 1, NormalLog = 2, SysLog = 4 -}; -} // namespace base -template -class Callback : protected base::threading::ThreadSafe { - public: - Callback(void) : m_enabled(true) {} - inline bool enabled(void) const { - return m_enabled; - } - inline void setEnabled(bool enabled) { - base::threading::ScopedLock scopedLock(lock()); - m_enabled = enabled; - } - protected: - virtual void handle(const T* handlePtr) = 0; - private: - bool m_enabled; -}; -class LogDispatchData { - public: - LogDispatchData() : m_logMessage(nullptr), m_dispatchAction(base::DispatchAction::None) {} - inline const LogMessage* logMessage(void) const { - return m_logMessage; - } - inline base::DispatchAction dispatchAction(void) const { - return m_dispatchAction; - } - inline void setLogMessage(LogMessage* logMessage) { - m_logMessage = logMessage; - } - inline void setDispatchAction(base::DispatchAction dispatchAction) { - m_dispatchAction = dispatchAction; - } - private: - LogMessage* m_logMessage; - base::DispatchAction m_dispatchAction; - friend class base::LogDispatcher; - -}; -class LogDispatchCallback : public Callback { - protected: - virtual void handle(const LogDispatchData* data); - base::threading::Mutex& fileHandle(const LogDispatchData* data); - private: - friend class base::LogDispatcher; - std::unordered_map> m_fileLocks; - base::threading::Mutex m_fileLocksMapLock; -}; -class PerformanceTrackingCallback : public Callback { - private: - friend class base::PerformanceTracker; -}; -class LoggerRegistrationCallback : public Callback { - private: - friend class base::RegisteredLoggers; -}; -class LogBuilder : base::NoCopy { - public: - LogBuilder() : m_termSupportsColor(base::utils::OS::termSupportsColor()) {} - virtual ~LogBuilder(void) { - ELPP_INTERNAL_INFO(3, "Destroying log builder...") - } - virtual base::type::string_t build(const LogMessage* logMessage, bool appendNewLine) const = 0; - void convertToColoredOutput(base::type::string_t* logLine, Level level); - private: - bool m_termSupportsColor; - friend class el::base::DefaultLogDispatchCallback; -}; -typedef std::shared_ptr LogBuilderPtr; -/// @brief Represents a logger holding ID and configurations we need to write logs -/// -/// @detail This class does not write logs itself instead its used by writer to read configuations from. -class Logger : public base::threading::ThreadSafe, public Loggable { - public: - Logger(const std::string& id, base::LogStreamsReferenceMap* logStreamsReference); - Logger(const std::string& id, const Configurations& configurations, base::LogStreamsReferenceMap* logStreamsReference); - Logger(const Logger& logger); - Logger& operator=(const Logger& logger); - - virtual ~Logger(void) { - base::utils::safeDelete(m_typedConfigurations); - } - - virtual inline void log(el::base::type::ostream_t& os) const { - os << m_id.c_str(); - } - - /// @brief Configures the logger using specified configurations. - void configure(const Configurations& configurations); - - /// @brief Reconfigures logger using existing configurations - void reconfigure(void); - - inline const std::string& id(void) const { - return m_id; - } - - inline const std::string& parentApplicationName(void) const { - return m_parentApplicationName; - } - - inline void setParentApplicationName(const std::string& parentApplicationName) { - m_parentApplicationName = parentApplicationName; - } - - inline Configurations* configurations(void) { - return &m_configurations; - } - - inline base::TypedConfigurations* typedConfigurations(void) { - return m_typedConfigurations; - } - - static bool isValidId(const std::string& id); - - /// @brief Flushes logger to sync all log files for all levels - void flush(void); - - void flush(Level level, base::type::fstream_t* fs); - - inline bool isFlushNeeded(Level level) { - return ++m_unflushedCount.find(level)->second >= m_typedConfigurations->logFlushThreshold(level); - } - - inline LogBuilder* logBuilder(void) const { - return m_logBuilder.get(); - } - - inline void setLogBuilder(const LogBuilderPtr& logBuilder) { - m_logBuilder = logBuilder; - } - - inline bool enabled(Level level) const { - return m_typedConfigurations->enabled(level); - } - -#if ELPP_VARIADIC_TEMPLATES_SUPPORTED -# define LOGGER_LEVEL_WRITERS_SIGNATURES(FUNCTION_NAME)\ -template \ -inline void FUNCTION_NAME(const char*, const T&, const Args&...);\ -template \ -inline void FUNCTION_NAME(const T&); - - template - inline void verbose(int, const char*, const T&, const Args&...); - - template - inline void verbose(int, const T&); - - LOGGER_LEVEL_WRITERS_SIGNATURES(info) - LOGGER_LEVEL_WRITERS_SIGNATURES(debug) - LOGGER_LEVEL_WRITERS_SIGNATURES(warn) - LOGGER_LEVEL_WRITERS_SIGNATURES(error) - LOGGER_LEVEL_WRITERS_SIGNATURES(fatal) - LOGGER_LEVEL_WRITERS_SIGNATURES(trace) -# undef LOGGER_LEVEL_WRITERS_SIGNATURES -#endif // ELPP_VARIADIC_TEMPLATES_SUPPORTED - private: - std::string m_id; - base::TypedConfigurations* m_typedConfigurations; - base::type::stringstream_t m_stream; - std::string m_parentApplicationName; - bool m_isConfigured; - Configurations m_configurations; - std::unordered_map m_unflushedCount; - base::LogStreamsReferenceMap* m_logStreamsReference; - LogBuilderPtr m_logBuilder; - - friend class el::LogMessage; - friend class el::Loggers; - friend class el::Helpers; - friend class el::base::RegisteredLoggers; - friend class el::base::DefaultLogDispatchCallback; - friend class el::base::MessageBuilder; - friend class el::base::Writer; - friend class el::base::PErrorWriter; - friend class el::base::Storage; - friend class el::base::PerformanceTracker; - friend class el::base::LogDispatcher; - - Logger(void); - -#if ELPP_VARIADIC_TEMPLATES_SUPPORTED - template - void log_(Level, int, const char*, const T&, const Args&...); - - template - inline void log_(Level, int, const T&); - - template - void log(Level, const char*, const T&, const Args&...); - - template - inline void log(Level, const T&); -#endif // ELPP_VARIADIC_TEMPLATES_SUPPORTED - - void initUnflushedCount(void); - - inline base::type::stringstream_t& stream(void) { - return m_stream; - } - - void resolveLoggerFormatSpec(void) const; -}; -namespace base { -/// @brief Loggers repository -class RegisteredLoggers : public base::utils::Registry { - public: - explicit RegisteredLoggers(const LogBuilderPtr& defaultLogBuilder); - - virtual ~RegisteredLoggers(void) { - unsafeFlushAll(); - } - - inline void setDefaultConfigurations(const Configurations& configurations) { - base::threading::ScopedLock scopedLock(lock()); - m_defaultConfigurations.setFromBase(const_cast(&configurations)); - } - - inline Configurations* defaultConfigurations(void) { - return &m_defaultConfigurations; - } - - Logger* get(const std::string& id, bool forceCreation = true); - - template - inline bool installLoggerRegistrationCallback(const std::string& id) { - return base::utils::Utils::installCallback(id, - &m_loggerRegistrationCallbacks); - } - - template - inline void uninstallLoggerRegistrationCallback(const std::string& id) { - base::utils::Utils::uninstallCallback(id, &m_loggerRegistrationCallbacks); - } - - template - inline T* loggerRegistrationCallback(const std::string& id) { - return base::utils::Utils::callback(id, &m_loggerRegistrationCallbacks); - } - - bool remove(const std::string& id); - - inline bool has(const std::string& id) { - return get(id, false) != nullptr; - } - - inline void unregister(Logger*& logger) { - base::threading::ScopedLock scopedLock(lock()); - base::utils::Registry::unregister(logger->id()); - } - - inline base::LogStreamsReferenceMap* logStreamsReference(void) { - return &m_logStreamsReference; - } - - inline void flushAll(void) { - base::threading::ScopedLock scopedLock(lock()); - unsafeFlushAll(); - } - - inline void setDefaultLogBuilder(LogBuilderPtr& logBuilderPtr) { - base::threading::ScopedLock scopedLock(lock()); - m_defaultLogBuilder = logBuilderPtr; - } - - private: - LogBuilderPtr m_defaultLogBuilder; - Configurations m_defaultConfigurations; - base::LogStreamsReferenceMap m_logStreamsReference; - std::unordered_map m_loggerRegistrationCallbacks; - friend class el::base::Storage; - - void unsafeFlushAll(void); -}; -/// @brief Represents registries for verbose logging -class VRegistry : base::NoCopy, public base::threading::ThreadSafe { - public: - explicit VRegistry(base::type::VerboseLevel level, base::type::EnumType* pFlags); - - /// @brief Sets verbose level. Accepted range is 0-9 - void setLevel(base::type::VerboseLevel level); - - inline base::type::VerboseLevel level(void) const { - return m_level; - } - - inline void clearModules(void) { - base::threading::ScopedLock scopedLock(lock()); - m_modules.clear(); - } - - void setModules(const char* modules); - - bool allowed(base::type::VerboseLevel vlevel, const char* file); - - inline const std::unordered_map& modules(void) const { - return m_modules; - } - - void setFromArgs(const base::utils::CommandLineArgs* commandLineArgs); - - /// @brief Whether or not vModules enabled - inline bool vModulesEnabled(void) { - return !base::utils::hasFlag(LoggingFlag::DisableVModules, *m_pFlags); - } - - private: - base::type::VerboseLevel m_level; - base::type::EnumType* m_pFlags; - std::unordered_map m_modules; -}; -} // namespace base -class LogMessage { - public: - LogMessage(Level level, const std::string& file, base::type::LineNumber line, const std::string& func, - base::type::VerboseLevel verboseLevel, Logger* logger) : - m_level(level), m_file(file), m_line(line), m_func(func), - m_verboseLevel(verboseLevel), m_logger(logger), m_message(logger->stream().str()) { - } - inline Level level(void) const { - return m_level; - } - inline const std::string& file(void) const { - return m_file; - } - inline base::type::LineNumber line(void) const { - return m_line; - } - inline const std::string& func(void) const { - return m_func; - } - inline base::type::VerboseLevel verboseLevel(void) const { - return m_verboseLevel; - } - inline Logger* logger(void) const { - return m_logger; - } - inline const base::type::string_t& message(void) const { - return m_message; - } - private: - Level m_level; - std::string m_file; - base::type::LineNumber m_line; - std::string m_func; - base::type::VerboseLevel m_verboseLevel; - Logger* m_logger; - base::type::string_t m_message; -}; -namespace base { -#if ELPP_ASYNC_LOGGING -class AsyncLogItem { - public: - explicit AsyncLogItem(const LogMessage& logMessage, const LogDispatchData& data, const base::type::string_t& logLine) - : m_logMessage(logMessage), m_dispatchData(data), m_logLine(logLine) {} - virtual ~AsyncLogItem() {} - inline LogMessage* logMessage(void) { - return &m_logMessage; - } - inline LogDispatchData* data(void) { - return &m_dispatchData; - } - inline base::type::string_t logLine(void) { - return m_logLine; - } - private: - LogMessage m_logMessage; - LogDispatchData m_dispatchData; - base::type::string_t m_logLine; -}; -class AsyncLogQueue : public base::threading::ThreadSafe { - public: - virtual ~AsyncLogQueue() { - ELPP_INTERNAL_INFO(6, "~AsyncLogQueue"); - } - - inline AsyncLogItem next(void) { - base::threading::ScopedLock scopedLock(lock()); - AsyncLogItem result = m_queue.front(); - m_queue.pop(); - return result; - } - - inline void push(const AsyncLogItem& item) { - base::threading::ScopedLock scopedLock(lock()); - m_queue.push(item); - } - inline void pop(void) { - base::threading::ScopedLock scopedLock(lock()); - m_queue.pop(); - } - inline AsyncLogItem front(void) { - base::threading::ScopedLock scopedLock(lock()); - return m_queue.front(); - } - inline bool empty(void) { - base::threading::ScopedLock scopedLock(lock()); - return m_queue.empty(); - } - private: - std::queue m_queue; -}; -class IWorker { - public: - virtual ~IWorker() {} - virtual void start() = 0; -}; -#endif // ELPP_ASYNC_LOGGING -/// @brief Easylogging++ management storage -class Storage : base::NoCopy, public base::threading::ThreadSafe { - public: -#if ELPP_ASYNC_LOGGING - Storage(const LogBuilderPtr& defaultLogBuilder, base::IWorker* asyncDispatchWorker); -#else - explicit Storage(const LogBuilderPtr& defaultLogBuilder); -#endif // ELPP_ASYNC_LOGGING - - virtual ~Storage(void); - - inline bool validateEveryNCounter(const char* filename, base::type::LineNumber lineNumber, std::size_t occasion) { - return hitCounters()->validateEveryN(filename, lineNumber, occasion); - } - - inline bool validateAfterNCounter(const char* filename, base::type::LineNumber lineNumber, std::size_t n) { - return hitCounters()->validateAfterN(filename, lineNumber, n); - } - - inline bool validateNTimesCounter(const char* filename, base::type::LineNumber lineNumber, std::size_t n) { - return hitCounters()->validateNTimes(filename, lineNumber, n); - } - - inline base::RegisteredHitCounters* hitCounters(void) const { - return m_registeredHitCounters; - } - - inline base::RegisteredLoggers* registeredLoggers(void) const { - return m_registeredLoggers; - } - - inline base::VRegistry* vRegistry(void) const { - return m_vRegistry; - } - -#if ELPP_ASYNC_LOGGING - inline base::AsyncLogQueue* asyncLogQueue(void) const { - return m_asyncLogQueue; - } -#endif // ELPP_ASYNC_LOGGING - - inline const base::utils::CommandLineArgs* commandLineArgs(void) const { - return &m_commandLineArgs; - } - - inline void addFlag(LoggingFlag flag) { - base::utils::addFlag(flag, &m_flags); - } - - inline void removeFlag(LoggingFlag flag) { - base::utils::removeFlag(flag, &m_flags); - } - - inline bool hasFlag(LoggingFlag flag) const { - return base::utils::hasFlag(flag, m_flags); - } - - inline base::type::EnumType flags(void) const { - return m_flags; - } - - inline void setFlags(base::type::EnumType flags) { - m_flags = flags; - } - - inline void setPreRollOutCallback(const PreRollOutCallback& callback) { - m_preRollOutCallback = callback; - } - - inline void unsetPreRollOutCallback(void) { - m_preRollOutCallback = base::defaultPreRollOutCallback; - } - - inline PreRollOutCallback& preRollOutCallback(void) { - return m_preRollOutCallback; - } - - bool hasCustomFormatSpecifier(const char* formatSpecifier); - void installCustomFormatSpecifier(const CustomFormatSpecifier& customFormatSpecifier); - bool uninstallCustomFormatSpecifier(const char* formatSpecifier); - - const std::vector* customFormatSpecifiers(void) const { - return &m_customFormatSpecifiers; - } - - base::threading::Mutex& customFormatSpecifiersLock() { - return m_customFormatSpecifiersLock; - } - - inline void setLoggingLevel(Level level) { - m_loggingLevel = level; - } - - template - inline bool installLogDispatchCallback(const std::string& id) { - return base::utils::Utils::installCallback(id, &m_logDispatchCallbacks); - } - - template - inline void uninstallLogDispatchCallback(const std::string& id) { - base::utils::Utils::uninstallCallback(id, &m_logDispatchCallbacks); - } - template - inline T* logDispatchCallback(const std::string& id) { - return base::utils::Utils::callback(id, &m_logDispatchCallbacks); - } - -#if defined(ELPP_FEATURE_ALL) || defined(ELPP_FEATURE_PERFORMANCE_TRACKING) - template - inline bool installPerformanceTrackingCallback(const std::string& id) { - return base::utils::Utils::installCallback(id, - &m_performanceTrackingCallbacks); - } - - template - inline void uninstallPerformanceTrackingCallback(const std::string& id) { - base::utils::Utils::uninstallCallback(id, - &m_performanceTrackingCallbacks); - } - - template - inline T* performanceTrackingCallback(const std::string& id) { - return base::utils::Utils::callback(id, &m_performanceTrackingCallbacks); - } -#endif // defined(ELPP_FEATURE_ALL) || defined(ELPP_FEATURE_PERFORMANCE_TRACKING) - - /// @brief Sets thread name for current thread. Requires std::thread - inline void setThreadName(const std::string& name) { - if (name.empty()) return; - base::threading::ScopedLock scopedLock(m_threadNamesLock); - m_threadNames[base::threading::getCurrentThreadId()] = name; - } - - inline std::string getThreadName(const std::string& threadId) { - base::threading::ScopedLock scopedLock(m_threadNamesLock); - std::unordered_map::const_iterator it = m_threadNames.find(threadId); - if (it == m_threadNames.end()) { - return threadId; - } - return it->second; - } - private: - base::RegisteredHitCounters* m_registeredHitCounters; - base::RegisteredLoggers* m_registeredLoggers; - base::type::EnumType m_flags; - base::VRegistry* m_vRegistry; -#if ELPP_ASYNC_LOGGING - base::AsyncLogQueue* m_asyncLogQueue; - base::IWorker* m_asyncDispatchWorker; -#endif // ELPP_ASYNC_LOGGING - base::utils::CommandLineArgs m_commandLineArgs; - PreRollOutCallback m_preRollOutCallback; - std::unordered_map m_logDispatchCallbacks; - std::unordered_map m_performanceTrackingCallbacks; - std::unordered_map m_threadNames; - std::vector m_customFormatSpecifiers; - base::threading::Mutex m_customFormatSpecifiersLock; - base::threading::Mutex m_threadNamesLock; - Level m_loggingLevel; - - friend class el::Helpers; - friend class el::base::DefaultLogDispatchCallback; - friend class el::LogBuilder; - friend class el::base::MessageBuilder; - friend class el::base::Writer; - friend class el::base::PerformanceTracker; - friend class el::base::LogDispatcher; - - void setApplicationArguments(int argc, char** argv); - - inline void setApplicationArguments(int argc, const char** argv) { - setApplicationArguments(argc, const_cast(argv)); - } -}; -extern ELPP_EXPORT base::type::StoragePointer elStorage; -#define ELPP el::base::elStorage -class DefaultLogDispatchCallback : public LogDispatchCallback { - protected: - void handle(const LogDispatchData* data); - private: - const LogDispatchData* m_data; - void dispatch(base::type::string_t&& logLine); -}; -#if ELPP_ASYNC_LOGGING -class AsyncLogDispatchCallback : public LogDispatchCallback { - protected: - void handle(const LogDispatchData* data); -}; -class AsyncDispatchWorker : public base::IWorker, public base::threading::ThreadSafe { - public: - AsyncDispatchWorker(); - virtual ~AsyncDispatchWorker(); - - bool clean(void); - void emptyQueue(void); - virtual void start(void); - void handle(AsyncLogItem* logItem); - void run(void); - - void setContinueRunning(bool value) { - base::threading::ScopedLock scopedLock(m_continueRunningLock); - m_continueRunning = value; - } - - bool continueRunning(void) const { - return m_continueRunning; - } - private: - std::condition_variable cv; - bool m_continueRunning; - base::threading::Mutex m_continueRunningLock; -}; -#endif // ELPP_ASYNC_LOGGING -} // namespace base -namespace base { -class DefaultLogBuilder : public LogBuilder { - public: - base::type::string_t build(const LogMessage* logMessage, bool appendNewLine) const; -}; -/// @brief Dispatches log messages -class LogDispatcher : base::NoCopy { - public: - LogDispatcher(bool proceed, LogMessage* logMessage, base::DispatchAction dispatchAction) : - m_proceed(proceed), - m_logMessage(logMessage), - m_dispatchAction(std::move(dispatchAction)) { - } - - void dispatch(void); - - private: - bool m_proceed; - LogMessage* m_logMessage; - base::DispatchAction m_dispatchAction; -}; -#if defined(ELPP_STL_LOGGING) -/// @brief Workarounds to write some STL logs -/// -/// @detail There is workaround needed to loop through some stl containers. In order to do that, we need iterable containers -/// of same type and provide iterator interface and pass it on to writeIterator(). -/// Remember, this is passed by value in constructor so that we dont change original containers. -/// This operation is as expensive as Big-O(std::min(class_.size(), base::consts::kMaxLogPerContainer)) -namespace workarounds { -/// @brief Abstract IterableContainer template that provides interface for iterable classes of type T -template -class IterableContainer { - public: - typedef typename Container::iterator iterator; - typedef typename Container::const_iterator const_iterator; - IterableContainer(void) {} - virtual ~IterableContainer(void) {} - iterator begin(void) { - return getContainer().begin(); - } - iterator end(void) { - return getContainer().end(); - } - private: - virtual Container& getContainer(void) = 0; -}; -/// @brief Implements IterableContainer and provides iterable std::priority_queue class -template, typename Comparator = std::less> -class IterablePriorityQueue : public IterableContainer, - public std::priority_queue { - public: - IterablePriorityQueue(std::priority_queue queue_) { - std::size_t count_ = 0; - while (++count_ < base::consts::kMaxLogPerContainer && !queue_.empty()) { - this->push(queue_.top()); - queue_.pop(); - } - } - private: - inline Container& getContainer(void) { - return this->c; - } -}; -/// @brief Implements IterableContainer and provides iterable std::queue class -template> -class IterableQueue : public IterableContainer, public std::queue { - public: - IterableQueue(std::queue queue_) { - std::size_t count_ = 0; - while (++count_ < base::consts::kMaxLogPerContainer && !queue_.empty()) { - this->push(queue_.front()); - queue_.pop(); - } - } - private: - inline Container& getContainer(void) { - return this->c; - } -}; -/// @brief Implements IterableContainer and provides iterable std::stack class -template> -class IterableStack : public IterableContainer, public std::stack { - public: - IterableStack(std::stack stack_) { - std::size_t count_ = 0; - while (++count_ < base::consts::kMaxLogPerContainer && !stack_.empty()) { - this->push(stack_.top()); - stack_.pop(); - } - } - private: - inline Container& getContainer(void) { - return this->c; - } -}; -} // namespace workarounds -#endif // defined(ELPP_STL_LOGGING) -// Log message builder -class MessageBuilder { - public: - MessageBuilder(void) : m_logger(nullptr), m_containerLogSeperator(ELPP_LITERAL("")) {} - void initialize(Logger* logger); - -# define ELPP_SIMPLE_LOG(LOG_TYPE)\ -MessageBuilder& operator<<(LOG_TYPE msg) {\ -m_logger->stream() << msg;\ -if (ELPP->hasFlag(LoggingFlag::AutoSpacing)) {\ -m_logger->stream() << " ";\ -}\ -return *this;\ -} - - inline MessageBuilder& operator<<(const std::string& msg) { - return operator<<(msg.c_str()); - } - ELPP_SIMPLE_LOG(char) - ELPP_SIMPLE_LOG(bool) - ELPP_SIMPLE_LOG(signed short) - ELPP_SIMPLE_LOG(unsigned short) - ELPP_SIMPLE_LOG(signed int) - ELPP_SIMPLE_LOG(unsigned int) - ELPP_SIMPLE_LOG(signed long) - ELPP_SIMPLE_LOG(unsigned long) - ELPP_SIMPLE_LOG(float) - ELPP_SIMPLE_LOG(double) - ELPP_SIMPLE_LOG(char*) - ELPP_SIMPLE_LOG(const char*) - ELPP_SIMPLE_LOG(const void*) - ELPP_SIMPLE_LOG(long double) - inline MessageBuilder& operator<<(const std::wstring& msg) { - return operator<<(msg.c_str()); - } - MessageBuilder& operator<<(const wchar_t* msg); - // ostream manipulators - inline MessageBuilder& operator<<(std::ostream& (*OStreamMani)(std::ostream&)) { - m_logger->stream() << OStreamMani; - return *this; - } -#define ELPP_ITERATOR_CONTAINER_LOG_ONE_ARG(temp) \ -template \ -inline MessageBuilder& operator<<(const temp& template_inst) { \ -return writeIterator(template_inst.begin(), template_inst.end(), template_inst.size()); \ -} -#define ELPP_ITERATOR_CONTAINER_LOG_TWO_ARG(temp) \ -template \ -inline MessageBuilder& operator<<(const temp& template_inst) { \ -return writeIterator(template_inst.begin(), template_inst.end(), template_inst.size()); \ -} -#define ELPP_ITERATOR_CONTAINER_LOG_THREE_ARG(temp) \ -template \ -inline MessageBuilder& operator<<(const temp& template_inst) { \ -return writeIterator(template_inst.begin(), template_inst.end(), template_inst.size()); \ -} -#define ELPP_ITERATOR_CONTAINER_LOG_FOUR_ARG(temp) \ -template \ -inline MessageBuilder& operator<<(const temp& template_inst) { \ -return writeIterator(template_inst.begin(), template_inst.end(), template_inst.size()); \ -} -#define ELPP_ITERATOR_CONTAINER_LOG_FIVE_ARG(temp) \ -template \ -inline MessageBuilder& operator<<(const temp& template_inst) { \ -return writeIterator(template_inst.begin(), template_inst.end(), template_inst.size()); \ -} - -#if defined(ELPP_STL_LOGGING) - ELPP_ITERATOR_CONTAINER_LOG_TWO_ARG(std::vector) - ELPP_ITERATOR_CONTAINER_LOG_TWO_ARG(std::list) - ELPP_ITERATOR_CONTAINER_LOG_TWO_ARG(std::deque) - ELPP_ITERATOR_CONTAINER_LOG_THREE_ARG(std::set) - ELPP_ITERATOR_CONTAINER_LOG_THREE_ARG(std::multiset) - ELPP_ITERATOR_CONTAINER_LOG_FOUR_ARG(std::map) - ELPP_ITERATOR_CONTAINER_LOG_FOUR_ARG(std::multimap) - template - inline MessageBuilder& operator<<(const std::queue& queue_) { - base::workarounds::IterableQueue iterableQueue_ = - static_cast >(queue_); - return writeIterator(iterableQueue_.begin(), iterableQueue_.end(), iterableQueue_.size()); - } - template - inline MessageBuilder& operator<<(const std::stack& stack_) { - base::workarounds::IterableStack iterableStack_ = - static_cast >(stack_); - return writeIterator(iterableStack_.begin(), iterableStack_.end(), iterableStack_.size()); - } - template - inline MessageBuilder& operator<<(const std::priority_queue& priorityQueue_) { - base::workarounds::IterablePriorityQueue iterablePriorityQueue_ = - static_cast >(priorityQueue_); - return writeIterator(iterablePriorityQueue_.begin(), iterablePriorityQueue_.end(), iterablePriorityQueue_.size()); - } - template - MessageBuilder& operator<<(const std::pair& pair_) { - m_logger->stream() << ELPP_LITERAL("("); - operator << (static_cast(pair_.first)); - m_logger->stream() << ELPP_LITERAL(", "); - operator << (static_cast(pair_.second)); - m_logger->stream() << ELPP_LITERAL(")"); - return *this; - } - template - MessageBuilder& operator<<(const std::bitset& bitset_) { - m_logger->stream() << ELPP_LITERAL("["); - operator << (bitset_.to_string()); - m_logger->stream() << ELPP_LITERAL("]"); - return *this; - } -# if defined(ELPP_LOG_STD_ARRAY) - template - inline MessageBuilder& operator<<(const std::array& array) { - return writeIterator(array.begin(), array.end(), array.size()); - } -# endif // defined(ELPP_LOG_STD_ARRAY) -# if defined(ELPP_LOG_UNORDERED_MAP) - ELPP_ITERATOR_CONTAINER_LOG_FIVE_ARG(std::unordered_map) - ELPP_ITERATOR_CONTAINER_LOG_FIVE_ARG(std::unordered_multimap) -# endif // defined(ELPP_LOG_UNORDERED_MAP) -# if defined(ELPP_LOG_UNORDERED_SET) - ELPP_ITERATOR_CONTAINER_LOG_FOUR_ARG(std::unordered_set) - ELPP_ITERATOR_CONTAINER_LOG_FOUR_ARG(std::unordered_multiset) -# endif // defined(ELPP_LOG_UNORDERED_SET) -#endif // defined(ELPP_STL_LOGGING) -#if defined(ELPP_QT_LOGGING) - inline MessageBuilder& operator<<(const QString& msg) { -# if defined(ELPP_UNICODE) - m_logger->stream() << msg.toStdWString(); -# else - m_logger->stream() << msg.toStdString(); -# endif // defined(ELPP_UNICODE) - return *this; - } - inline MessageBuilder& operator<<(const QByteArray& msg) { - return operator << (QString(msg)); - } - inline MessageBuilder& operator<<(const QStringRef& msg) { - return operator<<(msg.toString()); - } - inline MessageBuilder& operator<<(qint64 msg) { -# if defined(ELPP_UNICODE) - m_logger->stream() << QString::number(msg).toStdWString(); -# else - m_logger->stream() << QString::number(msg).toStdString(); -# endif // defined(ELPP_UNICODE) - return *this; - } - inline MessageBuilder& operator<<(quint64 msg) { -# if defined(ELPP_UNICODE) - m_logger->stream() << QString::number(msg).toStdWString(); -# else - m_logger->stream() << QString::number(msg).toStdString(); -# endif // defined(ELPP_UNICODE) - return *this; - } - inline MessageBuilder& operator<<(QChar msg) { - m_logger->stream() << msg.toLatin1(); - return *this; - } - inline MessageBuilder& operator<<(const QLatin1String& msg) { - m_logger->stream() << msg.latin1(); - return *this; - } - ELPP_ITERATOR_CONTAINER_LOG_ONE_ARG(QList) - ELPP_ITERATOR_CONTAINER_LOG_ONE_ARG(QVector) - ELPP_ITERATOR_CONTAINER_LOG_ONE_ARG(QQueue) - ELPP_ITERATOR_CONTAINER_LOG_ONE_ARG(QSet) - ELPP_ITERATOR_CONTAINER_LOG_ONE_ARG(QLinkedList) - ELPP_ITERATOR_CONTAINER_LOG_ONE_ARG(QStack) - template - MessageBuilder& operator<<(const QPair& pair_) { - m_logger->stream() << ELPP_LITERAL("("); - operator << (static_cast(pair_.first)); - m_logger->stream() << ELPP_LITERAL(", "); - operator << (static_cast(pair_.second)); - m_logger->stream() << ELPP_LITERAL(")"); - return *this; - } - template - MessageBuilder& operator<<(const QMap& map_) { - m_logger->stream() << ELPP_LITERAL("["); - QList keys = map_.keys(); - typename QList::const_iterator begin = keys.begin(); - typename QList::const_iterator end = keys.end(); - int max_ = static_cast(base::consts::kMaxLogPerContainer); // to prevent warning - for (int index_ = 0; begin != end && index_ < max_; ++index_, ++begin) { - m_logger->stream() << ELPP_LITERAL("("); - operator << (static_cast(*begin)); - m_logger->stream() << ELPP_LITERAL(", "); - operator << (static_cast(map_.value(*begin))); - m_logger->stream() << ELPP_LITERAL(")"); - m_logger->stream() << ((index_ < keys.size() -1) ? m_containerLogSeperator : ELPP_LITERAL("")); - } - if (begin != end) { - m_logger->stream() << ELPP_LITERAL("..."); - } - m_logger->stream() << ELPP_LITERAL("]"); - return *this; - } - template - inline MessageBuilder& operator<<(const QMultiMap& map_) { - operator << (static_cast>(map_)); - return *this; - } - template - MessageBuilder& operator<<(const QHash& hash_) { - m_logger->stream() << ELPP_LITERAL("["); - QList keys = hash_.keys(); - typename QList::const_iterator begin = keys.begin(); - typename QList::const_iterator end = keys.end(); - int max_ = static_cast(base::consts::kMaxLogPerContainer); // prevent type warning - for (int index_ = 0; begin != end && index_ < max_; ++index_, ++begin) { - m_logger->stream() << ELPP_LITERAL("("); - operator << (static_cast(*begin)); - m_logger->stream() << ELPP_LITERAL(", "); - operator << (static_cast(hash_.value(*begin))); - m_logger->stream() << ELPP_LITERAL(")"); - m_logger->stream() << ((index_ < keys.size() -1) ? m_containerLogSeperator : ELPP_LITERAL("")); - } - if (begin != end) { - m_logger->stream() << ELPP_LITERAL("..."); - } - m_logger->stream() << ELPP_LITERAL("]"); - return *this; - } - template - inline MessageBuilder& operator<<(const QMultiHash& multiHash_) { - operator << (static_cast>(multiHash_)); - return *this; - } -#endif // defined(ELPP_QT_LOGGING) -#if defined(ELPP_BOOST_LOGGING) - ELPP_ITERATOR_CONTAINER_LOG_TWO_ARG(boost::container::vector) - ELPP_ITERATOR_CONTAINER_LOG_TWO_ARG(boost::container::stable_vector) - ELPP_ITERATOR_CONTAINER_LOG_TWO_ARG(boost::container::list) - ELPP_ITERATOR_CONTAINER_LOG_TWO_ARG(boost::container::deque) - ELPP_ITERATOR_CONTAINER_LOG_FOUR_ARG(boost::container::map) - ELPP_ITERATOR_CONTAINER_LOG_FOUR_ARG(boost::container::flat_map) - ELPP_ITERATOR_CONTAINER_LOG_THREE_ARG(boost::container::set) - ELPP_ITERATOR_CONTAINER_LOG_THREE_ARG(boost::container::flat_set) -#endif // defined(ELPP_BOOST_LOGGING) - - /// @brief Macro used internally that can be used externally to make containers easylogging++ friendly - /// - /// @detail This macro expands to write an ostream& operator<< for container. This container is expected to - /// have begin() and end() methods that return respective iterators - /// @param ContainerType Type of container e.g, MyList from WX_DECLARE_LIST(int, MyList); in wxwidgets - /// @param SizeMethod Method used to get size of container. - /// @param ElementInstance Instance of element to be fed out. Insance name is "elem". See WXELPP_ENABLED macro - /// for an example usage -#define MAKE_CONTAINERELPP_FRIENDLY(ContainerType, SizeMethod, ElementInstance) \ -el::base::type::ostream_t& operator<<(el::base::type::ostream_t& ss, const ContainerType& container) {\ -const el::base::type::char_t* sep = ELPP->hasFlag(el::LoggingFlag::NewLineForContainer) ? \ -ELPP_LITERAL("\n ") : ELPP_LITERAL(", ");\ -ContainerType::const_iterator elem = container.begin();\ -ContainerType::const_iterator endElem = container.end();\ -std::size_t size_ = container.SizeMethod; \ -ss << ELPP_LITERAL("[");\ -for (std::size_t i = 0; elem != endElem && i < el::base::consts::kMaxLogPerContainer; ++i, ++elem) { \ -ss << ElementInstance;\ -ss << ((i < size_ - 1) ? sep : ELPP_LITERAL(""));\ -}\ -if (elem != endElem) {\ -ss << ELPP_LITERAL("...");\ -}\ -ss << ELPP_LITERAL("]");\ -return ss;\ -} -#if defined(ELPP_WXWIDGETS_LOGGING) - ELPP_ITERATOR_CONTAINER_LOG_ONE_ARG(wxVector) -# define ELPP_WX_PTR_ENABLED(ContainerType) MAKE_CONTAINERELPP_FRIENDLY(ContainerType, size(), *(*elem)) -# define ELPP_WX_ENABLED(ContainerType) MAKE_CONTAINERELPP_FRIENDLY(ContainerType, size(), (*elem)) -# define ELPP_WX_HASH_MAP_ENABLED(ContainerType) MAKE_CONTAINERELPP_FRIENDLY(ContainerType, size(), \ -ELPP_LITERAL("(") << elem->first << ELPP_LITERAL(", ") << elem->second << ELPP_LITERAL(")") -#else -# define ELPP_WX_PTR_ENABLED(ContainerType) -# define ELPP_WX_ENABLED(ContainerType) -# define ELPP_WX_HASH_MAP_ENABLED(ContainerType) -#endif // defined(ELPP_WXWIDGETS_LOGGING) - // Other classes - template - ELPP_SIMPLE_LOG(const Class&) -#undef ELPP_SIMPLE_LOG -#undef ELPP_ITERATOR_CONTAINER_LOG_ONE_ARG -#undef ELPP_ITERATOR_CONTAINER_LOG_TWO_ARG -#undef ELPP_ITERATOR_CONTAINER_LOG_THREE_ARG -#undef ELPP_ITERATOR_CONTAINER_LOG_FOUR_ARG -#undef ELPP_ITERATOR_CONTAINER_LOG_FIVE_ARG - private: - Logger* m_logger; - const base::type::char_t* m_containerLogSeperator; - - template - MessageBuilder& writeIterator(Iterator begin_, Iterator end_, std::size_t size_) { - m_logger->stream() << ELPP_LITERAL("["); - for (std::size_t i = 0; begin_ != end_ && i < base::consts::kMaxLogPerContainer; ++i, ++begin_) { - operator << (*begin_); - m_logger->stream() << ((i < size_ - 1) ? m_containerLogSeperator : ELPP_LITERAL("")); - } - if (begin_ != end_) { - m_logger->stream() << ELPP_LITERAL("..."); - } - m_logger->stream() << ELPP_LITERAL("]"); - if (ELPP->hasFlag(LoggingFlag::AutoSpacing)) { - m_logger->stream() << " "; - } - return *this; - } -}; -/// @brief Writes nothing - Used when certain log is disabled -class NullWriter : base::NoCopy { - public: - NullWriter(void) {} - - // Null manipulator - inline NullWriter& operator<<(std::ostream& (*)(std::ostream&)) { - return *this; - } - - template - inline NullWriter& operator<<(const T&) { - return *this; - } - - inline operator bool() { - return true; - } -}; -/// @brief Main entry point of each logging -class Writer : base::NoCopy { - public: - Writer(Level level, const char* file, base::type::LineNumber line, - const char* func, base::DispatchAction dispatchAction = base::DispatchAction::NormalLog, - base::type::VerboseLevel verboseLevel = 0) : - m_msg(nullptr), m_level(level), m_file(file), m_line(line), m_func(func), m_verboseLevel(verboseLevel), - m_logger(nullptr), m_proceed(false), m_dispatchAction(dispatchAction) { - } - - Writer(LogMessage* msg, base::DispatchAction dispatchAction = base::DispatchAction::NormalLog) : - m_msg(msg), m_level(msg != nullptr ? msg->level() : Level::Unknown), - m_line(0), m_logger(nullptr), m_proceed(false), m_dispatchAction(dispatchAction) { - } - - virtual ~Writer(void) { - processDispatch(); - } - - template - inline Writer& operator<<(const T& log) { -#if ELPP_LOGGING_ENABLED - if (m_proceed) { - m_messageBuilder << log; - } -#endif // ELPP_LOGGING_ENABLED - return *this; - } - - inline Writer& operator<<(std::ostream& (*log)(std::ostream&)) { -#if ELPP_LOGGING_ENABLED - if (m_proceed) { - m_messageBuilder << log; - } -#endif // ELPP_LOGGING_ENABLED - return *this; - } - - inline operator bool() { - return true; - } - - Writer& construct(Logger* logger, bool needLock = true); - Writer& construct(int count, const char* loggerIds, ...); - protected: - LogMessage* m_msg; - Level m_level; - const char* m_file; - const base::type::LineNumber m_line; - const char* m_func; - base::type::VerboseLevel m_verboseLevel; - Logger* m_logger; - bool m_proceed; - base::MessageBuilder m_messageBuilder; - base::DispatchAction m_dispatchAction; - std::vector m_loggerIds; - friend class el::Helpers; - - void initializeLogger(const std::string& loggerId, bool lookup = true, bool needLock = true); - void processDispatch(); - void triggerDispatch(void); -}; -class PErrorWriter : public base::Writer { - public: - PErrorWriter(Level level, const char* file, base::type::LineNumber line, - const char* func, base::DispatchAction dispatchAction = base::DispatchAction::NormalLog, - base::type::VerboseLevel verboseLevel = 0) : - base::Writer(level, file, line, func, dispatchAction, verboseLevel) { - } - - virtual ~PErrorWriter(void); -}; -} // namespace base -// Logging from Logger class. Why this is here? Because we have Storage and Writer class available -#if ELPP_VARIADIC_TEMPLATES_SUPPORTED -template -void Logger::log_(Level level, int vlevel, const char* s, const T& value, const Args&... args) { - base::MessageBuilder b; - b.initialize(this); - while (*s) { - if (*s == base::consts::kFormatSpecifierChar) { - if (*(s + 1) == base::consts::kFormatSpecifierChar) { - ++s; - } else { - if (*(s + 1) == base::consts::kFormatSpecifierCharValue) { - ++s; - b << value; - log_(level, vlevel, ++s, args...); - return; - } - } - } - b << *s++; - } - ELPP_INTERNAL_ERROR("Too many arguments provided. Unable to handle. Please provide more format specifiers", false); -} -template -void Logger::log_(Level level, int vlevel, const T& log) { - if (level == Level::Verbose) { - if (ELPP->vRegistry()->allowed(vlevel, __FILE__)) { - base::Writer(Level::Verbose, "FILE", 0, "FUNCTION", - base::DispatchAction::NormalLog, vlevel).construct(this, false) << log; - } else { - stream().str(ELPP_LITERAL("")); - releaseLock(); - } - } else { - base::Writer(level, "FILE", 0, "FUNCTION").construct(this, false) << log; - } -} -template -inline void Logger::log(Level level, const char* s, const T& value, const Args&... args) { - acquireLock(); // released in Writer! - log_(level, 0, s, value, args...); -} -template -inline void Logger::log(Level level, const T& log) { - acquireLock(); // released in Writer! - log_(level, 0, log); -} -# if ELPP_VERBOSE_LOG -template -inline void Logger::verbose(int vlevel, const char* s, const T& value, const Args&... args) { - acquireLock(); // released in Writer! - log_(el::Level::Verbose, vlevel, s, value, args...); -} -template -inline void Logger::verbose(int vlevel, const T& log) { - acquireLock(); // released in Writer! - log_(el::Level::Verbose, vlevel, log); -} -# else -template -inline void Logger::verbose(int, const char*, const T&, const Args&...) { - return; -} -template -inline void Logger::verbose(int, const T&) { - return; -} -# endif // ELPP_VERBOSE_LOG -# define LOGGER_LEVEL_WRITERS(FUNCTION_NAME, LOG_LEVEL)\ -template \ -inline void Logger::FUNCTION_NAME(const char* s, const T& value, const Args&... args) {\ -log(LOG_LEVEL, s, value, args...);\ -}\ -template \ -inline void Logger::FUNCTION_NAME(const T& value) {\ -log(LOG_LEVEL, value);\ -} -# define LOGGER_LEVEL_WRITERS_DISABLED(FUNCTION_NAME, LOG_LEVEL)\ -template \ -inline void Logger::FUNCTION_NAME(const char*, const T&, const Args&...) {\ -return;\ -}\ -template \ -inline void Logger::FUNCTION_NAME(const T&) {\ -return;\ -} - -# if ELPP_INFO_LOG -LOGGER_LEVEL_WRITERS(info, Level::Info) -# else -LOGGER_LEVEL_WRITERS_DISABLED(info, Level::Info) -# endif // ELPP_INFO_LOG -# if ELPP_DEBUG_LOG -LOGGER_LEVEL_WRITERS(debug, Level::Debug) -# else -LOGGER_LEVEL_WRITERS_DISABLED(debug, Level::Debug) -# endif // ELPP_DEBUG_LOG -# if ELPP_WARNING_LOG -LOGGER_LEVEL_WRITERS(warn, Level::Warning) -# else -LOGGER_LEVEL_WRITERS_DISABLED(warn, Level::Warning) -# endif // ELPP_WARNING_LOG -# if ELPP_ERROR_LOG -LOGGER_LEVEL_WRITERS(error, Level::Error) -# else -LOGGER_LEVEL_WRITERS_DISABLED(error, Level::Error) -# endif // ELPP_ERROR_LOG -# if ELPP_FATAL_LOG -LOGGER_LEVEL_WRITERS(fatal, Level::Fatal) -# else -LOGGER_LEVEL_WRITERS_DISABLED(fatal, Level::Fatal) -# endif // ELPP_FATAL_LOG -# if ELPP_TRACE_LOG -LOGGER_LEVEL_WRITERS(trace, Level::Trace) -# else -LOGGER_LEVEL_WRITERS_DISABLED(trace, Level::Trace) -# endif // ELPP_TRACE_LOG -# undef LOGGER_LEVEL_WRITERS -# undef LOGGER_LEVEL_WRITERS_DISABLED -#endif // ELPP_VARIADIC_TEMPLATES_SUPPORTED -#if ELPP_COMPILER_MSVC -# define ELPP_VARIADIC_FUNC_MSVC(variadicFunction, variadicArgs) variadicFunction variadicArgs -# define ELPP_VARIADIC_FUNC_MSVC_RUN(variadicFunction, ...) ELPP_VARIADIC_FUNC_MSVC(variadicFunction, (__VA_ARGS__)) -# define el_getVALength(...) ELPP_VARIADIC_FUNC_MSVC_RUN(el_resolveVALength, 0, ## __VA_ARGS__,\ -10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0) -#else -# if ELPP_COMPILER_CLANG -# define el_getVALength(...) el_resolveVALength(0, __VA_ARGS__, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0) -# else -# define el_getVALength(...) el_resolveVALength(0, ## __VA_ARGS__, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0) -# endif // ELPP_COMPILER_CLANG -#endif // ELPP_COMPILER_MSVC -#define el_resolveVALength(_0, _1, _2, _3, _4, _5, _6, _7, _8, _9, _10, N, ...) N -#define ELPP_WRITE_LOG(writer, level, dispatchAction, ...) \ -writer(level, __FILE__, __LINE__, ELPP_FUNC, dispatchAction).construct(el_getVALength(__VA_ARGS__), __VA_ARGS__) -#define ELPP_WRITE_LOG_IF(writer, condition, level, dispatchAction, ...) if (condition) \ -writer(level, __FILE__, __LINE__, ELPP_FUNC, dispatchAction).construct(el_getVALength(__VA_ARGS__), __VA_ARGS__) -#define ELPP_WRITE_LOG_EVERY_N(writer, occasion, level, dispatchAction, ...) \ -ELPP->validateEveryNCounter(__FILE__, __LINE__, occasion) && \ -writer(level, __FILE__, __LINE__, ELPP_FUNC, dispatchAction).construct(el_getVALength(__VA_ARGS__), __VA_ARGS__) -#define ELPP_WRITE_LOG_AFTER_N(writer, n, level, dispatchAction, ...) \ -ELPP->validateAfterNCounter(__FILE__, __LINE__, n) && \ -writer(level, __FILE__, __LINE__, ELPP_FUNC, dispatchAction).construct(el_getVALength(__VA_ARGS__), __VA_ARGS__) -#define ELPP_WRITE_LOG_N_TIMES(writer, n, level, dispatchAction, ...) \ -ELPP->validateNTimesCounter(__FILE__, __LINE__, n) && \ -writer(level, __FILE__, __LINE__, ELPP_FUNC, dispatchAction).construct(el_getVALength(__VA_ARGS__), __VA_ARGS__) -#if defined(ELPP_FEATURE_ALL) || defined(ELPP_FEATURE_PERFORMANCE_TRACKING) -class PerformanceTrackingData { - public: - enum class DataType : base::type::EnumType { - Checkpoint = 1, Complete = 2 - }; - // Do not use constructor, will run into multiple definition error, use init(PerformanceTracker*) - explicit PerformanceTrackingData(DataType dataType) : m_performanceTracker(nullptr), - m_dataType(dataType), m_firstCheckpoint(false), m_file(""), m_line(0), m_func("") {} - inline const std::string* blockName(void) const; - inline const struct timeval* startTime(void) const; - inline const struct timeval* endTime(void) const; - inline const struct timeval* lastCheckpointTime(void) const; - inline const base::PerformanceTracker* performanceTracker(void) const { - return m_performanceTracker; - } - inline PerformanceTrackingData::DataType dataType(void) const { - return m_dataType; - } - inline bool firstCheckpoint(void) const { - return m_firstCheckpoint; - } - inline std::string checkpointId(void) const { - return m_checkpointId; - } - inline const char* file(void) const { - return m_file; - } - inline base::type::LineNumber line(void) const { - return m_line; - } - inline const char* func(void) const { - return m_func; - } - inline const base::type::string_t* formattedTimeTaken() const { - return &m_formattedTimeTaken; - } - inline const std::string& loggerId(void) const; - private: - base::PerformanceTracker* m_performanceTracker; - base::type::string_t m_formattedTimeTaken; - PerformanceTrackingData::DataType m_dataType; - bool m_firstCheckpoint; - std::string m_checkpointId; - const char* m_file; - base::type::LineNumber m_line; - const char* m_func; - inline void init(base::PerformanceTracker* performanceTracker, bool firstCheckpoint = false) { - m_performanceTracker = performanceTracker; - m_firstCheckpoint = firstCheckpoint; - } - - friend class el::base::PerformanceTracker; -}; -namespace base { -/// @brief Represents performanceTracker block of code that conditionally adds performance status to log -/// either when goes outside the scope of when checkpoint() is called -class PerformanceTracker : public base::threading::ThreadSafe, public Loggable { - public: - PerformanceTracker(const std::string& blockName, - base::TimestampUnit timestampUnit = base::TimestampUnit::Millisecond, - const std::string& loggerId = std::string(el::base::consts::kPerformanceLoggerId), - bool scopedLog = true, Level level = base::consts::kPerformanceTrackerDefaultLevel); - /// @brief Copy constructor - PerformanceTracker(const PerformanceTracker& t) : - m_blockName(t.m_blockName), m_timestampUnit(t.m_timestampUnit), m_loggerId(t.m_loggerId), m_scopedLog(t.m_scopedLog), - m_level(t.m_level), m_hasChecked(t.m_hasChecked), m_lastCheckpointId(t.m_lastCheckpointId), m_enabled(t.m_enabled), - m_startTime(t.m_startTime), m_endTime(t.m_endTime), m_lastCheckpointTime(t.m_lastCheckpointTime) { - } - virtual ~PerformanceTracker(void); - /// @brief A checkpoint for current performanceTracker block. - void checkpoint(const std::string& id = std::string(), const char* file = __FILE__, - base::type::LineNumber line = __LINE__, - const char* func = ""); - inline Level level(void) const { - return m_level; - } - private: - std::string m_blockName; - base::TimestampUnit m_timestampUnit; - std::string m_loggerId; - bool m_scopedLog; - Level m_level; - bool m_hasChecked; - std::string m_lastCheckpointId; - bool m_enabled; - struct timeval m_startTime, m_endTime, m_lastCheckpointTime; - - PerformanceTracker(void); - - friend class el::PerformanceTrackingData; - friend class base::DefaultPerformanceTrackingCallback; - - const inline base::type::string_t getFormattedTimeTaken() const { - return getFormattedTimeTaken(m_startTime); - } - - const base::type::string_t getFormattedTimeTaken(struct timeval startTime) const; - - virtual inline void log(el::base::type::ostream_t& os) const { - os << getFormattedTimeTaken(); - } -}; -class DefaultPerformanceTrackingCallback : public PerformanceTrackingCallback { - protected: - void handle(const PerformanceTrackingData* data) { - m_data = data; - base::type::stringstream_t ss; - if (m_data->dataType() == PerformanceTrackingData::DataType::Complete) { - ss << ELPP_LITERAL("Executed [") << m_data->blockName()->c_str() << ELPP_LITERAL("] in [") << - *m_data->formattedTimeTaken() << ELPP_LITERAL("]"); - } else { - ss << ELPP_LITERAL("Performance checkpoint"); - if (!m_data->checkpointId().empty()) { - ss << ELPP_LITERAL(" [") << m_data->checkpointId().c_str() << ELPP_LITERAL("]"); - } - ss << ELPP_LITERAL(" for block [") << m_data->blockName()->c_str() << ELPP_LITERAL("] : [") << - *m_data->performanceTracker(); - if (!ELPP->hasFlag(LoggingFlag::DisablePerformanceTrackingCheckpointComparison) - && m_data->performanceTracker()->m_hasChecked) { - ss << ELPP_LITERAL(" ([") << *m_data->formattedTimeTaken() << ELPP_LITERAL("] from "); - if (m_data->performanceTracker()->m_lastCheckpointId.empty()) { - ss << ELPP_LITERAL("last checkpoint"); - } else { - ss << ELPP_LITERAL("checkpoint '") << m_data->performanceTracker()->m_lastCheckpointId.c_str() << ELPP_LITERAL("'"); - } - ss << ELPP_LITERAL(")]"); - } else { - ss << ELPP_LITERAL("]"); - } - } - el::base::Writer(m_data->performanceTracker()->level(), m_data->file(), m_data->line(), m_data->func()).construct(1, - m_data->loggerId().c_str()) << ss.str(); - } - private: - const PerformanceTrackingData* m_data; -}; -} // namespace base -inline const std::string* PerformanceTrackingData::blockName() const { - return const_cast(&m_performanceTracker->m_blockName); -} -inline const struct timeval* PerformanceTrackingData::startTime() const { - return const_cast(&m_performanceTracker->m_startTime); -} -inline const struct timeval* PerformanceTrackingData::endTime() const { - return const_cast(&m_performanceTracker->m_endTime); -} -inline const struct timeval* PerformanceTrackingData::lastCheckpointTime() const { - return const_cast(&m_performanceTracker->m_lastCheckpointTime); -} -inline const std::string& PerformanceTrackingData::loggerId(void) const { - return m_performanceTracker->m_loggerId; -} -#endif // defined(ELPP_FEATURE_ALL) || defined(ELPP_FEATURE_PERFORMANCE_TRACKING) -namespace base { -/// @brief Contains some internal debugging tools like crash handler and stack tracer -namespace debug { -#if defined(ELPP_FEATURE_ALL) || defined(ELPP_FEATURE_CRASH_LOG) -class StackTrace : base::NoCopy { - public: - static const unsigned int kMaxStack = 64; - static const unsigned int kStackStart = 2; // We want to skip c'tor and StackTrace::generateNew() - class StackTraceEntry { - public: - StackTraceEntry(std::size_t index, const std::string& loc, const std::string& demang, const std::string& hex, - const std::string& addr); - StackTraceEntry(std::size_t index, const std::string& loc) : - m_index(index), - m_location(loc) { - } - std::size_t m_index; - std::string m_location; - std::string m_demangled; - std::string m_hex; - std::string m_addr; - friend std::ostream& operator<<(std::ostream& ss, const StackTraceEntry& si); - - private: - StackTraceEntry(void); - }; - - StackTrace(void) { - generateNew(); - } - - virtual ~StackTrace(void) { - } - - inline std::vector& getLatestStack(void) { - return m_stack; - } - - friend std::ostream& operator<<(std::ostream& os, const StackTrace& st); - - private: - std::vector m_stack; - - void generateNew(void); -}; -/// @brief Handles unexpected crashes -class CrashHandler : base::NoCopy { - public: - typedef void (*Handler)(int); - - explicit CrashHandler(bool useDefault); - explicit CrashHandler(const Handler& cHandler) { - setHandler(cHandler); - } - void setHandler(const Handler& cHandler); - - private: - Handler m_handler; -}; -#else -class CrashHandler { - public: - explicit CrashHandler(bool) {} -}; -#endif // defined(ELPP_FEATURE_ALL) || defined(ELPP_FEATURE_CRASH_LOG) -} // namespace debug -} // namespace base -extern base::debug::CrashHandler elCrashHandler; -#define MAKE_LOGGABLE(ClassType, ClassInstance, OutputStreamInstance) \ -el::base::type::ostream_t& operator<<(el::base::type::ostream_t& OutputStreamInstance, const ClassType& ClassInstance) -/// @brief Initializes syslog with process ID, options and facility. calls closelog() on d'tor -class SysLogInitializer { - public: - SysLogInitializer(const char* processIdent, int options = 0, int facility = 0) { -#if defined(ELPP_SYSLOG) - openlog(processIdent, options, facility); -#else - ELPP_UNUSED(processIdent); - ELPP_UNUSED(options); - ELPP_UNUSED(facility); -#endif // defined(ELPP_SYSLOG) - } - virtual ~SysLogInitializer(void) { -#if defined(ELPP_SYSLOG) - closelog(); -#endif // defined(ELPP_SYSLOG) - } -}; -#define ELPP_INITIALIZE_SYSLOG(id, opt, fac) el::SysLogInitializer elSyslogInit(id, opt, fac) -/// @brief Static helpers for developers -class Helpers : base::StaticClass { - public: - /// @brief Shares logging repository (base::Storage) - static inline void setStorage(base::type::StoragePointer storage) { - ELPP = storage; - } - /// @return Main storage repository - static inline base::type::StoragePointer storage() { - return ELPP; - } - /// @brief Sets application arguments and figures out whats active for logging and whats not. - static inline void setArgs(int argc, char** argv) { - ELPP->setApplicationArguments(argc, argv); - } - /// @copydoc setArgs(int argc, char** argv) - static inline void setArgs(int argc, const char** argv) { - ELPP->setApplicationArguments(argc, const_cast(argv)); - } - /// @brief Sets thread name for current thread. Requires std::thread - static inline void setThreadName(const std::string& name) { - ELPP->setThreadName(name); - } - static inline std::string getThreadName() { - return ELPP->getThreadName(base::threading::getCurrentThreadId()); - } -#if defined(ELPP_FEATURE_ALL) || defined(ELPP_FEATURE_CRASH_LOG) - /// @brief Overrides default crash handler and installs custom handler. - /// @param crashHandler A functor with no return type that takes single int argument. - /// Handler is a typedef with specification: void (*Handler)(int) - static inline void setCrashHandler(const el::base::debug::CrashHandler::Handler& crashHandler) { - el::elCrashHandler.setHandler(crashHandler); - } - /// @brief Abort due to crash with signal in parameter - /// @param sig Crash signal - static void crashAbort(int sig, const char* sourceFile = "", unsigned int long line = 0); - /// @brief Logs reason of crash as per sig - /// @param sig Crash signal - /// @param stackTraceIfAvailable Includes stack trace if available - /// @param level Logging level - /// @param logger Logger to use for logging - static void logCrashReason(int sig, bool stackTraceIfAvailable = false, - Level level = Level::Fatal, const char* logger = base::consts::kDefaultLoggerId); -#endif // defined(ELPP_FEATURE_ALL) || defined(ELPP_FEATURE_CRASH_LOG) - /// @brief Installs pre rollout callback, this callback is triggered when log file is about to be rolled out - /// (can be useful for backing up) - static inline void installPreRollOutCallback(const PreRollOutCallback& callback) { - ELPP->setPreRollOutCallback(callback); - } - /// @brief Uninstalls pre rollout callback - static inline void uninstallPreRollOutCallback(void) { - ELPP->unsetPreRollOutCallback(); - } - /// @brief Installs post log dispatch callback, this callback is triggered when log is dispatched - template - static inline bool installLogDispatchCallback(const std::string& id) { - return ELPP->installLogDispatchCallback(id); - } - /// @brief Uninstalls log dispatch callback - template - static inline void uninstallLogDispatchCallback(const std::string& id) { - ELPP->uninstallLogDispatchCallback(id); - } - template - static inline T* logDispatchCallback(const std::string& id) { - return ELPP->logDispatchCallback(id); - } -#if defined(ELPP_FEATURE_ALL) || defined(ELPP_FEATURE_PERFORMANCE_TRACKING) - /// @brief Installs post performance tracking callback, this callback is triggered when performance tracking is finished - template - static inline bool installPerformanceTrackingCallback(const std::string& id) { - return ELPP->installPerformanceTrackingCallback(id); - } - /// @brief Uninstalls post performance tracking handler - template - static inline void uninstallPerformanceTrackingCallback(const std::string& id) { - ELPP->uninstallPerformanceTrackingCallback(id); - } - template - static inline T* performanceTrackingCallback(const std::string& id) { - return ELPP->performanceTrackingCallback(id); - } -#endif // defined(ELPP_FEATURE_ALL) || defined(ELPP_FEATURE_PERFORMANCE_TRACKING) - /// @brief Converts template to std::string - useful for loggable classes to log containers within log(std::ostream&) const - template - static std::string convertTemplateToStdString(const T& templ) { - el::Logger* logger = - ELPP->registeredLoggers()->get(el::base::consts::kDefaultLoggerId); - if (logger == nullptr) { - return std::string(); - } - base::MessageBuilder b; - b.initialize(logger); - logger->acquireLock(); - b << templ; -#if defined(ELPP_UNICODE) - std::string s = std::string(logger->stream().str().begin(), logger->stream().str().end()); -#else - std::string s = logger->stream().str(); -#endif // defined(ELPP_UNICODE) - logger->stream().str(ELPP_LITERAL("")); - logger->releaseLock(); - return s; - } - /// @brief Returns command line arguments (pointer) provided to easylogging++ - static inline const el::base::utils::CommandLineArgs* commandLineArgs(void) { - return ELPP->commandLineArgs(); - } - /// @brief Reserve space for custom format specifiers for performance - /// @see std::vector::reserve - static inline void reserveCustomFormatSpecifiers(std::size_t size) { - ELPP->m_customFormatSpecifiers.reserve(size); - } - /// @brief Installs user defined format specifier and handler - static inline void installCustomFormatSpecifier(const CustomFormatSpecifier& customFormatSpecifier) { - ELPP->installCustomFormatSpecifier(customFormatSpecifier); - } - /// @brief Uninstalls user defined format specifier and handler - static inline bool uninstallCustomFormatSpecifier(const char* formatSpecifier) { - return ELPP->uninstallCustomFormatSpecifier(formatSpecifier); - } - /// @brief Returns true if custom format specifier is installed - static inline bool hasCustomFormatSpecifier(const char* formatSpecifier) { - return ELPP->hasCustomFormatSpecifier(formatSpecifier); - } - static inline void validateFileRolling(Logger* logger, Level level) { - if (ELPP == nullptr || logger == nullptr) return; - logger->m_typedConfigurations->validateFileRolling(level, ELPP->preRollOutCallback()); - } -}; -/// @brief Static helpers to deal with loggers and their configurations -class Loggers : base::StaticClass { - public: - /// @brief Gets existing or registers new logger - static Logger* getLogger(const std::string& identity, bool registerIfNotAvailable = true); - /// @brief Changes default log builder for future loggers - static void setDefaultLogBuilder(el::LogBuilderPtr& logBuilderPtr); - /// @brief Installs logger registration callback, this callback is triggered when new logger is registered - template - static inline bool installLoggerRegistrationCallback(const std::string& id) { - return ELPP->registeredLoggers()->installLoggerRegistrationCallback(id); - } - /// @brief Uninstalls log dispatch callback - template - static inline void uninstallLoggerRegistrationCallback(const std::string& id) { - ELPP->registeredLoggers()->uninstallLoggerRegistrationCallback(id); - } - template - static inline T* loggerRegistrationCallback(const std::string& id) { - return ELPP->registeredLoggers()->loggerRegistrationCallback(id); - } - /// @brief Unregisters logger - use it only when you know what you are doing, you may unregister - /// loggers initialized / used by third-party libs. - static bool unregisterLogger(const std::string& identity); - /// @brief Whether or not logger with id is registered - static bool hasLogger(const std::string& identity); - /// @brief Reconfigures specified logger with new configurations - static Logger* reconfigureLogger(Logger* logger, const Configurations& configurations); - /// @brief Reconfigures logger with new configurations after looking it up using identity - static Logger* reconfigureLogger(const std::string& identity, const Configurations& configurations); - /// @brief Reconfigures logger's single configuration - static Logger* reconfigureLogger(const std::string& identity, ConfigurationType configurationType, - const std::string& value); - /// @brief Reconfigures all the existing loggers with new configurations - static void reconfigureAllLoggers(const Configurations& configurations); - /// @brief Reconfigures single configuration for all the loggers - static inline void reconfigureAllLoggers(ConfigurationType configurationType, const std::string& value) { - reconfigureAllLoggers(Level::Global, configurationType, value); - } - /// @brief Reconfigures single configuration for all the loggers for specified level - static void reconfigureAllLoggers(Level level, ConfigurationType configurationType, - const std::string& value); - /// @brief Sets default configurations. This configuration is used for future (and conditionally for existing) loggers - static void setDefaultConfigurations(const Configurations& configurations, - bool reconfigureExistingLoggers = false); - /// @brief Returns current default - static const Configurations* defaultConfigurations(void); - /// @brief Returns log stream reference pointer if needed by user - static const base::LogStreamsReferenceMap* logStreamsReference(void); - /// @brief Default typed configuration based on existing defaultConf - static base::TypedConfigurations defaultTypedConfigurations(void); - /// @brief Populates all logger IDs in current repository. - /// @param [out] targetList List of fill up. - static std::vector* populateAllLoggerIds(std::vector* targetList); - /// @brief Sets configurations from global configuration file. - static void configureFromGlobal(const char* globalConfigurationFilePath); - /// @brief Configures loggers using command line arg. Ensure you have already set command line args, - /// @return False if invalid argument or argument with no value provided, true if attempted to configure logger. - /// If true is returned that does not mean it has been configured successfully, it only means that it - /// has attempeted to configure logger using configuration file provided in argument - static bool configureFromArg(const char* argKey); - /// @brief Flushes all loggers for all levels - Be careful if you dont know how many loggers are registered - static void flushAll(void); - /// @brief Adds logging flag used internally. - static inline void addFlag(LoggingFlag flag) { - ELPP->addFlag(flag); - } - /// @brief Removes logging flag used internally. - static inline void removeFlag(LoggingFlag flag) { - ELPP->removeFlag(flag); - } - /// @brief Determines whether or not certain flag is active - static inline bool hasFlag(LoggingFlag flag) { - return ELPP->hasFlag(flag); - } - /// @brief Adds flag and removes it when scope goes out - class ScopedAddFlag { - public: - ScopedAddFlag(LoggingFlag flag) : m_flag(flag) { - Loggers::addFlag(m_flag); - } - ~ScopedAddFlag(void) { - Loggers::removeFlag(m_flag); - } - private: - LoggingFlag m_flag; - }; - /// @brief Removes flag and add it when scope goes out - class ScopedRemoveFlag { - public: - ScopedRemoveFlag(LoggingFlag flag) : m_flag(flag) { - Loggers::removeFlag(m_flag); - } - ~ScopedRemoveFlag(void) { - Loggers::addFlag(m_flag); - } - private: - LoggingFlag m_flag; - }; - /// @brief Sets hierarchy for logging. Needs to enable logging flag (HierarchicalLogging) - static void setLoggingLevel(Level level) { - ELPP->setLoggingLevel(level); - } - /// @brief Sets verbose level on the fly - static void setVerboseLevel(base::type::VerboseLevel level); - /// @brief Gets current verbose level - static base::type::VerboseLevel verboseLevel(void); - /// @brief Sets vmodules as specified (on the fly) - static void setVModules(const char* modules); - /// @brief Clears vmodules - static void clearVModules(void); -}; -class VersionInfo : base::StaticClass { - public: - /// @brief Current version number - static const std::string version(void); - - /// @brief Release date of current version - static const std::string releaseDate(void); -}; -} // namespace el -#undef VLOG_IS_ON -/// @brief Determines whether verbose logging is on for specified level current file. -#define VLOG_IS_ON(verboseLevel) (ELPP->vRegistry()->allowed(verboseLevel, __FILE__)) -#undef TIMED_BLOCK -#undef TIMED_SCOPE -#undef TIMED_SCOPE_IF -#undef TIMED_FUNC -#undef TIMED_FUNC_IF -#undef ELPP_MIN_UNIT -#if defined(ELPP_PERFORMANCE_MICROSECONDS) -# define ELPP_MIN_UNIT el::base::TimestampUnit::Microsecond -#else -# define ELPP_MIN_UNIT el::base::TimestampUnit::Millisecond -#endif // (defined(ELPP_PERFORMANCE_MICROSECONDS)) -/// @brief Performance tracked scope. Performance gets written when goes out of scope using -/// 'performance' logger. -/// -/// @detail Please note in order to check the performance at a certain time you can use obj->checkpoint(); -/// @see el::base::PerformanceTracker -/// @see el::base::PerformanceTracker::checkpoint -// Note: Do not surround this definition with null macro because of obj instance -#define TIMED_SCOPE_IF(obj, blockname, condition) el::base::type::PerformanceTrackerPtr obj( condition ? \ - new el::base::PerformanceTracker(blockname, ELPP_MIN_UNIT) : nullptr ) -#define TIMED_SCOPE(obj, blockname) TIMED_SCOPE_IF(obj, blockname, true) -#define TIMED_BLOCK(obj, blockName) for (struct { int i; el::base::type::PerformanceTrackerPtr timer; } obj = { 0, \ - el::base::type::PerformanceTrackerPtr(new el::base::PerformanceTracker(blockName, ELPP_MIN_UNIT)) }; obj.i < 1; ++obj.i) -/// @brief Performance tracked function. Performance gets written when goes out of scope using -/// 'performance' logger. -/// -/// @detail Please note in order to check the performance at a certain time you can use obj->checkpoint(); -/// @see el::base::PerformanceTracker -/// @see el::base::PerformanceTracker::checkpoint -#define TIMED_FUNC_IF(obj,condition) TIMED_SCOPE_IF(obj, ELPP_FUNC, condition) -#define TIMED_FUNC(obj) TIMED_SCOPE(obj, ELPP_FUNC) -#undef PERFORMANCE_CHECKPOINT -#undef PERFORMANCE_CHECKPOINT_WITH_ID -#define PERFORMANCE_CHECKPOINT(obj) obj->checkpoint(std::string(), __FILE__, __LINE__, ELPP_FUNC) -#define PERFORMANCE_CHECKPOINT_WITH_ID(obj, id) obj->checkpoint(id, __FILE__, __LINE__, ELPP_FUNC) -#undef ELPP_COUNTER -#undef ELPP_COUNTER_POS -/// @brief Gets hit counter for file/line -#define ELPP_COUNTER (ELPP->hitCounters()->getCounter(__FILE__, __LINE__)) -/// @brief Gets hit counter position for file/line, -1 if not registered yet -#define ELPP_COUNTER_POS (ELPP_COUNTER == nullptr ? -1 : ELPP_COUNTER->hitCounts()) -// Undef levels to support LOG(LEVEL) -#undef INFO -#undef WARNING -#undef DEBUG -#undef ERROR -#undef FATAL -#undef TRACE -#undef VERBOSE -// Undef existing -#undef CINFO -#undef CWARNING -#undef CDEBUG -#undef CFATAL -#undef CERROR -#undef CTRACE -#undef CVERBOSE -#undef CINFO_IF -#undef CWARNING_IF -#undef CDEBUG_IF -#undef CERROR_IF -#undef CFATAL_IF -#undef CTRACE_IF -#undef CVERBOSE_IF -#undef CINFO_EVERY_N -#undef CWARNING_EVERY_N -#undef CDEBUG_EVERY_N -#undef CERROR_EVERY_N -#undef CFATAL_EVERY_N -#undef CTRACE_EVERY_N -#undef CVERBOSE_EVERY_N -#undef CINFO_AFTER_N -#undef CWARNING_AFTER_N -#undef CDEBUG_AFTER_N -#undef CERROR_AFTER_N -#undef CFATAL_AFTER_N -#undef CTRACE_AFTER_N -#undef CVERBOSE_AFTER_N -#undef CINFO_N_TIMES -#undef CWARNING_N_TIMES -#undef CDEBUG_N_TIMES -#undef CERROR_N_TIMES -#undef CFATAL_N_TIMES -#undef CTRACE_N_TIMES -#undef CVERBOSE_N_TIMES -// Normal logs -#if ELPP_INFO_LOG -# define CINFO(writer, dispatchAction, ...) ELPP_WRITE_LOG(writer, el::Level::Info, dispatchAction, __VA_ARGS__) -#else -# define CINFO(writer, dispatchAction, ...) el::base::NullWriter() -#endif // ELPP_INFO_LOG -#if ELPP_WARNING_LOG -# define CWARNING(writer, dispatchAction, ...) ELPP_WRITE_LOG(writer, el::Level::Warning, dispatchAction, __VA_ARGS__) -#else -# define CWARNING(writer, dispatchAction, ...) el::base::NullWriter() -#endif // ELPP_WARNING_LOG -#if ELPP_DEBUG_LOG -# define CDEBUG(writer, dispatchAction, ...) ELPP_WRITE_LOG(writer, el::Level::Debug, dispatchAction, __VA_ARGS__) -#else -# define CDEBUG(writer, dispatchAction, ...) el::base::NullWriter() -#endif // ELPP_DEBUG_LOG -#if ELPP_ERROR_LOG -# define CERROR(writer, dispatchAction, ...) ELPP_WRITE_LOG(writer, el::Level::Error, dispatchAction, __VA_ARGS__) -#else -# define CERROR(writer, dispatchAction, ...) el::base::NullWriter() -#endif // ELPP_ERROR_LOG -#if ELPP_FATAL_LOG -# define CFATAL(writer, dispatchAction, ...) ELPP_WRITE_LOG(writer, el::Level::Fatal, dispatchAction, __VA_ARGS__) -#else -# define CFATAL(writer, dispatchAction, ...) el::base::NullWriter() -#endif // ELPP_FATAL_LOG -#if ELPP_TRACE_LOG -# define CTRACE(writer, dispatchAction, ...) ELPP_WRITE_LOG(writer, el::Level::Trace, dispatchAction, __VA_ARGS__) -#else -# define CTRACE(writer, dispatchAction, ...) el::base::NullWriter() -#endif // ELPP_TRACE_LOG -#if ELPP_VERBOSE_LOG -# define CVERBOSE(writer, vlevel, dispatchAction, ...) if (VLOG_IS_ON(vlevel)) writer(\ -el::Level::Verbose, __FILE__, __LINE__, ELPP_FUNC, dispatchAction, vlevel).construct(el_getVALength(__VA_ARGS__), __VA_ARGS__) -#else -# define CVERBOSE(writer, vlevel, dispatchAction, ...) el::base::NullWriter() -#endif // ELPP_VERBOSE_LOG -// Conditional logs -#if ELPP_INFO_LOG -# define CINFO_IF(writer, condition_, dispatchAction, ...) \ -ELPP_WRITE_LOG_IF(writer, (condition_), el::Level::Info, dispatchAction, __VA_ARGS__) -#else -# define CINFO_IF(writer, condition_, dispatchAction, ...) el::base::NullWriter() -#endif // ELPP_INFO_LOG -#if ELPP_WARNING_LOG -# define CWARNING_IF(writer, condition_, dispatchAction, ...)\ -ELPP_WRITE_LOG_IF(writer, (condition_), el::Level::Warning, dispatchAction, __VA_ARGS__) -#else -# define CWARNING_IF(writer, condition_, dispatchAction, ...) el::base::NullWriter() -#endif // ELPP_WARNING_LOG -#if ELPP_DEBUG_LOG -# define CDEBUG_IF(writer, condition_, dispatchAction, ...)\ -ELPP_WRITE_LOG_IF(writer, (condition_), el::Level::Debug, dispatchAction, __VA_ARGS__) -#else -# define CDEBUG_IF(writer, condition_, dispatchAction, ...) el::base::NullWriter() -#endif // ELPP_DEBUG_LOG -#if ELPP_ERROR_LOG -# define CERROR_IF(writer, condition_, dispatchAction, ...)\ -ELPP_WRITE_LOG_IF(writer, (condition_), el::Level::Error, dispatchAction, __VA_ARGS__) -#else -# define CERROR_IF(writer, condition_, dispatchAction, ...) el::base::NullWriter() -#endif // ELPP_ERROR_LOG -#if ELPP_FATAL_LOG -# define CFATAL_IF(writer, condition_, dispatchAction, ...)\ -ELPP_WRITE_LOG_IF(writer, (condition_), el::Level::Fatal, dispatchAction, __VA_ARGS__) -#else -# define CFATAL_IF(writer, condition_, dispatchAction, ...) el::base::NullWriter() -#endif // ELPP_FATAL_LOG -#if ELPP_TRACE_LOG -# define CTRACE_IF(writer, condition_, dispatchAction, ...)\ -ELPP_WRITE_LOG_IF(writer, (condition_), el::Level::Trace, dispatchAction, __VA_ARGS__) -#else -# define CTRACE_IF(writer, condition_, dispatchAction, ...) el::base::NullWriter() -#endif // ELPP_TRACE_LOG -#if ELPP_VERBOSE_LOG -# define CVERBOSE_IF(writer, condition_, vlevel, dispatchAction, ...) if (VLOG_IS_ON(vlevel) && (condition_)) writer( \ -el::Level::Verbose, __FILE__, __LINE__, ELPP_FUNC, dispatchAction, vlevel).construct(el_getVALength(__VA_ARGS__), __VA_ARGS__) -#else -# define CVERBOSE_IF(writer, condition_, vlevel, dispatchAction, ...) el::base::NullWriter() -#endif // ELPP_VERBOSE_LOG -// Occasional logs -#if ELPP_INFO_LOG -# define CINFO_EVERY_N(writer, occasion, dispatchAction, ...)\ -ELPP_WRITE_LOG_EVERY_N(writer, occasion, el::Level::Info, dispatchAction, __VA_ARGS__) -#else -# define CINFO_EVERY_N(writer, occasion, dispatchAction, ...) el::base::NullWriter() -#endif // ELPP_INFO_LOG -#if ELPP_WARNING_LOG -# define CWARNING_EVERY_N(writer, occasion, dispatchAction, ...)\ -ELPP_WRITE_LOG_EVERY_N(writer, occasion, el::Level::Warning, dispatchAction, __VA_ARGS__) -#else -# define CWARNING_EVERY_N(writer, occasion, dispatchAction, ...) el::base::NullWriter() -#endif // ELPP_WARNING_LOG -#if ELPP_DEBUG_LOG -# define CDEBUG_EVERY_N(writer, occasion, dispatchAction, ...)\ -ELPP_WRITE_LOG_EVERY_N(writer, occasion, el::Level::Debug, dispatchAction, __VA_ARGS__) -#else -# define CDEBUG_EVERY_N(writer, occasion, dispatchAction, ...) el::base::NullWriter() -#endif // ELPP_DEBUG_LOG -#if ELPP_ERROR_LOG -# define CERROR_EVERY_N(writer, occasion, dispatchAction, ...)\ -ELPP_WRITE_LOG_EVERY_N(writer, occasion, el::Level::Error, dispatchAction, __VA_ARGS__) -#else -# define CERROR_EVERY_N(writer, occasion, dispatchAction, ...) el::base::NullWriter() -#endif // ELPP_ERROR_LOG -#if ELPP_FATAL_LOG -# define CFATAL_EVERY_N(writer, occasion, dispatchAction, ...)\ -ELPP_WRITE_LOG_EVERY_N(writer, occasion, el::Level::Fatal, dispatchAction, __VA_ARGS__) -#else -# define CFATAL_EVERY_N(writer, occasion, dispatchAction, ...) el::base::NullWriter() -#endif // ELPP_FATAL_LOG -#if ELPP_TRACE_LOG -# define CTRACE_EVERY_N(writer, occasion, dispatchAction, ...)\ -ELPP_WRITE_LOG_EVERY_N(writer, occasion, el::Level::Trace, dispatchAction, __VA_ARGS__) -#else -# define CTRACE_EVERY_N(writer, occasion, dispatchAction, ...) el::base::NullWriter() -#endif // ELPP_TRACE_LOG -#if ELPP_VERBOSE_LOG -# define CVERBOSE_EVERY_N(writer, occasion, vlevel, dispatchAction, ...)\ -CVERBOSE_IF(writer, ELPP->validateEveryNCounter(__FILE__, __LINE__, occasion), vlevel, dispatchAction, __VA_ARGS__) -#else -# define CVERBOSE_EVERY_N(writer, occasion, vlevel, dispatchAction, ...) el::base::NullWriter() -#endif // ELPP_VERBOSE_LOG -// After N logs -#if ELPP_INFO_LOG -# define CINFO_AFTER_N(writer, n, dispatchAction, ...)\ -ELPP_WRITE_LOG_AFTER_N(writer, n, el::Level::Info, dispatchAction, __VA_ARGS__) -#else -# define CINFO_AFTER_N(writer, n, dispatchAction, ...) el::base::NullWriter() -#endif // ELPP_INFO_LOG -#if ELPP_WARNING_LOG -# define CWARNING_AFTER_N(writer, n, dispatchAction, ...)\ -ELPP_WRITE_LOG_AFTER_N(writer, n, el::Level::Warning, dispatchAction, __VA_ARGS__) -#else -# define CWARNING_AFTER_N(writer, n, dispatchAction, ...) el::base::NullWriter() -#endif // ELPP_WARNING_LOG -#if ELPP_DEBUG_LOG -# define CDEBUG_AFTER_N(writer, n, dispatchAction, ...)\ -ELPP_WRITE_LOG_AFTER_N(writer, n, el::Level::Debug, dispatchAction, __VA_ARGS__) -#else -# define CDEBUG_AFTER_N(writer, n, dispatchAction, ...) el::base::NullWriter() -#endif // ELPP_DEBUG_LOG -#if ELPP_ERROR_LOG -# define CERROR_AFTER_N(writer, n, dispatchAction, ...)\ -ELPP_WRITE_LOG_AFTER_N(writer, n, el::Level::Error, dispatchAction, __VA_ARGS__) -#else -# define CERROR_AFTER_N(writer, n, dispatchAction, ...) el::base::NullWriter() -#endif // ELPP_ERROR_LOG -#if ELPP_FATAL_LOG -# define CFATAL_AFTER_N(writer, n, dispatchAction, ...)\ -ELPP_WRITE_LOG_AFTER_N(writer, n, el::Level::Fatal, dispatchAction, __VA_ARGS__) -#else -# define CFATAL_AFTER_N(writer, n, dispatchAction, ...) el::base::NullWriter() -#endif // ELPP_FATAL_LOG -#if ELPP_TRACE_LOG -# define CTRACE_AFTER_N(writer, n, dispatchAction, ...)\ -ELPP_WRITE_LOG_AFTER_N(writer, n, el::Level::Trace, dispatchAction, __VA_ARGS__) -#else -# define CTRACE_AFTER_N(writer, n, dispatchAction, ...) el::base::NullWriter() -#endif // ELPP_TRACE_LOG -#if ELPP_VERBOSE_LOG -# define CVERBOSE_AFTER_N(writer, n, vlevel, dispatchAction, ...)\ -CVERBOSE_IF(writer, ELPP->validateAfterNCounter(__FILE__, __LINE__, n), vlevel, dispatchAction, __VA_ARGS__) -#else -# define CVERBOSE_AFTER_N(writer, n, vlevel, dispatchAction, ...) el::base::NullWriter() -#endif // ELPP_VERBOSE_LOG -// N Times logs -#if ELPP_INFO_LOG -# define CINFO_N_TIMES(writer, n, dispatchAction, ...)\ -ELPP_WRITE_LOG_N_TIMES(writer, n, el::Level::Info, dispatchAction, __VA_ARGS__) -#else -# define CINFO_N_TIMES(writer, n, dispatchAction, ...) el::base::NullWriter() -#endif // ELPP_INFO_LOG -#if ELPP_WARNING_LOG -# define CWARNING_N_TIMES(writer, n, dispatchAction, ...)\ -ELPP_WRITE_LOG_N_TIMES(writer, n, el::Level::Warning, dispatchAction, __VA_ARGS__) -#else -# define CWARNING_N_TIMES(writer, n, dispatchAction, ...) el::base::NullWriter() -#endif // ELPP_WARNING_LOG -#if ELPP_DEBUG_LOG -# define CDEBUG_N_TIMES(writer, n, dispatchAction, ...)\ -ELPP_WRITE_LOG_N_TIMES(writer, n, el::Level::Debug, dispatchAction, __VA_ARGS__) -#else -# define CDEBUG_N_TIMES(writer, n, dispatchAction, ...) el::base::NullWriter() -#endif // ELPP_DEBUG_LOG -#if ELPP_ERROR_LOG -# define CERROR_N_TIMES(writer, n, dispatchAction, ...)\ -ELPP_WRITE_LOG_N_TIMES(writer, n, el::Level::Error, dispatchAction, __VA_ARGS__) -#else -# define CERROR_N_TIMES(writer, n, dispatchAction, ...) el::base::NullWriter() -#endif // ELPP_ERROR_LOG -#if ELPP_FATAL_LOG -# define CFATAL_N_TIMES(writer, n, dispatchAction, ...)\ -ELPP_WRITE_LOG_N_TIMES(writer, n, el::Level::Fatal, dispatchAction, __VA_ARGS__) -#else -# define CFATAL_N_TIMES(writer, n, dispatchAction, ...) el::base::NullWriter() -#endif // ELPP_FATAL_LOG -#if ELPP_TRACE_LOG -# define CTRACE_N_TIMES(writer, n, dispatchAction, ...)\ -ELPP_WRITE_LOG_N_TIMES(writer, n, el::Level::Trace, dispatchAction, __VA_ARGS__) -#else -# define CTRACE_N_TIMES(writer, n, dispatchAction, ...) el::base::NullWriter() -#endif // ELPP_TRACE_LOG -#if ELPP_VERBOSE_LOG -# define CVERBOSE_N_TIMES(writer, n, vlevel, dispatchAction, ...)\ -CVERBOSE_IF(writer, ELPP->validateNTimesCounter(__FILE__, __LINE__, n), vlevel, dispatchAction, __VA_ARGS__) -#else -# define CVERBOSE_N_TIMES(writer, n, vlevel, dispatchAction, ...) el::base::NullWriter() -#endif // ELPP_VERBOSE_LOG -// -// Custom Loggers - Requires (level, dispatchAction, loggerId/s) -// -// undef existing -#undef CLOG -#undef CLOG_VERBOSE -#undef CVLOG -#undef CLOG_IF -#undef CLOG_VERBOSE_IF -#undef CVLOG_IF -#undef CLOG_EVERY_N -#undef CVLOG_EVERY_N -#undef CLOG_AFTER_N -#undef CVLOG_AFTER_N -#undef CLOG_N_TIMES -#undef CVLOG_N_TIMES -// Normal logs -#define CLOG(LEVEL, ...)\ -C##LEVEL(el::base::Writer, el::base::DispatchAction::NormalLog, __VA_ARGS__) -#define CVLOG(vlevel, ...) CVERBOSE(el::base::Writer, vlevel, el::base::DispatchAction::NormalLog, __VA_ARGS__) -// Conditional logs -#define CLOG_IF(condition, LEVEL, ...)\ -C##LEVEL##_IF(el::base::Writer, condition, el::base::DispatchAction::NormalLog, __VA_ARGS__) -#define CVLOG_IF(condition, vlevel, ...)\ -CVERBOSE_IF(el::base::Writer, condition, vlevel, el::base::DispatchAction::NormalLog, __VA_ARGS__) -// Hit counts based logs -#define CLOG_EVERY_N(n, LEVEL, ...)\ -C##LEVEL##_EVERY_N(el::base::Writer, n, el::base::DispatchAction::NormalLog, __VA_ARGS__) -#define CVLOG_EVERY_N(n, vlevel, ...)\ -CVERBOSE_EVERY_N(el::base::Writer, n, vlevel, el::base::DispatchAction::NormalLog, __VA_ARGS__) -#define CLOG_AFTER_N(n, LEVEL, ...)\ -C##LEVEL##_AFTER_N(el::base::Writer, n, el::base::DispatchAction::NormalLog, __VA_ARGS__) -#define CVLOG_AFTER_N(n, vlevel, ...)\ -CVERBOSE_AFTER_N(el::base::Writer, n, vlevel, el::base::DispatchAction::NormalLog, __VA_ARGS__) -#define CLOG_N_TIMES(n, LEVEL, ...)\ -C##LEVEL##_N_TIMES(el::base::Writer, n, el::base::DispatchAction::NormalLog, __VA_ARGS__) -#define CVLOG_N_TIMES(n, vlevel, ...)\ -CVERBOSE_N_TIMES(el::base::Writer, n, vlevel, el::base::DispatchAction::NormalLog, __VA_ARGS__) -// -// Default Loggers macro using CLOG(), CLOG_VERBOSE() and CVLOG() macros -// -// undef existing -#undef LOG -#undef VLOG -#undef LOG_IF -#undef VLOG_IF -#undef LOG_EVERY_N -#undef VLOG_EVERY_N -#undef LOG_AFTER_N -#undef VLOG_AFTER_N -#undef LOG_N_TIMES -#undef VLOG_N_TIMES -#undef ELPP_CURR_FILE_LOGGER_ID -#if defined(ELPP_DEFAULT_LOGGER) -# define ELPP_CURR_FILE_LOGGER_ID ELPP_DEFAULT_LOGGER -#else -# define ELPP_CURR_FILE_LOGGER_ID el::base::consts::kDefaultLoggerId -#endif -#undef ELPP_TRACE -#define ELPP_TRACE CLOG(TRACE, ELPP_CURR_FILE_LOGGER_ID) -// Normal logs -#define LOG(LEVEL) CLOG(LEVEL, ELPP_CURR_FILE_LOGGER_ID) -#define VLOG(vlevel) CVLOG(vlevel, ELPP_CURR_FILE_LOGGER_ID) -// Conditional logs -#define LOG_IF(condition, LEVEL) CLOG_IF(condition, LEVEL, ELPP_CURR_FILE_LOGGER_ID) -#define VLOG_IF(condition, vlevel) CVLOG_IF(condition, vlevel, ELPP_CURR_FILE_LOGGER_ID) -// Hit counts based logs -#define LOG_EVERY_N(n, LEVEL) CLOG_EVERY_N(n, LEVEL, ELPP_CURR_FILE_LOGGER_ID) -#define VLOG_EVERY_N(n, vlevel) CVLOG_EVERY_N(n, vlevel, ELPP_CURR_FILE_LOGGER_ID) -#define LOG_AFTER_N(n, LEVEL) CLOG_AFTER_N(n, LEVEL, ELPP_CURR_FILE_LOGGER_ID) -#define VLOG_AFTER_N(n, vlevel) CVLOG_AFTER_N(n, vlevel, ELPP_CURR_FILE_LOGGER_ID) -#define LOG_N_TIMES(n, LEVEL) CLOG_N_TIMES(n, LEVEL, ELPP_CURR_FILE_LOGGER_ID) -#define VLOG_N_TIMES(n, vlevel) CVLOG_N_TIMES(n, vlevel, ELPP_CURR_FILE_LOGGER_ID) -// Generic PLOG() -#undef CPLOG -#undef CPLOG_IF -#undef PLOG -#undef PLOG_IF -#undef DCPLOG -#undef DCPLOG_IF -#undef DPLOG -#undef DPLOG_IF -#define CPLOG(LEVEL, ...)\ -C##LEVEL(el::base::PErrorWriter, el::base::DispatchAction::NormalLog, __VA_ARGS__) -#define CPLOG_IF(condition, LEVEL, ...)\ -C##LEVEL##_IF(el::base::PErrorWriter, condition, el::base::DispatchAction::NormalLog, __VA_ARGS__) -#define DCPLOG(LEVEL, ...)\ -if (ELPP_DEBUG_LOG) C##LEVEL(el::base::PErrorWriter, el::base::DispatchAction::NormalLog, __VA_ARGS__) -#define DCPLOG_IF(condition, LEVEL, ...)\ -C##LEVEL##_IF(el::base::PErrorWriter, (ELPP_DEBUG_LOG) && (condition), el::base::DispatchAction::NormalLog, __VA_ARGS__) -#define PLOG(LEVEL) CPLOG(LEVEL, ELPP_CURR_FILE_LOGGER_ID) -#define PLOG_IF(condition, LEVEL) CPLOG_IF(condition, LEVEL, ELPP_CURR_FILE_LOGGER_ID) -#define DPLOG(LEVEL) DCPLOG(LEVEL, ELPP_CURR_FILE_LOGGER_ID) -#define DPLOG_IF(condition, LEVEL) DCPLOG_IF(condition, LEVEL, ELPP_CURR_FILE_LOGGER_ID) -// Generic SYSLOG() -#undef CSYSLOG -#undef CSYSLOG_IF -#undef CSYSLOG_EVERY_N -#undef CSYSLOG_AFTER_N -#undef CSYSLOG_N_TIMES -#undef SYSLOG -#undef SYSLOG_IF -#undef SYSLOG_EVERY_N -#undef SYSLOG_AFTER_N -#undef SYSLOG_N_TIMES -#undef DCSYSLOG -#undef DCSYSLOG_IF -#undef DCSYSLOG_EVERY_N -#undef DCSYSLOG_AFTER_N -#undef DCSYSLOG_N_TIMES -#undef DSYSLOG -#undef DSYSLOG_IF -#undef DSYSLOG_EVERY_N -#undef DSYSLOG_AFTER_N -#undef DSYSLOG_N_TIMES -#if defined(ELPP_SYSLOG) -# define CSYSLOG(LEVEL, ...)\ -C##LEVEL(el::base::Writer, el::base::DispatchAction::SysLog, __VA_ARGS__) -# define CSYSLOG_IF(condition, LEVEL, ...)\ -C##LEVEL##_IF(el::base::Writer, condition, el::base::DispatchAction::SysLog, __VA_ARGS__) -# define CSYSLOG_EVERY_N(n, LEVEL, ...) C##LEVEL##_EVERY_N(el::base::Writer, n, el::base::DispatchAction::SysLog, __VA_ARGS__) -# define CSYSLOG_AFTER_N(n, LEVEL, ...) C##LEVEL##_AFTER_N(el::base::Writer, n, el::base::DispatchAction::SysLog, __VA_ARGS__) -# define CSYSLOG_N_TIMES(n, LEVEL, ...) C##LEVEL##_N_TIMES(el::base::Writer, n, el::base::DispatchAction::SysLog, __VA_ARGS__) -# define SYSLOG(LEVEL) CSYSLOG(LEVEL, el::base::consts::kSysLogLoggerId) -# define SYSLOG_IF(condition, LEVEL) CSYSLOG_IF(condition, LEVEL, el::base::consts::kSysLogLoggerId) -# define SYSLOG_EVERY_N(n, LEVEL) CSYSLOG_EVERY_N(n, LEVEL, el::base::consts::kSysLogLoggerId) -# define SYSLOG_AFTER_N(n, LEVEL) CSYSLOG_AFTER_N(n, LEVEL, el::base::consts::kSysLogLoggerId) -# define SYSLOG_N_TIMES(n, LEVEL) CSYSLOG_N_TIMES(n, LEVEL, el::base::consts::kSysLogLoggerId) -# define DCSYSLOG(LEVEL, ...) if (ELPP_DEBUG_LOG) C##LEVEL(el::base::Writer, el::base::DispatchAction::SysLog, __VA_ARGS__) -# define DCSYSLOG_IF(condition, LEVEL, ...)\ -C##LEVEL##_IF(el::base::Writer, (ELPP_DEBUG_LOG) && (condition), el::base::DispatchAction::SysLog, __VA_ARGS__) -# define DCSYSLOG_EVERY_N(n, LEVEL, ...)\ -if (ELPP_DEBUG_LOG) C##LEVEL##_EVERY_N(el::base::Writer, n, el::base::DispatchAction::SysLog, __VA_ARGS__) -# define DCSYSLOG_AFTER_N(n, LEVEL, ...)\ -if (ELPP_DEBUG_LOG) C##LEVEL##_AFTER_N(el::base::Writer, n, el::base::DispatchAction::SysLog, __VA_ARGS__) -# define DCSYSLOG_N_TIMES(n, LEVEL, ...)\ -if (ELPP_DEBUG_LOG) C##LEVEL##_EVERY_N(el::base::Writer, n, el::base::DispatchAction::SysLog, __VA_ARGS__) -# define DSYSLOG(LEVEL) DCSYSLOG(LEVEL, el::base::consts::kSysLogLoggerId) -# define DSYSLOG_IF(condition, LEVEL) DCSYSLOG_IF(condition, LEVEL, el::base::consts::kSysLogLoggerId) -# define DSYSLOG_EVERY_N(n, LEVEL) DCSYSLOG_EVERY_N(n, LEVEL, el::base::consts::kSysLogLoggerId) -# define DSYSLOG_AFTER_N(n, LEVEL) DCSYSLOG_AFTER_N(n, LEVEL, el::base::consts::kSysLogLoggerId) -# define DSYSLOG_N_TIMES(n, LEVEL) DCSYSLOG_N_TIMES(n, LEVEL, el::base::consts::kSysLogLoggerId) -#else -# define CSYSLOG(LEVEL, ...) el::base::NullWriter() -# define CSYSLOG_IF(condition, LEVEL, ...) el::base::NullWriter() -# define CSYSLOG_EVERY_N(n, LEVEL, ...) el::base::NullWriter() -# define CSYSLOG_AFTER_N(n, LEVEL, ...) el::base::NullWriter() -# define CSYSLOG_N_TIMES(n, LEVEL, ...) el::base::NullWriter() -# define SYSLOG(LEVEL) el::base::NullWriter() -# define SYSLOG_IF(condition, LEVEL) el::base::NullWriter() -# define SYSLOG_EVERY_N(n, LEVEL) el::base::NullWriter() -# define SYSLOG_AFTER_N(n, LEVEL) el::base::NullWriter() -# define SYSLOG_N_TIMES(n, LEVEL) el::base::NullWriter() -# define DCSYSLOG(LEVEL, ...) el::base::NullWriter() -# define DCSYSLOG_IF(condition, LEVEL, ...) el::base::NullWriter() -# define DCSYSLOG_EVERY_N(n, LEVEL, ...) el::base::NullWriter() -# define DCSYSLOG_AFTER_N(n, LEVEL, ...) el::base::NullWriter() -# define DCSYSLOG_N_TIMES(n, LEVEL, ...) el::base::NullWriter() -# define DSYSLOG(LEVEL) el::base::NullWriter() -# define DSYSLOG_IF(condition, LEVEL) el::base::NullWriter() -# define DSYSLOG_EVERY_N(n, LEVEL) el::base::NullWriter() -# define DSYSLOG_AFTER_N(n, LEVEL) el::base::NullWriter() -# define DSYSLOG_N_TIMES(n, LEVEL) el::base::NullWriter() -#endif // defined(ELPP_SYSLOG) -// -// Custom Debug Only Loggers - Requires (level, loggerId/s) -// -// undef existing -#undef DCLOG -#undef DCVLOG -#undef DCLOG_IF -#undef DCVLOG_IF -#undef DCLOG_EVERY_N -#undef DCVLOG_EVERY_N -#undef DCLOG_AFTER_N -#undef DCVLOG_AFTER_N -#undef DCLOG_N_TIMES -#undef DCVLOG_N_TIMES -// Normal logs -#define DCLOG(LEVEL, ...) if (ELPP_DEBUG_LOG) CLOG(LEVEL, __VA_ARGS__) -#define DCLOG_VERBOSE(vlevel, ...) if (ELPP_DEBUG_LOG) CLOG_VERBOSE(vlevel, __VA_ARGS__) -#define DCVLOG(vlevel, ...) if (ELPP_DEBUG_LOG) CVLOG(vlevel, __VA_ARGS__) -// Conditional logs -#define DCLOG_IF(condition, LEVEL, ...) if (ELPP_DEBUG_LOG) CLOG_IF(condition, LEVEL, __VA_ARGS__) -#define DCVLOG_IF(condition, vlevel, ...) if (ELPP_DEBUG_LOG) CVLOG_IF(condition, vlevel, __VA_ARGS__) -// Hit counts based logs -#define DCLOG_EVERY_N(n, LEVEL, ...) if (ELPP_DEBUG_LOG) CLOG_EVERY_N(n, LEVEL, __VA_ARGS__) -#define DCVLOG_EVERY_N(n, vlevel, ...) if (ELPP_DEBUG_LOG) CVLOG_EVERY_N(n, vlevel, __VA_ARGS__) -#define DCLOG_AFTER_N(n, LEVEL, ...) if (ELPP_DEBUG_LOG) CLOG_AFTER_N(n, LEVEL, __VA_ARGS__) -#define DCVLOG_AFTER_N(n, vlevel, ...) if (ELPP_DEBUG_LOG) CVLOG_AFTER_N(n, vlevel, __VA_ARGS__) -#define DCLOG_N_TIMES(n, LEVEL, ...) if (ELPP_DEBUG_LOG) CLOG_N_TIMES(n, LEVEL, __VA_ARGS__) -#define DCVLOG_N_TIMES(n, vlevel, ...) if (ELPP_DEBUG_LOG) CVLOG_N_TIMES(n, vlevel, __VA_ARGS__) -// -// Default Debug Only Loggers macro using CLOG(), CLOG_VERBOSE() and CVLOG() macros -// -#if !defined(ELPP_NO_DEBUG_MACROS) -// undef existing -#undef DLOG -#undef DVLOG -#undef DLOG_IF -#undef DVLOG_IF -#undef DLOG_EVERY_N -#undef DVLOG_EVERY_N -#undef DLOG_AFTER_N -#undef DVLOG_AFTER_N -#undef DLOG_N_TIMES -#undef DVLOG_N_TIMES -// Normal logs -#define DLOG(LEVEL) DCLOG(LEVEL, ELPP_CURR_FILE_LOGGER_ID) -#define DVLOG(vlevel) DCVLOG(vlevel, ELPP_CURR_FILE_LOGGER_ID) -// Conditional logs -#define DLOG_IF(condition, LEVEL) DCLOG_IF(condition, LEVEL, ELPP_CURR_FILE_LOGGER_ID) -#define DVLOG_IF(condition, vlevel) DCVLOG_IF(condition, vlevel, ELPP_CURR_FILE_LOGGER_ID) -// Hit counts based logs -#define DLOG_EVERY_N(n, LEVEL) DCLOG_EVERY_N(n, LEVEL, ELPP_CURR_FILE_LOGGER_ID) -#define DVLOG_EVERY_N(n, vlevel) DCVLOG_EVERY_N(n, vlevel, ELPP_CURR_FILE_LOGGER_ID) -#define DLOG_AFTER_N(n, LEVEL) DCLOG_AFTER_N(n, LEVEL, ELPP_CURR_FILE_LOGGER_ID) -#define DVLOG_AFTER_N(n, vlevel) DCVLOG_AFTER_N(n, vlevel, ELPP_CURR_FILE_LOGGER_ID) -#define DLOG_N_TIMES(n, LEVEL) DCLOG_N_TIMES(n, LEVEL, ELPP_CURR_FILE_LOGGER_ID) -#define DVLOG_N_TIMES(n, vlevel) DCVLOG_N_TIMES(n, vlevel, ELPP_CURR_FILE_LOGGER_ID) -#endif // defined(ELPP_NO_DEBUG_MACROS) -#if !defined(ELPP_NO_CHECK_MACROS) -// Check macros -#undef CCHECK -#undef CPCHECK -#undef CCHECK_EQ -#undef CCHECK_NE -#undef CCHECK_LT -#undef CCHECK_GT -#undef CCHECK_LE -#undef CCHECK_GE -#undef CCHECK_BOUNDS -#undef CCHECK_NOTNULL -#undef CCHECK_STRCASEEQ -#undef CCHECK_STRCASENE -#undef CHECK -#undef PCHECK -#undef CHECK_EQ -#undef CHECK_NE -#undef CHECK_LT -#undef CHECK_GT -#undef CHECK_LE -#undef CHECK_GE -#undef CHECK_BOUNDS -#undef CHECK_NOTNULL -#undef CHECK_STRCASEEQ -#undef CHECK_STRCASENE -#define CCHECK(condition, ...) CLOG_IF(!(condition), FATAL, __VA_ARGS__) << "Check failed: [" << #condition << "] " -#define CPCHECK(condition, ...) CPLOG_IF(!(condition), FATAL, __VA_ARGS__) << "Check failed: [" << #condition << "] " -#define CHECK(condition) CCHECK(condition, ELPP_CURR_FILE_LOGGER_ID) -#define PCHECK(condition) CPCHECK(condition, ELPP_CURR_FILE_LOGGER_ID) -#define CCHECK_EQ(a, b, ...) CCHECK(a == b, __VA_ARGS__) -#define CCHECK_NE(a, b, ...) CCHECK(a != b, __VA_ARGS__) -#define CCHECK_LT(a, b, ...) CCHECK(a < b, __VA_ARGS__) -#define CCHECK_GT(a, b, ...) CCHECK(a > b, __VA_ARGS__) -#define CCHECK_LE(a, b, ...) CCHECK(a <= b, __VA_ARGS__) -#define CCHECK_GE(a, b, ...) CCHECK(a >= b, __VA_ARGS__) -#define CCHECK_BOUNDS(val, min, max, ...) CCHECK(val >= min && val <= max, __VA_ARGS__) -#define CHECK_EQ(a, b) CCHECK_EQ(a, b, ELPP_CURR_FILE_LOGGER_ID) -#define CHECK_NE(a, b) CCHECK_NE(a, b, ELPP_CURR_FILE_LOGGER_ID) -#define CHECK_LT(a, b) CCHECK_LT(a, b, ELPP_CURR_FILE_LOGGER_ID) -#define CHECK_GT(a, b) CCHECK_GT(a, b, ELPP_CURR_FILE_LOGGER_ID) -#define CHECK_LE(a, b) CCHECK_LE(a, b, ELPP_CURR_FILE_LOGGER_ID) -#define CHECK_GE(a, b) CCHECK_GE(a, b, ELPP_CURR_FILE_LOGGER_ID) -#define CHECK_BOUNDS(val, min, max) CCHECK_BOUNDS(val, min, max, ELPP_CURR_FILE_LOGGER_ID) -#define CCHECK_NOTNULL(ptr, ...) CCHECK((ptr) != nullptr, __VA_ARGS__) -#define CCHECK_STREQ(str1, str2, ...) CLOG_IF(!el::base::utils::Str::cStringEq(str1, str2), FATAL, __VA_ARGS__) \ -<< "Check failed: [" << #str1 << " == " << #str2 << "] " -#define CCHECK_STRNE(str1, str2, ...) CLOG_IF(el::base::utils::Str::cStringEq(str1, str2), FATAL, __VA_ARGS__) \ -<< "Check failed: [" << #str1 << " != " << #str2 << "] " -#define CCHECK_STRCASEEQ(str1, str2, ...) CLOG_IF(!el::base::utils::Str::cStringCaseEq(str1, str2), FATAL, __VA_ARGS__) \ -<< "Check failed: [" << #str1 << " == " << #str2 << "] " -#define CCHECK_STRCASENE(str1, str2, ...) CLOG_IF(el::base::utils::Str::cStringCaseEq(str1, str2), FATAL, __VA_ARGS__) \ -<< "Check failed: [" << #str1 << " != " << #str2 << "] " -#define CHECK_NOTNULL(ptr) CCHECK_NOTNULL((ptr), ELPP_CURR_FILE_LOGGER_ID) -#define CHECK_STREQ(str1, str2) CCHECK_STREQ(str1, str2, ELPP_CURR_FILE_LOGGER_ID) -#define CHECK_STRNE(str1, str2) CCHECK_STRNE(str1, str2, ELPP_CURR_FILE_LOGGER_ID) -#define CHECK_STRCASEEQ(str1, str2) CCHECK_STRCASEEQ(str1, str2, ELPP_CURR_FILE_LOGGER_ID) -#define CHECK_STRCASENE(str1, str2) CCHECK_STRCASENE(str1, str2, ELPP_CURR_FILE_LOGGER_ID) -#undef DCCHECK -#undef DCCHECK_EQ -#undef DCCHECK_NE -#undef DCCHECK_LT -#undef DCCHECK_GT -#undef DCCHECK_LE -#undef DCCHECK_GE -#undef DCCHECK_BOUNDS -#undef DCCHECK_NOTNULL -#undef DCCHECK_STRCASEEQ -#undef DCCHECK_STRCASENE -#undef DCPCHECK -#undef DCHECK -#undef DCHECK_EQ -#undef DCHECK_NE -#undef DCHECK_LT -#undef DCHECK_GT -#undef DCHECK_LE -#undef DCHECK_GE -#undef DCHECK_BOUNDS_ -#undef DCHECK_NOTNULL -#undef DCHECK_STRCASEEQ -#undef DCHECK_STRCASENE -#undef DPCHECK -#define DCCHECK(condition, ...) if (ELPP_DEBUG_LOG) CCHECK(condition, __VA_ARGS__) -#define DCCHECK_EQ(a, b, ...) if (ELPP_DEBUG_LOG) CCHECK_EQ(a, b, __VA_ARGS__) -#define DCCHECK_NE(a, b, ...) if (ELPP_DEBUG_LOG) CCHECK_NE(a, b, __VA_ARGS__) -#define DCCHECK_LT(a, b, ...) if (ELPP_DEBUG_LOG) CCHECK_LT(a, b, __VA_ARGS__) -#define DCCHECK_GT(a, b, ...) if (ELPP_DEBUG_LOG) CCHECK_GT(a, b, __VA_ARGS__) -#define DCCHECK_LE(a, b, ...) if (ELPP_DEBUG_LOG) CCHECK_LE(a, b, __VA_ARGS__) -#define DCCHECK_GE(a, b, ...) if (ELPP_DEBUG_LOG) CCHECK_GE(a, b, __VA_ARGS__) -#define DCCHECK_BOUNDS(val, min, max, ...) if (ELPP_DEBUG_LOG) CCHECK_BOUNDS(val, min, max, __VA_ARGS__) -#define DCCHECK_NOTNULL(ptr, ...) if (ELPP_DEBUG_LOG) CCHECK_NOTNULL((ptr), __VA_ARGS__) -#define DCCHECK_STREQ(str1, str2, ...) if (ELPP_DEBUG_LOG) CCHECK_STREQ(str1, str2, __VA_ARGS__) -#define DCCHECK_STRNE(str1, str2, ...) if (ELPP_DEBUG_LOG) CCHECK_STRNE(str1, str2, __VA_ARGS__) -#define DCCHECK_STRCASEEQ(str1, str2, ...) if (ELPP_DEBUG_LOG) CCHECK_STRCASEEQ(str1, str2, __VA_ARGS__) -#define DCCHECK_STRCASENE(str1, str2, ...) if (ELPP_DEBUG_LOG) CCHECK_STRCASENE(str1, str2, __VA_ARGS__) -#define DCPCHECK(condition, ...) if (ELPP_DEBUG_LOG) CPCHECK(condition, __VA_ARGS__) -#define DCHECK(condition) DCCHECK(condition, ELPP_CURR_FILE_LOGGER_ID) -#define DCHECK_EQ(a, b) DCCHECK_EQ(a, b, ELPP_CURR_FILE_LOGGER_ID) -#define DCHECK_NE(a, b) DCCHECK_NE(a, b, ELPP_CURR_FILE_LOGGER_ID) -#define DCHECK_LT(a, b) DCCHECK_LT(a, b, ELPP_CURR_FILE_LOGGER_ID) -#define DCHECK_GT(a, b) DCCHECK_GT(a, b, ELPP_CURR_FILE_LOGGER_ID) -#define DCHECK_LE(a, b) DCCHECK_LE(a, b, ELPP_CURR_FILE_LOGGER_ID) -#define DCHECK_GE(a, b) DCCHECK_GE(a, b, ELPP_CURR_FILE_LOGGER_ID) -#define DCHECK_BOUNDS(val, min, max) DCCHECK_BOUNDS(val, min, max, ELPP_CURR_FILE_LOGGER_ID) -#define DCHECK_NOTNULL(ptr) DCCHECK_NOTNULL((ptr), ELPP_CURR_FILE_LOGGER_ID) -#define DCHECK_STREQ(str1, str2) DCCHECK_STREQ(str1, str2, ELPP_CURR_FILE_LOGGER_ID) -#define DCHECK_STRNE(str1, str2) DCCHECK_STRNE(str1, str2, ELPP_CURR_FILE_LOGGER_ID) -#define DCHECK_STRCASEEQ(str1, str2) DCCHECK_STRCASEEQ(str1, str2, ELPP_CURR_FILE_LOGGER_ID) -#define DCHECK_STRCASENE(str1, str2) DCCHECK_STRCASENE(str1, str2, ELPP_CURR_FILE_LOGGER_ID) -#define DPCHECK(condition) DCPCHECK(condition, ELPP_CURR_FILE_LOGGER_ID) -#endif // defined(ELPP_NO_CHECK_MACROS) -#if defined(ELPP_DISABLE_DEFAULT_CRASH_HANDLING) -# define ELPP_USE_DEF_CRASH_HANDLER false -#else -# define ELPP_USE_DEF_CRASH_HANDLER true -#endif // defined(ELPP_DISABLE_DEFAULT_CRASH_HANDLING) -#define ELPP_CRASH_HANDLER_INIT -#define ELPP_INIT_EASYLOGGINGPP(val) \ -namespace el { \ -namespace base { \ -el::base::type::StoragePointer elStorage(val); \ -} \ -el::base::debug::CrashHandler elCrashHandler(ELPP_USE_DEF_CRASH_HANDLER); \ -} - -#if ELPP_ASYNC_LOGGING -# define INITIALIZE_EASYLOGGINGPP ELPP_INIT_EASYLOGGINGPP(new el::base::Storage(el::LogBuilderPtr(new el::base::DefaultLogBuilder()),\ -new el::base::AsyncDispatchWorker())) -#else -# define INITIALIZE_EASYLOGGINGPP ELPP_INIT_EASYLOGGINGPP(new el::base::Storage(el::LogBuilderPtr(new el::base::DefaultLogBuilder()))) -#endif // ELPP_ASYNC_LOGGING -#define INITIALIZE_NULL_EASYLOGGINGPP \ -namespace el {\ -namespace base {\ -el::base::type::StoragePointer elStorage;\ -}\ -el::base::debug::CrashHandler elCrashHandler(ELPP_USE_DEF_CRASH_HANDLER);\ -} -#define SHARE_EASYLOGGINGPP(initializedStorage)\ -namespace el {\ -namespace base {\ -el::base::type::StoragePointer elStorage(initializedStorage);\ -}\ -el::base::debug::CrashHandler elCrashHandler(ELPP_USE_DEF_CRASH_HANDLER);\ -} - -#if defined(ELPP_UNICODE) -# define START_EASYLOGGINGPP(argc, argv) el::Helpers::setArgs(argc, argv); std::locale::global(std::locale("")) -#else -# define START_EASYLOGGINGPP(argc, argv) el::Helpers::setArgs(argc, argv) -#endif // defined(ELPP_UNICODE) -#endif // EASYLOGGINGPP_H diff --git a/core/src/grpc/cpp_gen.sh b/core/src/grpc/cpp_gen.sh old mode 100644 new mode 100755 diff --git a/core/src/grpc/gen-milvus/milvus.grpc.pb.cc b/core/src/grpc/gen-milvus/milvus.grpc.pb.cc index 9fea388455..2bf8e321f8 100644 --- a/core/src/grpc/gen-milvus/milvus.grpc.pb.cc +++ b/core/src/grpc/gen-milvus/milvus.grpc.pb.cc @@ -22,19 +22,22 @@ namespace grpc { static const char* MilvusService_method_names[] = { "/milvus.grpc.MilvusService/CreateTable", "/milvus.grpc.MilvusService/HasTable", - "/milvus.grpc.MilvusService/DropTable", - "/milvus.grpc.MilvusService/CreateIndex", - "/milvus.grpc.MilvusService/Insert", - "/milvus.grpc.MilvusService/Search", - "/milvus.grpc.MilvusService/SearchInFiles", "/milvus.grpc.MilvusService/DescribeTable", "/milvus.grpc.MilvusService/CountTable", "/milvus.grpc.MilvusService/ShowTables", - "/milvus.grpc.MilvusService/Cmd", - "/milvus.grpc.MilvusService/DeleteByRange", - "/milvus.grpc.MilvusService/PreloadTable", + "/milvus.grpc.MilvusService/DropTable", + "/milvus.grpc.MilvusService/CreateIndex", "/milvus.grpc.MilvusService/DescribeIndex", "/milvus.grpc.MilvusService/DropIndex", + "/milvus.grpc.MilvusService/CreatePartition", + "/milvus.grpc.MilvusService/ShowPartitions", + "/milvus.grpc.MilvusService/DropPartition", + "/milvus.grpc.MilvusService/Insert", + "/milvus.grpc.MilvusService/Search", + "/milvus.grpc.MilvusService/SearchInFiles", + "/milvus.grpc.MilvusService/Cmd", + "/milvus.grpc.MilvusService/DeleteByDate", + "/milvus.grpc.MilvusService/PreloadTable", }; std::unique_ptr< MilvusService::Stub> MilvusService::NewStub(const std::shared_ptr< ::grpc::ChannelInterface>& channel, const ::grpc::StubOptions& options) { @@ -46,19 +49,22 @@ std::unique_ptr< MilvusService::Stub> MilvusService::NewStub(const std::shared_p MilvusService::Stub::Stub(const std::shared_ptr< ::grpc::ChannelInterface>& channel) : channel_(channel), rpcmethod_CreateTable_(MilvusService_method_names[0], ::grpc::internal::RpcMethod::NORMAL_RPC, channel) , rpcmethod_HasTable_(MilvusService_method_names[1], ::grpc::internal::RpcMethod::NORMAL_RPC, channel) - , rpcmethod_DropTable_(MilvusService_method_names[2], ::grpc::internal::RpcMethod::NORMAL_RPC, channel) - , rpcmethod_CreateIndex_(MilvusService_method_names[3], ::grpc::internal::RpcMethod::NORMAL_RPC, channel) - , rpcmethod_Insert_(MilvusService_method_names[4], ::grpc::internal::RpcMethod::NORMAL_RPC, channel) - , rpcmethod_Search_(MilvusService_method_names[5], ::grpc::internal::RpcMethod::NORMAL_RPC, channel) - , rpcmethod_SearchInFiles_(MilvusService_method_names[6], ::grpc::internal::RpcMethod::NORMAL_RPC, channel) - , rpcmethod_DescribeTable_(MilvusService_method_names[7], ::grpc::internal::RpcMethod::NORMAL_RPC, channel) - , rpcmethod_CountTable_(MilvusService_method_names[8], ::grpc::internal::RpcMethod::NORMAL_RPC, channel) - , rpcmethod_ShowTables_(MilvusService_method_names[9], ::grpc::internal::RpcMethod::NORMAL_RPC, channel) - , rpcmethod_Cmd_(MilvusService_method_names[10], ::grpc::internal::RpcMethod::NORMAL_RPC, channel) - , rpcmethod_DeleteByRange_(MilvusService_method_names[11], ::grpc::internal::RpcMethod::NORMAL_RPC, channel) - , rpcmethod_PreloadTable_(MilvusService_method_names[12], ::grpc::internal::RpcMethod::NORMAL_RPC, channel) - , rpcmethod_DescribeIndex_(MilvusService_method_names[13], ::grpc::internal::RpcMethod::NORMAL_RPC, channel) - , rpcmethod_DropIndex_(MilvusService_method_names[14], ::grpc::internal::RpcMethod::NORMAL_RPC, channel) + , rpcmethod_DescribeTable_(MilvusService_method_names[2], ::grpc::internal::RpcMethod::NORMAL_RPC, channel) + , rpcmethod_CountTable_(MilvusService_method_names[3], ::grpc::internal::RpcMethod::NORMAL_RPC, channel) + , rpcmethod_ShowTables_(MilvusService_method_names[4], ::grpc::internal::RpcMethod::NORMAL_RPC, channel) + , rpcmethod_DropTable_(MilvusService_method_names[5], ::grpc::internal::RpcMethod::NORMAL_RPC, channel) + , rpcmethod_CreateIndex_(MilvusService_method_names[6], ::grpc::internal::RpcMethod::NORMAL_RPC, channel) + , rpcmethod_DescribeIndex_(MilvusService_method_names[7], ::grpc::internal::RpcMethod::NORMAL_RPC, channel) + , rpcmethod_DropIndex_(MilvusService_method_names[8], ::grpc::internal::RpcMethod::NORMAL_RPC, channel) + , rpcmethod_CreatePartition_(MilvusService_method_names[9], ::grpc::internal::RpcMethod::NORMAL_RPC, channel) + , rpcmethod_ShowPartitions_(MilvusService_method_names[10], ::grpc::internal::RpcMethod::NORMAL_RPC, channel) + , rpcmethod_DropPartition_(MilvusService_method_names[11], ::grpc::internal::RpcMethod::NORMAL_RPC, channel) + , rpcmethod_Insert_(MilvusService_method_names[12], ::grpc::internal::RpcMethod::NORMAL_RPC, channel) + , rpcmethod_Search_(MilvusService_method_names[13], ::grpc::internal::RpcMethod::NORMAL_RPC, channel) + , rpcmethod_SearchInFiles_(MilvusService_method_names[14], ::grpc::internal::RpcMethod::NORMAL_RPC, channel) + , rpcmethod_Cmd_(MilvusService_method_names[15], ::grpc::internal::RpcMethod::NORMAL_RPC, channel) + , rpcmethod_DeleteByDate_(MilvusService_method_names[16], ::grpc::internal::RpcMethod::NORMAL_RPC, channel) + , rpcmethod_PreloadTable_(MilvusService_method_names[17], ::grpc::internal::RpcMethod::NORMAL_RPC, channel) {} ::grpc::Status MilvusService::Stub::CreateTable(::grpc::ClientContext* context, const ::milvus::grpc::TableSchema& request, ::milvus::grpc::Status* response) { @@ -117,146 +123,6 @@ void MilvusService::Stub::experimental_async::HasTable(::grpc::ClientContext* co return ::grpc_impl::internal::ClientAsyncResponseReaderFactory< ::milvus::grpc::BoolReply>::Create(channel_.get(), cq, rpcmethod_HasTable_, context, request, false); } -::grpc::Status MilvusService::Stub::DropTable(::grpc::ClientContext* context, const ::milvus::grpc::TableName& request, ::milvus::grpc::Status* response) { - return ::grpc::internal::BlockingUnaryCall(channel_.get(), rpcmethod_DropTable_, context, request, response); -} - -void MilvusService::Stub::experimental_async::DropTable(::grpc::ClientContext* context, const ::milvus::grpc::TableName* request, ::milvus::grpc::Status* response, std::function f) { - ::grpc_impl::internal::CallbackUnaryCall(stub_->channel_.get(), stub_->rpcmethod_DropTable_, context, request, response, std::move(f)); -} - -void MilvusService::Stub::experimental_async::DropTable(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::Status* response, std::function f) { - ::grpc_impl::internal::CallbackUnaryCall(stub_->channel_.get(), stub_->rpcmethod_DropTable_, context, request, response, std::move(f)); -} - -void MilvusService::Stub::experimental_async::DropTable(::grpc::ClientContext* context, const ::milvus::grpc::TableName* request, ::milvus::grpc::Status* response, ::grpc::experimental::ClientUnaryReactor* reactor) { - ::grpc_impl::internal::ClientCallbackUnaryFactory::Create(stub_->channel_.get(), stub_->rpcmethod_DropTable_, context, request, response, reactor); -} - -void MilvusService::Stub::experimental_async::DropTable(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::Status* response, ::grpc::experimental::ClientUnaryReactor* reactor) { - ::grpc_impl::internal::ClientCallbackUnaryFactory::Create(stub_->channel_.get(), stub_->rpcmethod_DropTable_, context, request, response, reactor); -} - -::grpc::ClientAsyncResponseReader< ::milvus::grpc::Status>* MilvusService::Stub::AsyncDropTableRaw(::grpc::ClientContext* context, const ::milvus::grpc::TableName& request, ::grpc::CompletionQueue* cq) { - return ::grpc_impl::internal::ClientAsyncResponseReaderFactory< ::milvus::grpc::Status>::Create(channel_.get(), cq, rpcmethod_DropTable_, context, request, true); -} - -::grpc::ClientAsyncResponseReader< ::milvus::grpc::Status>* MilvusService::Stub::PrepareAsyncDropTableRaw(::grpc::ClientContext* context, const ::milvus::grpc::TableName& request, ::grpc::CompletionQueue* cq) { - return ::grpc_impl::internal::ClientAsyncResponseReaderFactory< ::milvus::grpc::Status>::Create(channel_.get(), cq, rpcmethod_DropTable_, context, request, false); -} - -::grpc::Status MilvusService::Stub::CreateIndex(::grpc::ClientContext* context, const ::milvus::grpc::IndexParam& request, ::milvus::grpc::Status* response) { - return ::grpc::internal::BlockingUnaryCall(channel_.get(), rpcmethod_CreateIndex_, context, request, response); -} - -void MilvusService::Stub::experimental_async::CreateIndex(::grpc::ClientContext* context, const ::milvus::grpc::IndexParam* request, ::milvus::grpc::Status* response, std::function f) { - ::grpc_impl::internal::CallbackUnaryCall(stub_->channel_.get(), stub_->rpcmethod_CreateIndex_, context, request, response, std::move(f)); -} - -void MilvusService::Stub::experimental_async::CreateIndex(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::Status* response, std::function f) { - ::grpc_impl::internal::CallbackUnaryCall(stub_->channel_.get(), stub_->rpcmethod_CreateIndex_, context, request, response, std::move(f)); -} - -void MilvusService::Stub::experimental_async::CreateIndex(::grpc::ClientContext* context, const ::milvus::grpc::IndexParam* request, ::milvus::grpc::Status* response, ::grpc::experimental::ClientUnaryReactor* reactor) { - ::grpc_impl::internal::ClientCallbackUnaryFactory::Create(stub_->channel_.get(), stub_->rpcmethod_CreateIndex_, context, request, response, reactor); -} - -void MilvusService::Stub::experimental_async::CreateIndex(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::Status* response, ::grpc::experimental::ClientUnaryReactor* reactor) { - ::grpc_impl::internal::ClientCallbackUnaryFactory::Create(stub_->channel_.get(), stub_->rpcmethod_CreateIndex_, context, request, response, reactor); -} - -::grpc::ClientAsyncResponseReader< ::milvus::grpc::Status>* MilvusService::Stub::AsyncCreateIndexRaw(::grpc::ClientContext* context, const ::milvus::grpc::IndexParam& request, ::grpc::CompletionQueue* cq) { - return ::grpc_impl::internal::ClientAsyncResponseReaderFactory< ::milvus::grpc::Status>::Create(channel_.get(), cq, rpcmethod_CreateIndex_, context, request, true); -} - -::grpc::ClientAsyncResponseReader< ::milvus::grpc::Status>* MilvusService::Stub::PrepareAsyncCreateIndexRaw(::grpc::ClientContext* context, const ::milvus::grpc::IndexParam& request, ::grpc::CompletionQueue* cq) { - return ::grpc_impl::internal::ClientAsyncResponseReaderFactory< ::milvus::grpc::Status>::Create(channel_.get(), cq, rpcmethod_CreateIndex_, context, request, false); -} - -::grpc::Status MilvusService::Stub::Insert(::grpc::ClientContext* context, const ::milvus::grpc::InsertParam& request, ::milvus::grpc::VectorIds* response) { - return ::grpc::internal::BlockingUnaryCall(channel_.get(), rpcmethod_Insert_, context, request, response); -} - -void MilvusService::Stub::experimental_async::Insert(::grpc::ClientContext* context, const ::milvus::grpc::InsertParam* request, ::milvus::grpc::VectorIds* response, std::function f) { - ::grpc_impl::internal::CallbackUnaryCall(stub_->channel_.get(), stub_->rpcmethod_Insert_, context, request, response, std::move(f)); -} - -void MilvusService::Stub::experimental_async::Insert(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::VectorIds* response, std::function f) { - ::grpc_impl::internal::CallbackUnaryCall(stub_->channel_.get(), stub_->rpcmethod_Insert_, context, request, response, std::move(f)); -} - -void MilvusService::Stub::experimental_async::Insert(::grpc::ClientContext* context, const ::milvus::grpc::InsertParam* request, ::milvus::grpc::VectorIds* response, ::grpc::experimental::ClientUnaryReactor* reactor) { - ::grpc_impl::internal::ClientCallbackUnaryFactory::Create(stub_->channel_.get(), stub_->rpcmethod_Insert_, context, request, response, reactor); -} - -void MilvusService::Stub::experimental_async::Insert(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::VectorIds* response, ::grpc::experimental::ClientUnaryReactor* reactor) { - ::grpc_impl::internal::ClientCallbackUnaryFactory::Create(stub_->channel_.get(), stub_->rpcmethod_Insert_, context, request, response, reactor); -} - -::grpc::ClientAsyncResponseReader< ::milvus::grpc::VectorIds>* MilvusService::Stub::AsyncInsertRaw(::grpc::ClientContext* context, const ::milvus::grpc::InsertParam& request, ::grpc::CompletionQueue* cq) { - return ::grpc_impl::internal::ClientAsyncResponseReaderFactory< ::milvus::grpc::VectorIds>::Create(channel_.get(), cq, rpcmethod_Insert_, context, request, true); -} - -::grpc::ClientAsyncResponseReader< ::milvus::grpc::VectorIds>* MilvusService::Stub::PrepareAsyncInsertRaw(::grpc::ClientContext* context, const ::milvus::grpc::InsertParam& request, ::grpc::CompletionQueue* cq) { - return ::grpc_impl::internal::ClientAsyncResponseReaderFactory< ::milvus::grpc::VectorIds>::Create(channel_.get(), cq, rpcmethod_Insert_, context, request, false); -} - -::grpc::Status MilvusService::Stub::Search(::grpc::ClientContext* context, const ::milvus::grpc::SearchParam& request, ::milvus::grpc::TopKQueryResult* response) { - return ::grpc::internal::BlockingUnaryCall(channel_.get(), rpcmethod_Search_, context, request, response); -} - -void MilvusService::Stub::experimental_async::Search(::grpc::ClientContext* context, const ::milvus::grpc::SearchParam* request, ::milvus::grpc::TopKQueryResult* response, std::function f) { - ::grpc_impl::internal::CallbackUnaryCall(stub_->channel_.get(), stub_->rpcmethod_Search_, context, request, response, std::move(f)); -} - -void MilvusService::Stub::experimental_async::Search(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::TopKQueryResult* response, std::function f) { - ::grpc_impl::internal::CallbackUnaryCall(stub_->channel_.get(), stub_->rpcmethod_Search_, context, request, response, std::move(f)); -} - -void MilvusService::Stub::experimental_async::Search(::grpc::ClientContext* context, const ::milvus::grpc::SearchParam* request, ::milvus::grpc::TopKQueryResult* response, ::grpc::experimental::ClientUnaryReactor* reactor) { - ::grpc_impl::internal::ClientCallbackUnaryFactory::Create(stub_->channel_.get(), stub_->rpcmethod_Search_, context, request, response, reactor); -} - -void MilvusService::Stub::experimental_async::Search(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::TopKQueryResult* response, ::grpc::experimental::ClientUnaryReactor* reactor) { - ::grpc_impl::internal::ClientCallbackUnaryFactory::Create(stub_->channel_.get(), stub_->rpcmethod_Search_, context, request, response, reactor); -} - -::grpc::ClientAsyncResponseReader< ::milvus::grpc::TopKQueryResult>* MilvusService::Stub::AsyncSearchRaw(::grpc::ClientContext* context, const ::milvus::grpc::SearchParam& request, ::grpc::CompletionQueue* cq) { - return ::grpc_impl::internal::ClientAsyncResponseReaderFactory< ::milvus::grpc::TopKQueryResult>::Create(channel_.get(), cq, rpcmethod_Search_, context, request, true); -} - -::grpc::ClientAsyncResponseReader< ::milvus::grpc::TopKQueryResult>* MilvusService::Stub::PrepareAsyncSearchRaw(::grpc::ClientContext* context, const ::milvus::grpc::SearchParam& request, ::grpc::CompletionQueue* cq) { - return ::grpc_impl::internal::ClientAsyncResponseReaderFactory< ::milvus::grpc::TopKQueryResult>::Create(channel_.get(), cq, rpcmethod_Search_, context, request, false); -} - -::grpc::Status MilvusService::Stub::SearchInFiles(::grpc::ClientContext* context, const ::milvus::grpc::SearchInFilesParam& request, ::milvus::grpc::TopKQueryResult* response) { - return ::grpc::internal::BlockingUnaryCall(channel_.get(), rpcmethod_SearchInFiles_, context, request, response); -} - -void MilvusService::Stub::experimental_async::SearchInFiles(::grpc::ClientContext* context, const ::milvus::grpc::SearchInFilesParam* request, ::milvus::grpc::TopKQueryResult* response, std::function f) { - ::grpc_impl::internal::CallbackUnaryCall(stub_->channel_.get(), stub_->rpcmethod_SearchInFiles_, context, request, response, std::move(f)); -} - -void MilvusService::Stub::experimental_async::SearchInFiles(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::TopKQueryResult* response, std::function f) { - ::grpc_impl::internal::CallbackUnaryCall(stub_->channel_.get(), stub_->rpcmethod_SearchInFiles_, context, request, response, std::move(f)); -} - -void MilvusService::Stub::experimental_async::SearchInFiles(::grpc::ClientContext* context, const ::milvus::grpc::SearchInFilesParam* request, ::milvus::grpc::TopKQueryResult* response, ::grpc::experimental::ClientUnaryReactor* reactor) { - ::grpc_impl::internal::ClientCallbackUnaryFactory::Create(stub_->channel_.get(), stub_->rpcmethod_SearchInFiles_, context, request, response, reactor); -} - -void MilvusService::Stub::experimental_async::SearchInFiles(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::TopKQueryResult* response, ::grpc::experimental::ClientUnaryReactor* reactor) { - ::grpc_impl::internal::ClientCallbackUnaryFactory::Create(stub_->channel_.get(), stub_->rpcmethod_SearchInFiles_, context, request, response, reactor); -} - -::grpc::ClientAsyncResponseReader< ::milvus::grpc::TopKQueryResult>* MilvusService::Stub::AsyncSearchInFilesRaw(::grpc::ClientContext* context, const ::milvus::grpc::SearchInFilesParam& request, ::grpc::CompletionQueue* cq) { - return ::grpc_impl::internal::ClientAsyncResponseReaderFactory< ::milvus::grpc::TopKQueryResult>::Create(channel_.get(), cq, rpcmethod_SearchInFiles_, context, request, true); -} - -::grpc::ClientAsyncResponseReader< ::milvus::grpc::TopKQueryResult>* MilvusService::Stub::PrepareAsyncSearchInFilesRaw(::grpc::ClientContext* context, const ::milvus::grpc::SearchInFilesParam& request, ::grpc::CompletionQueue* cq) { - return ::grpc_impl::internal::ClientAsyncResponseReaderFactory< ::milvus::grpc::TopKQueryResult>::Create(channel_.get(), cq, rpcmethod_SearchInFiles_, context, request, false); -} - ::grpc::Status MilvusService::Stub::DescribeTable(::grpc::ClientContext* context, const ::milvus::grpc::TableName& request, ::milvus::grpc::TableSchema* response) { return ::grpc::internal::BlockingUnaryCall(channel_.get(), rpcmethod_DescribeTable_, context, request, response); } @@ -341,88 +207,60 @@ void MilvusService::Stub::experimental_async::ShowTables(::grpc::ClientContext* return ::grpc_impl::internal::ClientAsyncResponseReaderFactory< ::milvus::grpc::TableNameList>::Create(channel_.get(), cq, rpcmethod_ShowTables_, context, request, false); } -::grpc::Status MilvusService::Stub::Cmd(::grpc::ClientContext* context, const ::milvus::grpc::Command& request, ::milvus::grpc::StringReply* response) { - return ::grpc::internal::BlockingUnaryCall(channel_.get(), rpcmethod_Cmd_, context, request, response); +::grpc::Status MilvusService::Stub::DropTable(::grpc::ClientContext* context, const ::milvus::grpc::TableName& request, ::milvus::grpc::Status* response) { + return ::grpc::internal::BlockingUnaryCall(channel_.get(), rpcmethod_DropTable_, context, request, response); } -void MilvusService::Stub::experimental_async::Cmd(::grpc::ClientContext* context, const ::milvus::grpc::Command* request, ::milvus::grpc::StringReply* response, std::function f) { - ::grpc_impl::internal::CallbackUnaryCall(stub_->channel_.get(), stub_->rpcmethod_Cmd_, context, request, response, std::move(f)); +void MilvusService::Stub::experimental_async::DropTable(::grpc::ClientContext* context, const ::milvus::grpc::TableName* request, ::milvus::grpc::Status* response, std::function f) { + ::grpc_impl::internal::CallbackUnaryCall(stub_->channel_.get(), stub_->rpcmethod_DropTable_, context, request, response, std::move(f)); } -void MilvusService::Stub::experimental_async::Cmd(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::StringReply* response, std::function f) { - ::grpc_impl::internal::CallbackUnaryCall(stub_->channel_.get(), stub_->rpcmethod_Cmd_, context, request, response, std::move(f)); +void MilvusService::Stub::experimental_async::DropTable(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::Status* response, std::function f) { + ::grpc_impl::internal::CallbackUnaryCall(stub_->channel_.get(), stub_->rpcmethod_DropTable_, context, request, response, std::move(f)); } -void MilvusService::Stub::experimental_async::Cmd(::grpc::ClientContext* context, const ::milvus::grpc::Command* request, ::milvus::grpc::StringReply* response, ::grpc::experimental::ClientUnaryReactor* reactor) { - ::grpc_impl::internal::ClientCallbackUnaryFactory::Create(stub_->channel_.get(), stub_->rpcmethod_Cmd_, context, request, response, reactor); +void MilvusService::Stub::experimental_async::DropTable(::grpc::ClientContext* context, const ::milvus::grpc::TableName* request, ::milvus::grpc::Status* response, ::grpc::experimental::ClientUnaryReactor* reactor) { + ::grpc_impl::internal::ClientCallbackUnaryFactory::Create(stub_->channel_.get(), stub_->rpcmethod_DropTable_, context, request, response, reactor); } -void MilvusService::Stub::experimental_async::Cmd(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::StringReply* response, ::grpc::experimental::ClientUnaryReactor* reactor) { - ::grpc_impl::internal::ClientCallbackUnaryFactory::Create(stub_->channel_.get(), stub_->rpcmethod_Cmd_, context, request, response, reactor); +void MilvusService::Stub::experimental_async::DropTable(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::Status* response, ::grpc::experimental::ClientUnaryReactor* reactor) { + ::grpc_impl::internal::ClientCallbackUnaryFactory::Create(stub_->channel_.get(), stub_->rpcmethod_DropTable_, context, request, response, reactor); } -::grpc::ClientAsyncResponseReader< ::milvus::grpc::StringReply>* MilvusService::Stub::AsyncCmdRaw(::grpc::ClientContext* context, const ::milvus::grpc::Command& request, ::grpc::CompletionQueue* cq) { - return ::grpc_impl::internal::ClientAsyncResponseReaderFactory< ::milvus::grpc::StringReply>::Create(channel_.get(), cq, rpcmethod_Cmd_, context, request, true); +::grpc::ClientAsyncResponseReader< ::milvus::grpc::Status>* MilvusService::Stub::AsyncDropTableRaw(::grpc::ClientContext* context, const ::milvus::grpc::TableName& request, ::grpc::CompletionQueue* cq) { + return ::grpc_impl::internal::ClientAsyncResponseReaderFactory< ::milvus::grpc::Status>::Create(channel_.get(), cq, rpcmethod_DropTable_, context, request, true); } -::grpc::ClientAsyncResponseReader< ::milvus::grpc::StringReply>* MilvusService::Stub::PrepareAsyncCmdRaw(::grpc::ClientContext* context, const ::milvus::grpc::Command& request, ::grpc::CompletionQueue* cq) { - return ::grpc_impl::internal::ClientAsyncResponseReaderFactory< ::milvus::grpc::StringReply>::Create(channel_.get(), cq, rpcmethod_Cmd_, context, request, false); +::grpc::ClientAsyncResponseReader< ::milvus::grpc::Status>* MilvusService::Stub::PrepareAsyncDropTableRaw(::grpc::ClientContext* context, const ::milvus::grpc::TableName& request, ::grpc::CompletionQueue* cq) { + return ::grpc_impl::internal::ClientAsyncResponseReaderFactory< ::milvus::grpc::Status>::Create(channel_.get(), cq, rpcmethod_DropTable_, context, request, false); } -::grpc::Status MilvusService::Stub::DeleteByRange(::grpc::ClientContext* context, const ::milvus::grpc::DeleteByRangeParam& request, ::milvus::grpc::Status* response) { - return ::grpc::internal::BlockingUnaryCall(channel_.get(), rpcmethod_DeleteByRange_, context, request, response); +::grpc::Status MilvusService::Stub::CreateIndex(::grpc::ClientContext* context, const ::milvus::grpc::IndexParam& request, ::milvus::grpc::Status* response) { + return ::grpc::internal::BlockingUnaryCall(channel_.get(), rpcmethod_CreateIndex_, context, request, response); } -void MilvusService::Stub::experimental_async::DeleteByRange(::grpc::ClientContext* context, const ::milvus::grpc::DeleteByRangeParam* request, ::milvus::grpc::Status* response, std::function f) { - ::grpc_impl::internal::CallbackUnaryCall(stub_->channel_.get(), stub_->rpcmethod_DeleteByRange_, context, request, response, std::move(f)); +void MilvusService::Stub::experimental_async::CreateIndex(::grpc::ClientContext* context, const ::milvus::grpc::IndexParam* request, ::milvus::grpc::Status* response, std::function f) { + ::grpc_impl::internal::CallbackUnaryCall(stub_->channel_.get(), stub_->rpcmethod_CreateIndex_, context, request, response, std::move(f)); } -void MilvusService::Stub::experimental_async::DeleteByRange(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::Status* response, std::function f) { - ::grpc_impl::internal::CallbackUnaryCall(stub_->channel_.get(), stub_->rpcmethod_DeleteByRange_, context, request, response, std::move(f)); +void MilvusService::Stub::experimental_async::CreateIndex(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::Status* response, std::function f) { + ::grpc_impl::internal::CallbackUnaryCall(stub_->channel_.get(), stub_->rpcmethod_CreateIndex_, context, request, response, std::move(f)); } -void MilvusService::Stub::experimental_async::DeleteByRange(::grpc::ClientContext* context, const ::milvus::grpc::DeleteByRangeParam* request, ::milvus::grpc::Status* response, ::grpc::experimental::ClientUnaryReactor* reactor) { - ::grpc_impl::internal::ClientCallbackUnaryFactory::Create(stub_->channel_.get(), stub_->rpcmethod_DeleteByRange_, context, request, response, reactor); +void MilvusService::Stub::experimental_async::CreateIndex(::grpc::ClientContext* context, const ::milvus::grpc::IndexParam* request, ::milvus::grpc::Status* response, ::grpc::experimental::ClientUnaryReactor* reactor) { + ::grpc_impl::internal::ClientCallbackUnaryFactory::Create(stub_->channel_.get(), stub_->rpcmethod_CreateIndex_, context, request, response, reactor); } -void MilvusService::Stub::experimental_async::DeleteByRange(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::Status* response, ::grpc::experimental::ClientUnaryReactor* reactor) { - ::grpc_impl::internal::ClientCallbackUnaryFactory::Create(stub_->channel_.get(), stub_->rpcmethod_DeleteByRange_, context, request, response, reactor); +void MilvusService::Stub::experimental_async::CreateIndex(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::Status* response, ::grpc::experimental::ClientUnaryReactor* reactor) { + ::grpc_impl::internal::ClientCallbackUnaryFactory::Create(stub_->channel_.get(), stub_->rpcmethod_CreateIndex_, context, request, response, reactor); } -::grpc::ClientAsyncResponseReader< ::milvus::grpc::Status>* MilvusService::Stub::AsyncDeleteByRangeRaw(::grpc::ClientContext* context, const ::milvus::grpc::DeleteByRangeParam& request, ::grpc::CompletionQueue* cq) { - return ::grpc_impl::internal::ClientAsyncResponseReaderFactory< ::milvus::grpc::Status>::Create(channel_.get(), cq, rpcmethod_DeleteByRange_, context, request, true); +::grpc::ClientAsyncResponseReader< ::milvus::grpc::Status>* MilvusService::Stub::AsyncCreateIndexRaw(::grpc::ClientContext* context, const ::milvus::grpc::IndexParam& request, ::grpc::CompletionQueue* cq) { + return ::grpc_impl::internal::ClientAsyncResponseReaderFactory< ::milvus::grpc::Status>::Create(channel_.get(), cq, rpcmethod_CreateIndex_, context, request, true); } -::grpc::ClientAsyncResponseReader< ::milvus::grpc::Status>* MilvusService::Stub::PrepareAsyncDeleteByRangeRaw(::grpc::ClientContext* context, const ::milvus::grpc::DeleteByRangeParam& request, ::grpc::CompletionQueue* cq) { - return ::grpc_impl::internal::ClientAsyncResponseReaderFactory< ::milvus::grpc::Status>::Create(channel_.get(), cq, rpcmethod_DeleteByRange_, context, request, false); -} - -::grpc::Status MilvusService::Stub::PreloadTable(::grpc::ClientContext* context, const ::milvus::grpc::TableName& request, ::milvus::grpc::Status* response) { - return ::grpc::internal::BlockingUnaryCall(channel_.get(), rpcmethod_PreloadTable_, context, request, response); -} - -void MilvusService::Stub::experimental_async::PreloadTable(::grpc::ClientContext* context, const ::milvus::grpc::TableName* request, ::milvus::grpc::Status* response, std::function f) { - ::grpc_impl::internal::CallbackUnaryCall(stub_->channel_.get(), stub_->rpcmethod_PreloadTable_, context, request, response, std::move(f)); -} - -void MilvusService::Stub::experimental_async::PreloadTable(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::Status* response, std::function f) { - ::grpc_impl::internal::CallbackUnaryCall(stub_->channel_.get(), stub_->rpcmethod_PreloadTable_, context, request, response, std::move(f)); -} - -void MilvusService::Stub::experimental_async::PreloadTable(::grpc::ClientContext* context, const ::milvus::grpc::TableName* request, ::milvus::grpc::Status* response, ::grpc::experimental::ClientUnaryReactor* reactor) { - ::grpc_impl::internal::ClientCallbackUnaryFactory::Create(stub_->channel_.get(), stub_->rpcmethod_PreloadTable_, context, request, response, reactor); -} - -void MilvusService::Stub::experimental_async::PreloadTable(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::Status* response, ::grpc::experimental::ClientUnaryReactor* reactor) { - ::grpc_impl::internal::ClientCallbackUnaryFactory::Create(stub_->channel_.get(), stub_->rpcmethod_PreloadTable_, context, request, response, reactor); -} - -::grpc::ClientAsyncResponseReader< ::milvus::grpc::Status>* MilvusService::Stub::AsyncPreloadTableRaw(::grpc::ClientContext* context, const ::milvus::grpc::TableName& request, ::grpc::CompletionQueue* cq) { - return ::grpc_impl::internal::ClientAsyncResponseReaderFactory< ::milvus::grpc::Status>::Create(channel_.get(), cq, rpcmethod_PreloadTable_, context, request, true); -} - -::grpc::ClientAsyncResponseReader< ::milvus::grpc::Status>* MilvusService::Stub::PrepareAsyncPreloadTableRaw(::grpc::ClientContext* context, const ::milvus::grpc::TableName& request, ::grpc::CompletionQueue* cq) { - return ::grpc_impl::internal::ClientAsyncResponseReaderFactory< ::milvus::grpc::Status>::Create(channel_.get(), cq, rpcmethod_PreloadTable_, context, request, false); +::grpc::ClientAsyncResponseReader< ::milvus::grpc::Status>* MilvusService::Stub::PrepareAsyncCreateIndexRaw(::grpc::ClientContext* context, const ::milvus::grpc::IndexParam& request, ::grpc::CompletionQueue* cq) { + return ::grpc_impl::internal::ClientAsyncResponseReaderFactory< ::milvus::grpc::Status>::Create(channel_.get(), cq, rpcmethod_CreateIndex_, context, request, false); } ::grpc::Status MilvusService::Stub::DescribeIndex(::grpc::ClientContext* context, const ::milvus::grpc::TableName& request, ::milvus::grpc::IndexParam* response) { @@ -481,6 +319,258 @@ void MilvusService::Stub::experimental_async::DropIndex(::grpc::ClientContext* c return ::grpc_impl::internal::ClientAsyncResponseReaderFactory< ::milvus::grpc::Status>::Create(channel_.get(), cq, rpcmethod_DropIndex_, context, request, false); } +::grpc::Status MilvusService::Stub::CreatePartition(::grpc::ClientContext* context, const ::milvus::grpc::PartitionParam& request, ::milvus::grpc::Status* response) { + return ::grpc::internal::BlockingUnaryCall(channel_.get(), rpcmethod_CreatePartition_, context, request, response); +} + +void MilvusService::Stub::experimental_async::CreatePartition(::grpc::ClientContext* context, const ::milvus::grpc::PartitionParam* request, ::milvus::grpc::Status* response, std::function f) { + ::grpc_impl::internal::CallbackUnaryCall(stub_->channel_.get(), stub_->rpcmethod_CreatePartition_, context, request, response, std::move(f)); +} + +void MilvusService::Stub::experimental_async::CreatePartition(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::Status* response, std::function f) { + ::grpc_impl::internal::CallbackUnaryCall(stub_->channel_.get(), stub_->rpcmethod_CreatePartition_, context, request, response, std::move(f)); +} + +void MilvusService::Stub::experimental_async::CreatePartition(::grpc::ClientContext* context, const ::milvus::grpc::PartitionParam* request, ::milvus::grpc::Status* response, ::grpc::experimental::ClientUnaryReactor* reactor) { + ::grpc_impl::internal::ClientCallbackUnaryFactory::Create(stub_->channel_.get(), stub_->rpcmethod_CreatePartition_, context, request, response, reactor); +} + +void MilvusService::Stub::experimental_async::CreatePartition(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::Status* response, ::grpc::experimental::ClientUnaryReactor* reactor) { + ::grpc_impl::internal::ClientCallbackUnaryFactory::Create(stub_->channel_.get(), stub_->rpcmethod_CreatePartition_, context, request, response, reactor); +} + +::grpc::ClientAsyncResponseReader< ::milvus::grpc::Status>* MilvusService::Stub::AsyncCreatePartitionRaw(::grpc::ClientContext* context, const ::milvus::grpc::PartitionParam& request, ::grpc::CompletionQueue* cq) { + return ::grpc_impl::internal::ClientAsyncResponseReaderFactory< ::milvus::grpc::Status>::Create(channel_.get(), cq, rpcmethod_CreatePartition_, context, request, true); +} + +::grpc::ClientAsyncResponseReader< ::milvus::grpc::Status>* MilvusService::Stub::PrepareAsyncCreatePartitionRaw(::grpc::ClientContext* context, const ::milvus::grpc::PartitionParam& request, ::grpc::CompletionQueue* cq) { + return ::grpc_impl::internal::ClientAsyncResponseReaderFactory< ::milvus::grpc::Status>::Create(channel_.get(), cq, rpcmethod_CreatePartition_, context, request, false); +} + +::grpc::Status MilvusService::Stub::ShowPartitions(::grpc::ClientContext* context, const ::milvus::grpc::TableName& request, ::milvus::grpc::PartitionList* response) { + return ::grpc::internal::BlockingUnaryCall(channel_.get(), rpcmethod_ShowPartitions_, context, request, response); +} + +void MilvusService::Stub::experimental_async::ShowPartitions(::grpc::ClientContext* context, const ::milvus::grpc::TableName* request, ::milvus::grpc::PartitionList* response, std::function f) { + ::grpc_impl::internal::CallbackUnaryCall(stub_->channel_.get(), stub_->rpcmethod_ShowPartitions_, context, request, response, std::move(f)); +} + +void MilvusService::Stub::experimental_async::ShowPartitions(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::PartitionList* response, std::function f) { + ::grpc_impl::internal::CallbackUnaryCall(stub_->channel_.get(), stub_->rpcmethod_ShowPartitions_, context, request, response, std::move(f)); +} + +void MilvusService::Stub::experimental_async::ShowPartitions(::grpc::ClientContext* context, const ::milvus::grpc::TableName* request, ::milvus::grpc::PartitionList* response, ::grpc::experimental::ClientUnaryReactor* reactor) { + ::grpc_impl::internal::ClientCallbackUnaryFactory::Create(stub_->channel_.get(), stub_->rpcmethod_ShowPartitions_, context, request, response, reactor); +} + +void MilvusService::Stub::experimental_async::ShowPartitions(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::PartitionList* response, ::grpc::experimental::ClientUnaryReactor* reactor) { + ::grpc_impl::internal::ClientCallbackUnaryFactory::Create(stub_->channel_.get(), stub_->rpcmethod_ShowPartitions_, context, request, response, reactor); +} + +::grpc::ClientAsyncResponseReader< ::milvus::grpc::PartitionList>* MilvusService::Stub::AsyncShowPartitionsRaw(::grpc::ClientContext* context, const ::milvus::grpc::TableName& request, ::grpc::CompletionQueue* cq) { + return ::grpc_impl::internal::ClientAsyncResponseReaderFactory< ::milvus::grpc::PartitionList>::Create(channel_.get(), cq, rpcmethod_ShowPartitions_, context, request, true); +} + +::grpc::ClientAsyncResponseReader< ::milvus::grpc::PartitionList>* MilvusService::Stub::PrepareAsyncShowPartitionsRaw(::grpc::ClientContext* context, const ::milvus::grpc::TableName& request, ::grpc::CompletionQueue* cq) { + return ::grpc_impl::internal::ClientAsyncResponseReaderFactory< ::milvus::grpc::PartitionList>::Create(channel_.get(), cq, rpcmethod_ShowPartitions_, context, request, false); +} + +::grpc::Status MilvusService::Stub::DropPartition(::grpc::ClientContext* context, const ::milvus::grpc::PartitionParam& request, ::milvus::grpc::Status* response) { + return ::grpc::internal::BlockingUnaryCall(channel_.get(), rpcmethod_DropPartition_, context, request, response); +} + +void MilvusService::Stub::experimental_async::DropPartition(::grpc::ClientContext* context, const ::milvus::grpc::PartitionParam* request, ::milvus::grpc::Status* response, std::function f) { + ::grpc_impl::internal::CallbackUnaryCall(stub_->channel_.get(), stub_->rpcmethod_DropPartition_, context, request, response, std::move(f)); +} + +void MilvusService::Stub::experimental_async::DropPartition(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::Status* response, std::function f) { + ::grpc_impl::internal::CallbackUnaryCall(stub_->channel_.get(), stub_->rpcmethod_DropPartition_, context, request, response, std::move(f)); +} + +void MilvusService::Stub::experimental_async::DropPartition(::grpc::ClientContext* context, const ::milvus::grpc::PartitionParam* request, ::milvus::grpc::Status* response, ::grpc::experimental::ClientUnaryReactor* reactor) { + ::grpc_impl::internal::ClientCallbackUnaryFactory::Create(stub_->channel_.get(), stub_->rpcmethod_DropPartition_, context, request, response, reactor); +} + +void MilvusService::Stub::experimental_async::DropPartition(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::Status* response, ::grpc::experimental::ClientUnaryReactor* reactor) { + ::grpc_impl::internal::ClientCallbackUnaryFactory::Create(stub_->channel_.get(), stub_->rpcmethod_DropPartition_, context, request, response, reactor); +} + +::grpc::ClientAsyncResponseReader< ::milvus::grpc::Status>* MilvusService::Stub::AsyncDropPartitionRaw(::grpc::ClientContext* context, const ::milvus::grpc::PartitionParam& request, ::grpc::CompletionQueue* cq) { + return ::grpc_impl::internal::ClientAsyncResponseReaderFactory< ::milvus::grpc::Status>::Create(channel_.get(), cq, rpcmethod_DropPartition_, context, request, true); +} + +::grpc::ClientAsyncResponseReader< ::milvus::grpc::Status>* MilvusService::Stub::PrepareAsyncDropPartitionRaw(::grpc::ClientContext* context, const ::milvus::grpc::PartitionParam& request, ::grpc::CompletionQueue* cq) { + return ::grpc_impl::internal::ClientAsyncResponseReaderFactory< ::milvus::grpc::Status>::Create(channel_.get(), cq, rpcmethod_DropPartition_, context, request, false); +} + +::grpc::Status MilvusService::Stub::Insert(::grpc::ClientContext* context, const ::milvus::grpc::InsertParam& request, ::milvus::grpc::VectorIds* response) { + return ::grpc::internal::BlockingUnaryCall(channel_.get(), rpcmethod_Insert_, context, request, response); +} + +void MilvusService::Stub::experimental_async::Insert(::grpc::ClientContext* context, const ::milvus::grpc::InsertParam* request, ::milvus::grpc::VectorIds* response, std::function f) { + ::grpc_impl::internal::CallbackUnaryCall(stub_->channel_.get(), stub_->rpcmethod_Insert_, context, request, response, std::move(f)); +} + +void MilvusService::Stub::experimental_async::Insert(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::VectorIds* response, std::function f) { + ::grpc_impl::internal::CallbackUnaryCall(stub_->channel_.get(), stub_->rpcmethod_Insert_, context, request, response, std::move(f)); +} + +void MilvusService::Stub::experimental_async::Insert(::grpc::ClientContext* context, const ::milvus::grpc::InsertParam* request, ::milvus::grpc::VectorIds* response, ::grpc::experimental::ClientUnaryReactor* reactor) { + ::grpc_impl::internal::ClientCallbackUnaryFactory::Create(stub_->channel_.get(), stub_->rpcmethod_Insert_, context, request, response, reactor); +} + +void MilvusService::Stub::experimental_async::Insert(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::VectorIds* response, ::grpc::experimental::ClientUnaryReactor* reactor) { + ::grpc_impl::internal::ClientCallbackUnaryFactory::Create(stub_->channel_.get(), stub_->rpcmethod_Insert_, context, request, response, reactor); +} + +::grpc::ClientAsyncResponseReader< ::milvus::grpc::VectorIds>* MilvusService::Stub::AsyncInsertRaw(::grpc::ClientContext* context, const ::milvus::grpc::InsertParam& request, ::grpc::CompletionQueue* cq) { + return ::grpc_impl::internal::ClientAsyncResponseReaderFactory< ::milvus::grpc::VectorIds>::Create(channel_.get(), cq, rpcmethod_Insert_, context, request, true); +} + +::grpc::ClientAsyncResponseReader< ::milvus::grpc::VectorIds>* MilvusService::Stub::PrepareAsyncInsertRaw(::grpc::ClientContext* context, const ::milvus::grpc::InsertParam& request, ::grpc::CompletionQueue* cq) { + return ::grpc_impl::internal::ClientAsyncResponseReaderFactory< ::milvus::grpc::VectorIds>::Create(channel_.get(), cq, rpcmethod_Insert_, context, request, false); +} + +::grpc::Status MilvusService::Stub::Search(::grpc::ClientContext* context, const ::milvus::grpc::SearchParam& request, ::milvus::grpc::TopKQueryResult* response) { + return ::grpc::internal::BlockingUnaryCall(channel_.get(), rpcmethod_Search_, context, request, response); +} + +void MilvusService::Stub::experimental_async::Search(::grpc::ClientContext* context, const ::milvus::grpc::SearchParam* request, ::milvus::grpc::TopKQueryResult* response, std::function f) { + ::grpc_impl::internal::CallbackUnaryCall(stub_->channel_.get(), stub_->rpcmethod_Search_, context, request, response, std::move(f)); +} + +void MilvusService::Stub::experimental_async::Search(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::TopKQueryResult* response, std::function f) { + ::grpc_impl::internal::CallbackUnaryCall(stub_->channel_.get(), stub_->rpcmethod_Search_, context, request, response, std::move(f)); +} + +void MilvusService::Stub::experimental_async::Search(::grpc::ClientContext* context, const ::milvus::grpc::SearchParam* request, ::milvus::grpc::TopKQueryResult* response, ::grpc::experimental::ClientUnaryReactor* reactor) { + ::grpc_impl::internal::ClientCallbackUnaryFactory::Create(stub_->channel_.get(), stub_->rpcmethod_Search_, context, request, response, reactor); +} + +void MilvusService::Stub::experimental_async::Search(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::TopKQueryResult* response, ::grpc::experimental::ClientUnaryReactor* reactor) { + ::grpc_impl::internal::ClientCallbackUnaryFactory::Create(stub_->channel_.get(), stub_->rpcmethod_Search_, context, request, response, reactor); +} + +::grpc::ClientAsyncResponseReader< ::milvus::grpc::TopKQueryResult>* MilvusService::Stub::AsyncSearchRaw(::grpc::ClientContext* context, const ::milvus::grpc::SearchParam& request, ::grpc::CompletionQueue* cq) { + return ::grpc_impl::internal::ClientAsyncResponseReaderFactory< ::milvus::grpc::TopKQueryResult>::Create(channel_.get(), cq, rpcmethod_Search_, context, request, true); +} + +::grpc::ClientAsyncResponseReader< ::milvus::grpc::TopKQueryResult>* MilvusService::Stub::PrepareAsyncSearchRaw(::grpc::ClientContext* context, const ::milvus::grpc::SearchParam& request, ::grpc::CompletionQueue* cq) { + return ::grpc_impl::internal::ClientAsyncResponseReaderFactory< ::milvus::grpc::TopKQueryResult>::Create(channel_.get(), cq, rpcmethod_Search_, context, request, false); +} + +::grpc::Status MilvusService::Stub::SearchInFiles(::grpc::ClientContext* context, const ::milvus::grpc::SearchInFilesParam& request, ::milvus::grpc::TopKQueryResult* response) { + return ::grpc::internal::BlockingUnaryCall(channel_.get(), rpcmethod_SearchInFiles_, context, request, response); +} + +void MilvusService::Stub::experimental_async::SearchInFiles(::grpc::ClientContext* context, const ::milvus::grpc::SearchInFilesParam* request, ::milvus::grpc::TopKQueryResult* response, std::function f) { + ::grpc_impl::internal::CallbackUnaryCall(stub_->channel_.get(), stub_->rpcmethod_SearchInFiles_, context, request, response, std::move(f)); +} + +void MilvusService::Stub::experimental_async::SearchInFiles(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::TopKQueryResult* response, std::function f) { + ::grpc_impl::internal::CallbackUnaryCall(stub_->channel_.get(), stub_->rpcmethod_SearchInFiles_, context, request, response, std::move(f)); +} + +void MilvusService::Stub::experimental_async::SearchInFiles(::grpc::ClientContext* context, const ::milvus::grpc::SearchInFilesParam* request, ::milvus::grpc::TopKQueryResult* response, ::grpc::experimental::ClientUnaryReactor* reactor) { + ::grpc_impl::internal::ClientCallbackUnaryFactory::Create(stub_->channel_.get(), stub_->rpcmethod_SearchInFiles_, context, request, response, reactor); +} + +void MilvusService::Stub::experimental_async::SearchInFiles(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::TopKQueryResult* response, ::grpc::experimental::ClientUnaryReactor* reactor) { + ::grpc_impl::internal::ClientCallbackUnaryFactory::Create(stub_->channel_.get(), stub_->rpcmethod_SearchInFiles_, context, request, response, reactor); +} + +::grpc::ClientAsyncResponseReader< ::milvus::grpc::TopKQueryResult>* MilvusService::Stub::AsyncSearchInFilesRaw(::grpc::ClientContext* context, const ::milvus::grpc::SearchInFilesParam& request, ::grpc::CompletionQueue* cq) { + return ::grpc_impl::internal::ClientAsyncResponseReaderFactory< ::milvus::grpc::TopKQueryResult>::Create(channel_.get(), cq, rpcmethod_SearchInFiles_, context, request, true); +} + +::grpc::ClientAsyncResponseReader< ::milvus::grpc::TopKQueryResult>* MilvusService::Stub::PrepareAsyncSearchInFilesRaw(::grpc::ClientContext* context, const ::milvus::grpc::SearchInFilesParam& request, ::grpc::CompletionQueue* cq) { + return ::grpc_impl::internal::ClientAsyncResponseReaderFactory< ::milvus::grpc::TopKQueryResult>::Create(channel_.get(), cq, rpcmethod_SearchInFiles_, context, request, false); +} + +::grpc::Status MilvusService::Stub::Cmd(::grpc::ClientContext* context, const ::milvus::grpc::Command& request, ::milvus::grpc::StringReply* response) { + return ::grpc::internal::BlockingUnaryCall(channel_.get(), rpcmethod_Cmd_, context, request, response); +} + +void MilvusService::Stub::experimental_async::Cmd(::grpc::ClientContext* context, const ::milvus::grpc::Command* request, ::milvus::grpc::StringReply* response, std::function f) { + ::grpc_impl::internal::CallbackUnaryCall(stub_->channel_.get(), stub_->rpcmethod_Cmd_, context, request, response, std::move(f)); +} + +void MilvusService::Stub::experimental_async::Cmd(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::StringReply* response, std::function f) { + ::grpc_impl::internal::CallbackUnaryCall(stub_->channel_.get(), stub_->rpcmethod_Cmd_, context, request, response, std::move(f)); +} + +void MilvusService::Stub::experimental_async::Cmd(::grpc::ClientContext* context, const ::milvus::grpc::Command* request, ::milvus::grpc::StringReply* response, ::grpc::experimental::ClientUnaryReactor* reactor) { + ::grpc_impl::internal::ClientCallbackUnaryFactory::Create(stub_->channel_.get(), stub_->rpcmethod_Cmd_, context, request, response, reactor); +} + +void MilvusService::Stub::experimental_async::Cmd(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::StringReply* response, ::grpc::experimental::ClientUnaryReactor* reactor) { + ::grpc_impl::internal::ClientCallbackUnaryFactory::Create(stub_->channel_.get(), stub_->rpcmethod_Cmd_, context, request, response, reactor); +} + +::grpc::ClientAsyncResponseReader< ::milvus::grpc::StringReply>* MilvusService::Stub::AsyncCmdRaw(::grpc::ClientContext* context, const ::milvus::grpc::Command& request, ::grpc::CompletionQueue* cq) { + return ::grpc_impl::internal::ClientAsyncResponseReaderFactory< ::milvus::grpc::StringReply>::Create(channel_.get(), cq, rpcmethod_Cmd_, context, request, true); +} + +::grpc::ClientAsyncResponseReader< ::milvus::grpc::StringReply>* MilvusService::Stub::PrepareAsyncCmdRaw(::grpc::ClientContext* context, const ::milvus::grpc::Command& request, ::grpc::CompletionQueue* cq) { + return ::grpc_impl::internal::ClientAsyncResponseReaderFactory< ::milvus::grpc::StringReply>::Create(channel_.get(), cq, rpcmethod_Cmd_, context, request, false); +} + +::grpc::Status MilvusService::Stub::DeleteByDate(::grpc::ClientContext* context, const ::milvus::grpc::DeleteByDateParam& request, ::milvus::grpc::Status* response) { + return ::grpc::internal::BlockingUnaryCall(channel_.get(), rpcmethod_DeleteByDate_, context, request, response); +} + +void MilvusService::Stub::experimental_async::DeleteByDate(::grpc::ClientContext* context, const ::milvus::grpc::DeleteByDateParam* request, ::milvus::grpc::Status* response, std::function f) { + ::grpc_impl::internal::CallbackUnaryCall(stub_->channel_.get(), stub_->rpcmethod_DeleteByDate_, context, request, response, std::move(f)); +} + +void MilvusService::Stub::experimental_async::DeleteByDate(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::Status* response, std::function f) { + ::grpc_impl::internal::CallbackUnaryCall(stub_->channel_.get(), stub_->rpcmethod_DeleteByDate_, context, request, response, std::move(f)); +} + +void MilvusService::Stub::experimental_async::DeleteByDate(::grpc::ClientContext* context, const ::milvus::grpc::DeleteByDateParam* request, ::milvus::grpc::Status* response, ::grpc::experimental::ClientUnaryReactor* reactor) { + ::grpc_impl::internal::ClientCallbackUnaryFactory::Create(stub_->channel_.get(), stub_->rpcmethod_DeleteByDate_, context, request, response, reactor); +} + +void MilvusService::Stub::experimental_async::DeleteByDate(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::Status* response, ::grpc::experimental::ClientUnaryReactor* reactor) { + ::grpc_impl::internal::ClientCallbackUnaryFactory::Create(stub_->channel_.get(), stub_->rpcmethod_DeleteByDate_, context, request, response, reactor); +} + +::grpc::ClientAsyncResponseReader< ::milvus::grpc::Status>* MilvusService::Stub::AsyncDeleteByDateRaw(::grpc::ClientContext* context, const ::milvus::grpc::DeleteByDateParam& request, ::grpc::CompletionQueue* cq) { + return ::grpc_impl::internal::ClientAsyncResponseReaderFactory< ::milvus::grpc::Status>::Create(channel_.get(), cq, rpcmethod_DeleteByDate_, context, request, true); +} + +::grpc::ClientAsyncResponseReader< ::milvus::grpc::Status>* MilvusService::Stub::PrepareAsyncDeleteByDateRaw(::grpc::ClientContext* context, const ::milvus::grpc::DeleteByDateParam& request, ::grpc::CompletionQueue* cq) { + return ::grpc_impl::internal::ClientAsyncResponseReaderFactory< ::milvus::grpc::Status>::Create(channel_.get(), cq, rpcmethod_DeleteByDate_, context, request, false); +} + +::grpc::Status MilvusService::Stub::PreloadTable(::grpc::ClientContext* context, const ::milvus::grpc::TableName& request, ::milvus::grpc::Status* response) { + return ::grpc::internal::BlockingUnaryCall(channel_.get(), rpcmethod_PreloadTable_, context, request, response); +} + +void MilvusService::Stub::experimental_async::PreloadTable(::grpc::ClientContext* context, const ::milvus::grpc::TableName* request, ::milvus::grpc::Status* response, std::function f) { + ::grpc_impl::internal::CallbackUnaryCall(stub_->channel_.get(), stub_->rpcmethod_PreloadTable_, context, request, response, std::move(f)); +} + +void MilvusService::Stub::experimental_async::PreloadTable(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::Status* response, std::function f) { + ::grpc_impl::internal::CallbackUnaryCall(stub_->channel_.get(), stub_->rpcmethod_PreloadTable_, context, request, response, std::move(f)); +} + +void MilvusService::Stub::experimental_async::PreloadTable(::grpc::ClientContext* context, const ::milvus::grpc::TableName* request, ::milvus::grpc::Status* response, ::grpc::experimental::ClientUnaryReactor* reactor) { + ::grpc_impl::internal::ClientCallbackUnaryFactory::Create(stub_->channel_.get(), stub_->rpcmethod_PreloadTable_, context, request, response, reactor); +} + +void MilvusService::Stub::experimental_async::PreloadTable(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::Status* response, ::grpc::experimental::ClientUnaryReactor* reactor) { + ::grpc_impl::internal::ClientCallbackUnaryFactory::Create(stub_->channel_.get(), stub_->rpcmethod_PreloadTable_, context, request, response, reactor); +} + +::grpc::ClientAsyncResponseReader< ::milvus::grpc::Status>* MilvusService::Stub::AsyncPreloadTableRaw(::grpc::ClientContext* context, const ::milvus::grpc::TableName& request, ::grpc::CompletionQueue* cq) { + return ::grpc_impl::internal::ClientAsyncResponseReaderFactory< ::milvus::grpc::Status>::Create(channel_.get(), cq, rpcmethod_PreloadTable_, context, request, true); +} + +::grpc::ClientAsyncResponseReader< ::milvus::grpc::Status>* MilvusService::Stub::PrepareAsyncPreloadTableRaw(::grpc::ClientContext* context, const ::milvus::grpc::TableName& request, ::grpc::CompletionQueue* cq) { + return ::grpc_impl::internal::ClientAsyncResponseReaderFactory< ::milvus::grpc::Status>::Create(channel_.get(), cq, rpcmethod_PreloadTable_, context, request, false); +} + MilvusService::Service::Service() { AddMethod(new ::grpc::internal::RpcServiceMethod( MilvusService_method_names[0], @@ -495,68 +585,83 @@ MilvusService::Service::Service() { AddMethod(new ::grpc::internal::RpcServiceMethod( MilvusService_method_names[2], ::grpc::internal::RpcMethod::NORMAL_RPC, - new ::grpc::internal::RpcMethodHandler< MilvusService::Service, ::milvus::grpc::TableName, ::milvus::grpc::Status>( - std::mem_fn(&MilvusService::Service::DropTable), this))); - AddMethod(new ::grpc::internal::RpcServiceMethod( - MilvusService_method_names[3], - ::grpc::internal::RpcMethod::NORMAL_RPC, - new ::grpc::internal::RpcMethodHandler< MilvusService::Service, ::milvus::grpc::IndexParam, ::milvus::grpc::Status>( - std::mem_fn(&MilvusService::Service::CreateIndex), this))); - AddMethod(new ::grpc::internal::RpcServiceMethod( - MilvusService_method_names[4], - ::grpc::internal::RpcMethod::NORMAL_RPC, - new ::grpc::internal::RpcMethodHandler< MilvusService::Service, ::milvus::grpc::InsertParam, ::milvus::grpc::VectorIds>( - std::mem_fn(&MilvusService::Service::Insert), this))); - AddMethod(new ::grpc::internal::RpcServiceMethod( - MilvusService_method_names[5], - ::grpc::internal::RpcMethod::NORMAL_RPC, - new ::grpc::internal::RpcMethodHandler< MilvusService::Service, ::milvus::grpc::SearchParam, ::milvus::grpc::TopKQueryResult>( - std::mem_fn(&MilvusService::Service::Search), this))); - AddMethod(new ::grpc::internal::RpcServiceMethod( - MilvusService_method_names[6], - ::grpc::internal::RpcMethod::NORMAL_RPC, - new ::grpc::internal::RpcMethodHandler< MilvusService::Service, ::milvus::grpc::SearchInFilesParam, ::milvus::grpc::TopKQueryResult>( - std::mem_fn(&MilvusService::Service::SearchInFiles), this))); - AddMethod(new ::grpc::internal::RpcServiceMethod( - MilvusService_method_names[7], - ::grpc::internal::RpcMethod::NORMAL_RPC, new ::grpc::internal::RpcMethodHandler< MilvusService::Service, ::milvus::grpc::TableName, ::milvus::grpc::TableSchema>( std::mem_fn(&MilvusService::Service::DescribeTable), this))); AddMethod(new ::grpc::internal::RpcServiceMethod( - MilvusService_method_names[8], + MilvusService_method_names[3], ::grpc::internal::RpcMethod::NORMAL_RPC, new ::grpc::internal::RpcMethodHandler< MilvusService::Service, ::milvus::grpc::TableName, ::milvus::grpc::TableRowCount>( std::mem_fn(&MilvusService::Service::CountTable), this))); AddMethod(new ::grpc::internal::RpcServiceMethod( - MilvusService_method_names[9], + MilvusService_method_names[4], ::grpc::internal::RpcMethod::NORMAL_RPC, new ::grpc::internal::RpcMethodHandler< MilvusService::Service, ::milvus::grpc::Command, ::milvus::grpc::TableNameList>( std::mem_fn(&MilvusService::Service::ShowTables), this))); AddMethod(new ::grpc::internal::RpcServiceMethod( - MilvusService_method_names[10], - ::grpc::internal::RpcMethod::NORMAL_RPC, - new ::grpc::internal::RpcMethodHandler< MilvusService::Service, ::milvus::grpc::Command, ::milvus::grpc::StringReply>( - std::mem_fn(&MilvusService::Service::Cmd), this))); - AddMethod(new ::grpc::internal::RpcServiceMethod( - MilvusService_method_names[11], - ::grpc::internal::RpcMethod::NORMAL_RPC, - new ::grpc::internal::RpcMethodHandler< MilvusService::Service, ::milvus::grpc::DeleteByRangeParam, ::milvus::grpc::Status>( - std::mem_fn(&MilvusService::Service::DeleteByRange), this))); - AddMethod(new ::grpc::internal::RpcServiceMethod( - MilvusService_method_names[12], + MilvusService_method_names[5], ::grpc::internal::RpcMethod::NORMAL_RPC, new ::grpc::internal::RpcMethodHandler< MilvusService::Service, ::milvus::grpc::TableName, ::milvus::grpc::Status>( - std::mem_fn(&MilvusService::Service::PreloadTable), this))); + std::mem_fn(&MilvusService::Service::DropTable), this))); AddMethod(new ::grpc::internal::RpcServiceMethod( - MilvusService_method_names[13], + MilvusService_method_names[6], + ::grpc::internal::RpcMethod::NORMAL_RPC, + new ::grpc::internal::RpcMethodHandler< MilvusService::Service, ::milvus::grpc::IndexParam, ::milvus::grpc::Status>( + std::mem_fn(&MilvusService::Service::CreateIndex), this))); + AddMethod(new ::grpc::internal::RpcServiceMethod( + MilvusService_method_names[7], ::grpc::internal::RpcMethod::NORMAL_RPC, new ::grpc::internal::RpcMethodHandler< MilvusService::Service, ::milvus::grpc::TableName, ::milvus::grpc::IndexParam>( std::mem_fn(&MilvusService::Service::DescribeIndex), this))); AddMethod(new ::grpc::internal::RpcServiceMethod( - MilvusService_method_names[14], + MilvusService_method_names[8], ::grpc::internal::RpcMethod::NORMAL_RPC, new ::grpc::internal::RpcMethodHandler< MilvusService::Service, ::milvus::grpc::TableName, ::milvus::grpc::Status>( std::mem_fn(&MilvusService::Service::DropIndex), this))); + AddMethod(new ::grpc::internal::RpcServiceMethod( + MilvusService_method_names[9], + ::grpc::internal::RpcMethod::NORMAL_RPC, + new ::grpc::internal::RpcMethodHandler< MilvusService::Service, ::milvus::grpc::PartitionParam, ::milvus::grpc::Status>( + std::mem_fn(&MilvusService::Service::CreatePartition), this))); + AddMethod(new ::grpc::internal::RpcServiceMethod( + MilvusService_method_names[10], + ::grpc::internal::RpcMethod::NORMAL_RPC, + new ::grpc::internal::RpcMethodHandler< MilvusService::Service, ::milvus::grpc::TableName, ::milvus::grpc::PartitionList>( + std::mem_fn(&MilvusService::Service::ShowPartitions), this))); + AddMethod(new ::grpc::internal::RpcServiceMethod( + MilvusService_method_names[11], + ::grpc::internal::RpcMethod::NORMAL_RPC, + new ::grpc::internal::RpcMethodHandler< MilvusService::Service, ::milvus::grpc::PartitionParam, ::milvus::grpc::Status>( + std::mem_fn(&MilvusService::Service::DropPartition), this))); + AddMethod(new ::grpc::internal::RpcServiceMethod( + MilvusService_method_names[12], + ::grpc::internal::RpcMethod::NORMAL_RPC, + new ::grpc::internal::RpcMethodHandler< MilvusService::Service, ::milvus::grpc::InsertParam, ::milvus::grpc::VectorIds>( + std::mem_fn(&MilvusService::Service::Insert), this))); + AddMethod(new ::grpc::internal::RpcServiceMethod( + MilvusService_method_names[13], + ::grpc::internal::RpcMethod::NORMAL_RPC, + new ::grpc::internal::RpcMethodHandler< MilvusService::Service, ::milvus::grpc::SearchParam, ::milvus::grpc::TopKQueryResult>( + std::mem_fn(&MilvusService::Service::Search), this))); + AddMethod(new ::grpc::internal::RpcServiceMethod( + MilvusService_method_names[14], + ::grpc::internal::RpcMethod::NORMAL_RPC, + new ::grpc::internal::RpcMethodHandler< MilvusService::Service, ::milvus::grpc::SearchInFilesParam, ::milvus::grpc::TopKQueryResult>( + std::mem_fn(&MilvusService::Service::SearchInFiles), this))); + AddMethod(new ::grpc::internal::RpcServiceMethod( + MilvusService_method_names[15], + ::grpc::internal::RpcMethod::NORMAL_RPC, + new ::grpc::internal::RpcMethodHandler< MilvusService::Service, ::milvus::grpc::Command, ::milvus::grpc::StringReply>( + std::mem_fn(&MilvusService::Service::Cmd), this))); + AddMethod(new ::grpc::internal::RpcServiceMethod( + MilvusService_method_names[16], + ::grpc::internal::RpcMethod::NORMAL_RPC, + new ::grpc::internal::RpcMethodHandler< MilvusService::Service, ::milvus::grpc::DeleteByDateParam, ::milvus::grpc::Status>( + std::mem_fn(&MilvusService::Service::DeleteByDate), this))); + AddMethod(new ::grpc::internal::RpcServiceMethod( + MilvusService_method_names[17], + ::grpc::internal::RpcMethod::NORMAL_RPC, + new ::grpc::internal::RpcMethodHandler< MilvusService::Service, ::milvus::grpc::TableName, ::milvus::grpc::Status>( + std::mem_fn(&MilvusService::Service::PreloadTable), this))); } MilvusService::Service::~Service() { @@ -576,41 +681,6 @@ MilvusService::Service::~Service() { return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, ""); } -::grpc::Status MilvusService::Service::DropTable(::grpc::ServerContext* context, const ::milvus::grpc::TableName* request, ::milvus::grpc::Status* response) { - (void) context; - (void) request; - (void) response; - return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, ""); -} - -::grpc::Status MilvusService::Service::CreateIndex(::grpc::ServerContext* context, const ::milvus::grpc::IndexParam* request, ::milvus::grpc::Status* response) { - (void) context; - (void) request; - (void) response; - return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, ""); -} - -::grpc::Status MilvusService::Service::Insert(::grpc::ServerContext* context, const ::milvus::grpc::InsertParam* request, ::milvus::grpc::VectorIds* response) { - (void) context; - (void) request; - (void) response; - return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, ""); -} - -::grpc::Status MilvusService::Service::Search(::grpc::ServerContext* context, const ::milvus::grpc::SearchParam* request, ::milvus::grpc::TopKQueryResult* response) { - (void) context; - (void) request; - (void) response; - return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, ""); -} - -::grpc::Status MilvusService::Service::SearchInFiles(::grpc::ServerContext* context, const ::milvus::grpc::SearchInFilesParam* request, ::milvus::grpc::TopKQueryResult* response) { - (void) context; - (void) request; - (void) response; - return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, ""); -} - ::grpc::Status MilvusService::Service::DescribeTable(::grpc::ServerContext* context, const ::milvus::grpc::TableName* request, ::milvus::grpc::TableSchema* response) { (void) context; (void) request; @@ -632,21 +702,14 @@ MilvusService::Service::~Service() { return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, ""); } -::grpc::Status MilvusService::Service::Cmd(::grpc::ServerContext* context, const ::milvus::grpc::Command* request, ::milvus::grpc::StringReply* response) { +::grpc::Status MilvusService::Service::DropTable(::grpc::ServerContext* context, const ::milvus::grpc::TableName* request, ::milvus::grpc::Status* response) { (void) context; (void) request; (void) response; return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, ""); } -::grpc::Status MilvusService::Service::DeleteByRange(::grpc::ServerContext* context, const ::milvus::grpc::DeleteByRangeParam* request, ::milvus::grpc::Status* response) { - (void) context; - (void) request; - (void) response; - return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, ""); -} - -::grpc::Status MilvusService::Service::PreloadTable(::grpc::ServerContext* context, const ::milvus::grpc::TableName* request, ::milvus::grpc::Status* response) { +::grpc::Status MilvusService::Service::CreateIndex(::grpc::ServerContext* context, const ::milvus::grpc::IndexParam* request, ::milvus::grpc::Status* response) { (void) context; (void) request; (void) response; @@ -667,6 +730,69 @@ MilvusService::Service::~Service() { return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, ""); } +::grpc::Status MilvusService::Service::CreatePartition(::grpc::ServerContext* context, const ::milvus::grpc::PartitionParam* request, ::milvus::grpc::Status* response) { + (void) context; + (void) request; + (void) response; + return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, ""); +} + +::grpc::Status MilvusService::Service::ShowPartitions(::grpc::ServerContext* context, const ::milvus::grpc::TableName* request, ::milvus::grpc::PartitionList* response) { + (void) context; + (void) request; + (void) response; + return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, ""); +} + +::grpc::Status MilvusService::Service::DropPartition(::grpc::ServerContext* context, const ::milvus::grpc::PartitionParam* request, ::milvus::grpc::Status* response) { + (void) context; + (void) request; + (void) response; + return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, ""); +} + +::grpc::Status MilvusService::Service::Insert(::grpc::ServerContext* context, const ::milvus::grpc::InsertParam* request, ::milvus::grpc::VectorIds* response) { + (void) context; + (void) request; + (void) response; + return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, ""); +} + +::grpc::Status MilvusService::Service::Search(::grpc::ServerContext* context, const ::milvus::grpc::SearchParam* request, ::milvus::grpc::TopKQueryResult* response) { + (void) context; + (void) request; + (void) response; + return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, ""); +} + +::grpc::Status MilvusService::Service::SearchInFiles(::grpc::ServerContext* context, const ::milvus::grpc::SearchInFilesParam* request, ::milvus::grpc::TopKQueryResult* response) { + (void) context; + (void) request; + (void) response; + return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, ""); +} + +::grpc::Status MilvusService::Service::Cmd(::grpc::ServerContext* context, const ::milvus::grpc::Command* request, ::milvus::grpc::StringReply* response) { + (void) context; + (void) request; + (void) response; + return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, ""); +} + +::grpc::Status MilvusService::Service::DeleteByDate(::grpc::ServerContext* context, const ::milvus::grpc::DeleteByDateParam* request, ::milvus::grpc::Status* response) { + (void) context; + (void) request; + (void) response; + return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, ""); +} + +::grpc::Status MilvusService::Service::PreloadTable(::grpc::ServerContext* context, const ::milvus::grpc::TableName* request, ::milvus::grpc::Status* response) { + (void) context; + (void) request; + (void) response; + return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, ""); +} + } // namespace milvus } // namespace grpc diff --git a/core/src/grpc/gen-milvus/milvus.grpc.pb.h b/core/src/grpc/gen-milvus/milvus.grpc.pb.h index 4fa38c7abb..e19d90f832 100644 --- a/core/src/grpc/gen-milvus/milvus.grpc.pb.h +++ b/core/src/grpc/gen-milvus/milvus.grpc.pb.h @@ -48,12 +48,11 @@ class MilvusService final { public: virtual ~StubInterface() {} // * - // @brief Create table method + // @brief This method is used to create table // - // This method is used to create table - // - // @param param, use to provide table information to be created. + // @param TableSchema, use to provide table information to be created. // + // @return Status virtual ::grpc::Status CreateTable(::grpc::ClientContext* context, const ::milvus::grpc::TableSchema& request, ::milvus::grpc::Status* response) = 0; std::unique_ptr< ::grpc::ClientAsyncResponseReaderInterface< ::milvus::grpc::Status>> AsyncCreateTable(::grpc::ClientContext* context, const ::milvus::grpc::TableSchema& request, ::grpc::CompletionQueue* cq) { return std::unique_ptr< ::grpc::ClientAsyncResponseReaderInterface< ::milvus::grpc::Status>>(AsyncCreateTableRaw(context, request, cq)); @@ -62,12 +61,11 @@ class MilvusService final { return std::unique_ptr< ::grpc::ClientAsyncResponseReaderInterface< ::milvus::grpc::Status>>(PrepareAsyncCreateTableRaw(context, request, cq)); } // * - // @brief Test table existence method + // @brief This method is used to test table existence. // - // This method is used to test table existence. - // - // @param table_name, table name is going to be tested. + // @param TableName, table name is going to be tested. // + // @return BoolReply virtual ::grpc::Status HasTable(::grpc::ClientContext* context, const ::milvus::grpc::TableName& request, ::milvus::grpc::BoolReply* response) = 0; std::unique_ptr< ::grpc::ClientAsyncResponseReaderInterface< ::milvus::grpc::BoolReply>> AsyncHasTable(::grpc::ClientContext* context, const ::milvus::grpc::TableName& request, ::grpc::CompletionQueue* cq) { return std::unique_ptr< ::grpc::ClientAsyncResponseReaderInterface< ::milvus::grpc::BoolReply>>(AsyncHasTableRaw(context, request, cq)); @@ -76,93 +74,11 @@ class MilvusService final { return std::unique_ptr< ::grpc::ClientAsyncResponseReaderInterface< ::milvus::grpc::BoolReply>>(PrepareAsyncHasTableRaw(context, request, cq)); } // * - // @brief Delete table method + // @brief This method is used to get table schema. // - // This method is used to delete table. + // @param TableName, target table name. // - // @param table_name, table name is going to be deleted. - // - virtual ::grpc::Status DropTable(::grpc::ClientContext* context, const ::milvus::grpc::TableName& request, ::milvus::grpc::Status* response) = 0; - std::unique_ptr< ::grpc::ClientAsyncResponseReaderInterface< ::milvus::grpc::Status>> AsyncDropTable(::grpc::ClientContext* context, const ::milvus::grpc::TableName& request, ::grpc::CompletionQueue* cq) { - return std::unique_ptr< ::grpc::ClientAsyncResponseReaderInterface< ::milvus::grpc::Status>>(AsyncDropTableRaw(context, request, cq)); - } - std::unique_ptr< ::grpc::ClientAsyncResponseReaderInterface< ::milvus::grpc::Status>> PrepareAsyncDropTable(::grpc::ClientContext* context, const ::milvus::grpc::TableName& request, ::grpc::CompletionQueue* cq) { - return std::unique_ptr< ::grpc::ClientAsyncResponseReaderInterface< ::milvus::grpc::Status>>(PrepareAsyncDropTableRaw(context, request, cq)); - } - // * - // @brief Build index by table method - // - // This method is used to build index by table in sync mode. - // - // @param table_name, table is going to be built index. - // - virtual ::grpc::Status CreateIndex(::grpc::ClientContext* context, const ::milvus::grpc::IndexParam& request, ::milvus::grpc::Status* response) = 0; - std::unique_ptr< ::grpc::ClientAsyncResponseReaderInterface< ::milvus::grpc::Status>> AsyncCreateIndex(::grpc::ClientContext* context, const ::milvus::grpc::IndexParam& request, ::grpc::CompletionQueue* cq) { - return std::unique_ptr< ::grpc::ClientAsyncResponseReaderInterface< ::milvus::grpc::Status>>(AsyncCreateIndexRaw(context, request, cq)); - } - std::unique_ptr< ::grpc::ClientAsyncResponseReaderInterface< ::milvus::grpc::Status>> PrepareAsyncCreateIndex(::grpc::ClientContext* context, const ::milvus::grpc::IndexParam& request, ::grpc::CompletionQueue* cq) { - return std::unique_ptr< ::grpc::ClientAsyncResponseReaderInterface< ::milvus::grpc::Status>>(PrepareAsyncCreateIndexRaw(context, request, cq)); - } - // * - // @brief Add vector array to table - // - // This method is used to add vector array to table. - // - // @param table_name, table_name is inserted. - // @param record_array, vector array is inserted. - // - // @return vector id array - virtual ::grpc::Status Insert(::grpc::ClientContext* context, const ::milvus::grpc::InsertParam& request, ::milvus::grpc::VectorIds* response) = 0; - std::unique_ptr< ::grpc::ClientAsyncResponseReaderInterface< ::milvus::grpc::VectorIds>> AsyncInsert(::grpc::ClientContext* context, const ::milvus::grpc::InsertParam& request, ::grpc::CompletionQueue* cq) { - return std::unique_ptr< ::grpc::ClientAsyncResponseReaderInterface< ::milvus::grpc::VectorIds>>(AsyncInsertRaw(context, request, cq)); - } - std::unique_ptr< ::grpc::ClientAsyncResponseReaderInterface< ::milvus::grpc::VectorIds>> PrepareAsyncInsert(::grpc::ClientContext* context, const ::milvus::grpc::InsertParam& request, ::grpc::CompletionQueue* cq) { - return std::unique_ptr< ::grpc::ClientAsyncResponseReaderInterface< ::milvus::grpc::VectorIds>>(PrepareAsyncInsertRaw(context, request, cq)); - } - // * - // @brief Query vector - // - // This method is used to query vector in table. - // - // @param table_name, table_name is queried. - // @param query_record_array, all vector are going to be queried. - // @param query_range_array, optional ranges for conditional search. If not specified, search whole table - // @param topk, how many similarity vectors will be searched. - // - // @return query result array. - virtual ::grpc::Status Search(::grpc::ClientContext* context, const ::milvus::grpc::SearchParam& request, ::milvus::grpc::TopKQueryResult* response) = 0; - std::unique_ptr< ::grpc::ClientAsyncResponseReaderInterface< ::milvus::grpc::TopKQueryResult>> AsyncSearch(::grpc::ClientContext* context, const ::milvus::grpc::SearchParam& request, ::grpc::CompletionQueue* cq) { - return std::unique_ptr< ::grpc::ClientAsyncResponseReaderInterface< ::milvus::grpc::TopKQueryResult>>(AsyncSearchRaw(context, request, cq)); - } - std::unique_ptr< ::grpc::ClientAsyncResponseReaderInterface< ::milvus::grpc::TopKQueryResult>> PrepareAsyncSearch(::grpc::ClientContext* context, const ::milvus::grpc::SearchParam& request, ::grpc::CompletionQueue* cq) { - return std::unique_ptr< ::grpc::ClientAsyncResponseReaderInterface< ::milvus::grpc::TopKQueryResult>>(PrepareAsyncSearchRaw(context, request, cq)); - } - // * - // @brief Internal use query interface - // - // This method is used to query vector in specified files. - // - // @param file_id_array, specified files id array, queried. - // @param query_record_array, all vector are going to be queried. - // @param query_range_array, optional ranges for conditional search. If not specified, search whole table - // @param topk, how many similarity vectors will be searched. - // - // @return query result array. - virtual ::grpc::Status SearchInFiles(::grpc::ClientContext* context, const ::milvus::grpc::SearchInFilesParam& request, ::milvus::grpc::TopKQueryResult* response) = 0; - std::unique_ptr< ::grpc::ClientAsyncResponseReaderInterface< ::milvus::grpc::TopKQueryResult>> AsyncSearchInFiles(::grpc::ClientContext* context, const ::milvus::grpc::SearchInFilesParam& request, ::grpc::CompletionQueue* cq) { - return std::unique_ptr< ::grpc::ClientAsyncResponseReaderInterface< ::milvus::grpc::TopKQueryResult>>(AsyncSearchInFilesRaw(context, request, cq)); - } - std::unique_ptr< ::grpc::ClientAsyncResponseReaderInterface< ::milvus::grpc::TopKQueryResult>> PrepareAsyncSearchInFiles(::grpc::ClientContext* context, const ::milvus::grpc::SearchInFilesParam& request, ::grpc::CompletionQueue* cq) { - return std::unique_ptr< ::grpc::ClientAsyncResponseReaderInterface< ::milvus::grpc::TopKQueryResult>>(PrepareAsyncSearchInFilesRaw(context, request, cq)); - } - // * - // @brief Get table schema - // - // This method is used to get table schema. - // - // @param table_name, target table name. - // - // @return table schema + // @return TableSchema virtual ::grpc::Status DescribeTable(::grpc::ClientContext* context, const ::milvus::grpc::TableName& request, ::milvus::grpc::TableSchema* response) = 0; std::unique_ptr< ::grpc::ClientAsyncResponseReaderInterface< ::milvus::grpc::TableSchema>> AsyncDescribeTable(::grpc::ClientContext* context, const ::milvus::grpc::TableName& request, ::grpc::CompletionQueue* cq) { return std::unique_ptr< ::grpc::ClientAsyncResponseReaderInterface< ::milvus::grpc::TableSchema>>(AsyncDescribeTableRaw(context, request, cq)); @@ -171,13 +87,11 @@ class MilvusService final { return std::unique_ptr< ::grpc::ClientAsyncResponseReaderInterface< ::milvus::grpc::TableSchema>>(PrepareAsyncDescribeTableRaw(context, request, cq)); } // * - // @brief Get table schema + // @brief This method is used to get table schema. // - // This method is used to get table schema. + // @param TableName, target table name. // - // @param table_name, target table name. - // - // @return table schema + // @return TableRowCount virtual ::grpc::Status CountTable(::grpc::ClientContext* context, const ::milvus::grpc::TableName& request, ::milvus::grpc::TableRowCount* response) = 0; std::unique_ptr< ::grpc::ClientAsyncResponseReaderInterface< ::milvus::grpc::TableRowCount>> AsyncCountTable(::grpc::ClientContext* context, const ::milvus::grpc::TableName& request, ::grpc::CompletionQueue* cq) { return std::unique_ptr< ::grpc::ClientAsyncResponseReaderInterface< ::milvus::grpc::TableRowCount>>(AsyncCountTableRaw(context, request, cq)); @@ -186,12 +100,11 @@ class MilvusService final { return std::unique_ptr< ::grpc::ClientAsyncResponseReaderInterface< ::milvus::grpc::TableRowCount>>(PrepareAsyncCountTableRaw(context, request, cq)); } // * - // @brief List all tables in database + // @brief This method is used to list all tables. // - // This method is used to list all tables. + // @param Command, dummy parameter. // - // - // @return table names. + // @return TableNameList virtual ::grpc::Status ShowTables(::grpc::ClientContext* context, const ::milvus::grpc::Command& request, ::milvus::grpc::TableNameList* response) = 0; std::unique_ptr< ::grpc::ClientAsyncResponseReaderInterface< ::milvus::grpc::TableNameList>> AsyncShowTables(::grpc::ClientContext* context, const ::milvus::grpc::Command& request, ::grpc::CompletionQueue* cq) { return std::unique_ptr< ::grpc::ClientAsyncResponseReaderInterface< ::milvus::grpc::TableNameList>>(AsyncShowTablesRaw(context, request, cq)); @@ -200,50 +113,37 @@ class MilvusService final { return std::unique_ptr< ::grpc::ClientAsyncResponseReaderInterface< ::milvus::grpc::TableNameList>>(PrepareAsyncShowTablesRaw(context, request, cq)); } // * - // @brief Give the server status + // @brief This method is used to delete table. // - // This method is used to give the server status. + // @param TableName, table name is going to be deleted. // - // @return Server status. - virtual ::grpc::Status Cmd(::grpc::ClientContext* context, const ::milvus::grpc::Command& request, ::milvus::grpc::StringReply* response) = 0; - std::unique_ptr< ::grpc::ClientAsyncResponseReaderInterface< ::milvus::grpc::StringReply>> AsyncCmd(::grpc::ClientContext* context, const ::milvus::grpc::Command& request, ::grpc::CompletionQueue* cq) { - return std::unique_ptr< ::grpc::ClientAsyncResponseReaderInterface< ::milvus::grpc::StringReply>>(AsyncCmdRaw(context, request, cq)); + // @return TableNameList + virtual ::grpc::Status DropTable(::grpc::ClientContext* context, const ::milvus::grpc::TableName& request, ::milvus::grpc::Status* response) = 0; + std::unique_ptr< ::grpc::ClientAsyncResponseReaderInterface< ::milvus::grpc::Status>> AsyncDropTable(::grpc::ClientContext* context, const ::milvus::grpc::TableName& request, ::grpc::CompletionQueue* cq) { + return std::unique_ptr< ::grpc::ClientAsyncResponseReaderInterface< ::milvus::grpc::Status>>(AsyncDropTableRaw(context, request, cq)); } - std::unique_ptr< ::grpc::ClientAsyncResponseReaderInterface< ::milvus::grpc::StringReply>> PrepareAsyncCmd(::grpc::ClientContext* context, const ::milvus::grpc::Command& request, ::grpc::CompletionQueue* cq) { - return std::unique_ptr< ::grpc::ClientAsyncResponseReaderInterface< ::milvus::grpc::StringReply>>(PrepareAsyncCmdRaw(context, request, cq)); + std::unique_ptr< ::grpc::ClientAsyncResponseReaderInterface< ::milvus::grpc::Status>> PrepareAsyncDropTable(::grpc::ClientContext* context, const ::milvus::grpc::TableName& request, ::grpc::CompletionQueue* cq) { + return std::unique_ptr< ::grpc::ClientAsyncResponseReaderInterface< ::milvus::grpc::Status>>(PrepareAsyncDropTableRaw(context, request, cq)); } // * - // @brief delete table by range + // @brief This method is used to build index by table in sync mode. // - // This method is used to delete vector by range + // @param IndexParam, index paramters. // - // @return rpc status. - virtual ::grpc::Status DeleteByRange(::grpc::ClientContext* context, const ::milvus::grpc::DeleteByRangeParam& request, ::milvus::grpc::Status* response) = 0; - std::unique_ptr< ::grpc::ClientAsyncResponseReaderInterface< ::milvus::grpc::Status>> AsyncDeleteByRange(::grpc::ClientContext* context, const ::milvus::grpc::DeleteByRangeParam& request, ::grpc::CompletionQueue* cq) { - return std::unique_ptr< ::grpc::ClientAsyncResponseReaderInterface< ::milvus::grpc::Status>>(AsyncDeleteByRangeRaw(context, request, cq)); + // @return Status + virtual ::grpc::Status CreateIndex(::grpc::ClientContext* context, const ::milvus::grpc::IndexParam& request, ::milvus::grpc::Status* response) = 0; + std::unique_ptr< ::grpc::ClientAsyncResponseReaderInterface< ::milvus::grpc::Status>> AsyncCreateIndex(::grpc::ClientContext* context, const ::milvus::grpc::IndexParam& request, ::grpc::CompletionQueue* cq) { + return std::unique_ptr< ::grpc::ClientAsyncResponseReaderInterface< ::milvus::grpc::Status>>(AsyncCreateIndexRaw(context, request, cq)); } - std::unique_ptr< ::grpc::ClientAsyncResponseReaderInterface< ::milvus::grpc::Status>> PrepareAsyncDeleteByRange(::grpc::ClientContext* context, const ::milvus::grpc::DeleteByRangeParam& request, ::grpc::CompletionQueue* cq) { - return std::unique_ptr< ::grpc::ClientAsyncResponseReaderInterface< ::milvus::grpc::Status>>(PrepareAsyncDeleteByRangeRaw(context, request, cq)); + std::unique_ptr< ::grpc::ClientAsyncResponseReaderInterface< ::milvus::grpc::Status>> PrepareAsyncCreateIndex(::grpc::ClientContext* context, const ::milvus::grpc::IndexParam& request, ::grpc::CompletionQueue* cq) { + return std::unique_ptr< ::grpc::ClientAsyncResponseReaderInterface< ::milvus::grpc::Status>>(PrepareAsyncCreateIndexRaw(context, request, cq)); } // * - // @brief preload table + // @brief This method is used to describe index // - // This method is used to preload table + // @param TableName, target table name. // - // @return Status. - virtual ::grpc::Status PreloadTable(::grpc::ClientContext* context, const ::milvus::grpc::TableName& request, ::milvus::grpc::Status* response) = 0; - std::unique_ptr< ::grpc::ClientAsyncResponseReaderInterface< ::milvus::grpc::Status>> AsyncPreloadTable(::grpc::ClientContext* context, const ::milvus::grpc::TableName& request, ::grpc::CompletionQueue* cq) { - return std::unique_ptr< ::grpc::ClientAsyncResponseReaderInterface< ::milvus::grpc::Status>>(AsyncPreloadTableRaw(context, request, cq)); - } - std::unique_ptr< ::grpc::ClientAsyncResponseReaderInterface< ::milvus::grpc::Status>> PrepareAsyncPreloadTable(::grpc::ClientContext* context, const ::milvus::grpc::TableName& request, ::grpc::CompletionQueue* cq) { - return std::unique_ptr< ::grpc::ClientAsyncResponseReaderInterface< ::milvus::grpc::Status>>(PrepareAsyncPreloadTableRaw(context, request, cq)); - } - // * - // @brief describe index - // - // This method is used to describe index - // - // @return Status. + // @return IndexParam virtual ::grpc::Status DescribeIndex(::grpc::ClientContext* context, const ::milvus::grpc::TableName& request, ::milvus::grpc::IndexParam* response) = 0; std::unique_ptr< ::grpc::ClientAsyncResponseReaderInterface< ::milvus::grpc::IndexParam>> AsyncDescribeIndex(::grpc::ClientContext* context, const ::milvus::grpc::TableName& request, ::grpc::CompletionQueue* cq) { return std::unique_ptr< ::grpc::ClientAsyncResponseReaderInterface< ::milvus::grpc::IndexParam>>(AsyncDescribeIndexRaw(context, request, cq)); @@ -252,11 +152,11 @@ class MilvusService final { return std::unique_ptr< ::grpc::ClientAsyncResponseReaderInterface< ::milvus::grpc::IndexParam>>(PrepareAsyncDescribeIndexRaw(context, request, cq)); } // * - // @brief drop index + // @brief This method is used to drop index // - // This method is used to drop index + // @param TableName, target table name. // - // @return Status. + // @return Status virtual ::grpc::Status DropIndex(::grpc::ClientContext* context, const ::milvus::grpc::TableName& request, ::milvus::grpc::Status* response) = 0; std::unique_ptr< ::grpc::ClientAsyncResponseReaderInterface< ::milvus::grpc::Status>> AsyncDropIndex(::grpc::ClientContext* context, const ::milvus::grpc::TableName& request, ::grpc::CompletionQueue* cq) { return std::unique_ptr< ::grpc::ClientAsyncResponseReaderInterface< ::milvus::grpc::Status>>(AsyncDropIndexRaw(context, request, cq)); @@ -264,181 +164,306 @@ class MilvusService final { std::unique_ptr< ::grpc::ClientAsyncResponseReaderInterface< ::milvus::grpc::Status>> PrepareAsyncDropIndex(::grpc::ClientContext* context, const ::milvus::grpc::TableName& request, ::grpc::CompletionQueue* cq) { return std::unique_ptr< ::grpc::ClientAsyncResponseReaderInterface< ::milvus::grpc::Status>>(PrepareAsyncDropIndexRaw(context, request, cq)); } + // * + // @brief This method is used to create partition + // + // @param PartitionParam, partition parameters. + // + // @return Status + virtual ::grpc::Status CreatePartition(::grpc::ClientContext* context, const ::milvus::grpc::PartitionParam& request, ::milvus::grpc::Status* response) = 0; + std::unique_ptr< ::grpc::ClientAsyncResponseReaderInterface< ::milvus::grpc::Status>> AsyncCreatePartition(::grpc::ClientContext* context, const ::milvus::grpc::PartitionParam& request, ::grpc::CompletionQueue* cq) { + return std::unique_ptr< ::grpc::ClientAsyncResponseReaderInterface< ::milvus::grpc::Status>>(AsyncCreatePartitionRaw(context, request, cq)); + } + std::unique_ptr< ::grpc::ClientAsyncResponseReaderInterface< ::milvus::grpc::Status>> PrepareAsyncCreatePartition(::grpc::ClientContext* context, const ::milvus::grpc::PartitionParam& request, ::grpc::CompletionQueue* cq) { + return std::unique_ptr< ::grpc::ClientAsyncResponseReaderInterface< ::milvus::grpc::Status>>(PrepareAsyncCreatePartitionRaw(context, request, cq)); + } + // * + // @brief This method is used to show partition information + // + // @param TableName, target table name. + // + // @return PartitionList + virtual ::grpc::Status ShowPartitions(::grpc::ClientContext* context, const ::milvus::grpc::TableName& request, ::milvus::grpc::PartitionList* response) = 0; + std::unique_ptr< ::grpc::ClientAsyncResponseReaderInterface< ::milvus::grpc::PartitionList>> AsyncShowPartitions(::grpc::ClientContext* context, const ::milvus::grpc::TableName& request, ::grpc::CompletionQueue* cq) { + return std::unique_ptr< ::grpc::ClientAsyncResponseReaderInterface< ::milvus::grpc::PartitionList>>(AsyncShowPartitionsRaw(context, request, cq)); + } + std::unique_ptr< ::grpc::ClientAsyncResponseReaderInterface< ::milvus::grpc::PartitionList>> PrepareAsyncShowPartitions(::grpc::ClientContext* context, const ::milvus::grpc::TableName& request, ::grpc::CompletionQueue* cq) { + return std::unique_ptr< ::grpc::ClientAsyncResponseReaderInterface< ::milvus::grpc::PartitionList>>(PrepareAsyncShowPartitionsRaw(context, request, cq)); + } + // * + // @brief This method is used to drop partition + // + // @param PartitionParam, target partition. + // + // @return Status + virtual ::grpc::Status DropPartition(::grpc::ClientContext* context, const ::milvus::grpc::PartitionParam& request, ::milvus::grpc::Status* response) = 0; + std::unique_ptr< ::grpc::ClientAsyncResponseReaderInterface< ::milvus::grpc::Status>> AsyncDropPartition(::grpc::ClientContext* context, const ::milvus::grpc::PartitionParam& request, ::grpc::CompletionQueue* cq) { + return std::unique_ptr< ::grpc::ClientAsyncResponseReaderInterface< ::milvus::grpc::Status>>(AsyncDropPartitionRaw(context, request, cq)); + } + std::unique_ptr< ::grpc::ClientAsyncResponseReaderInterface< ::milvus::grpc::Status>> PrepareAsyncDropPartition(::grpc::ClientContext* context, const ::milvus::grpc::PartitionParam& request, ::grpc::CompletionQueue* cq) { + return std::unique_ptr< ::grpc::ClientAsyncResponseReaderInterface< ::milvus::grpc::Status>>(PrepareAsyncDropPartitionRaw(context, request, cq)); + } + // * + // @brief This method is used to add vector array to table. + // + // @param InsertParam, insert parameters. + // + // @return VectorIds + virtual ::grpc::Status Insert(::grpc::ClientContext* context, const ::milvus::grpc::InsertParam& request, ::milvus::grpc::VectorIds* response) = 0; + std::unique_ptr< ::grpc::ClientAsyncResponseReaderInterface< ::milvus::grpc::VectorIds>> AsyncInsert(::grpc::ClientContext* context, const ::milvus::grpc::InsertParam& request, ::grpc::CompletionQueue* cq) { + return std::unique_ptr< ::grpc::ClientAsyncResponseReaderInterface< ::milvus::grpc::VectorIds>>(AsyncInsertRaw(context, request, cq)); + } + std::unique_ptr< ::grpc::ClientAsyncResponseReaderInterface< ::milvus::grpc::VectorIds>> PrepareAsyncInsert(::grpc::ClientContext* context, const ::milvus::grpc::InsertParam& request, ::grpc::CompletionQueue* cq) { + return std::unique_ptr< ::grpc::ClientAsyncResponseReaderInterface< ::milvus::grpc::VectorIds>>(PrepareAsyncInsertRaw(context, request, cq)); + } + // * + // @brief This method is used to query vector in table. + // + // @param SearchParam, search parameters. + // + // @return TopKQueryResult + virtual ::grpc::Status Search(::grpc::ClientContext* context, const ::milvus::grpc::SearchParam& request, ::milvus::grpc::TopKQueryResult* response) = 0; + std::unique_ptr< ::grpc::ClientAsyncResponseReaderInterface< ::milvus::grpc::TopKQueryResult>> AsyncSearch(::grpc::ClientContext* context, const ::milvus::grpc::SearchParam& request, ::grpc::CompletionQueue* cq) { + return std::unique_ptr< ::grpc::ClientAsyncResponseReaderInterface< ::milvus::grpc::TopKQueryResult>>(AsyncSearchRaw(context, request, cq)); + } + std::unique_ptr< ::grpc::ClientAsyncResponseReaderInterface< ::milvus::grpc::TopKQueryResult>> PrepareAsyncSearch(::grpc::ClientContext* context, const ::milvus::grpc::SearchParam& request, ::grpc::CompletionQueue* cq) { + return std::unique_ptr< ::grpc::ClientAsyncResponseReaderInterface< ::milvus::grpc::TopKQueryResult>>(PrepareAsyncSearchRaw(context, request, cq)); + } + // * + // @brief This method is used to query vector in specified files. + // + // @param SearchInFilesParam, search in files paremeters. + // + // @return TopKQueryResult + virtual ::grpc::Status SearchInFiles(::grpc::ClientContext* context, const ::milvus::grpc::SearchInFilesParam& request, ::milvus::grpc::TopKQueryResult* response) = 0; + std::unique_ptr< ::grpc::ClientAsyncResponseReaderInterface< ::milvus::grpc::TopKQueryResult>> AsyncSearchInFiles(::grpc::ClientContext* context, const ::milvus::grpc::SearchInFilesParam& request, ::grpc::CompletionQueue* cq) { + return std::unique_ptr< ::grpc::ClientAsyncResponseReaderInterface< ::milvus::grpc::TopKQueryResult>>(AsyncSearchInFilesRaw(context, request, cq)); + } + std::unique_ptr< ::grpc::ClientAsyncResponseReaderInterface< ::milvus::grpc::TopKQueryResult>> PrepareAsyncSearchInFiles(::grpc::ClientContext* context, const ::milvus::grpc::SearchInFilesParam& request, ::grpc::CompletionQueue* cq) { + return std::unique_ptr< ::grpc::ClientAsyncResponseReaderInterface< ::milvus::grpc::TopKQueryResult>>(PrepareAsyncSearchInFilesRaw(context, request, cq)); + } + // * + // @brief This method is used to give the server status. + // + // @param Command, command string + // + // @return StringReply + virtual ::grpc::Status Cmd(::grpc::ClientContext* context, const ::milvus::grpc::Command& request, ::milvus::grpc::StringReply* response) = 0; + std::unique_ptr< ::grpc::ClientAsyncResponseReaderInterface< ::milvus::grpc::StringReply>> AsyncCmd(::grpc::ClientContext* context, const ::milvus::grpc::Command& request, ::grpc::CompletionQueue* cq) { + return std::unique_ptr< ::grpc::ClientAsyncResponseReaderInterface< ::milvus::grpc::StringReply>>(AsyncCmdRaw(context, request, cq)); + } + std::unique_ptr< ::grpc::ClientAsyncResponseReaderInterface< ::milvus::grpc::StringReply>> PrepareAsyncCmd(::grpc::ClientContext* context, const ::milvus::grpc::Command& request, ::grpc::CompletionQueue* cq) { + return std::unique_ptr< ::grpc::ClientAsyncResponseReaderInterface< ::milvus::grpc::StringReply>>(PrepareAsyncCmdRaw(context, request, cq)); + } + // * + // @brief This method is used to delete vector by date range + // + // @param DeleteByDateParam, delete parameters. + // + // @return status + virtual ::grpc::Status DeleteByDate(::grpc::ClientContext* context, const ::milvus::grpc::DeleteByDateParam& request, ::milvus::grpc::Status* response) = 0; + std::unique_ptr< ::grpc::ClientAsyncResponseReaderInterface< ::milvus::grpc::Status>> AsyncDeleteByDate(::grpc::ClientContext* context, const ::milvus::grpc::DeleteByDateParam& request, ::grpc::CompletionQueue* cq) { + return std::unique_ptr< ::grpc::ClientAsyncResponseReaderInterface< ::milvus::grpc::Status>>(AsyncDeleteByDateRaw(context, request, cq)); + } + std::unique_ptr< ::grpc::ClientAsyncResponseReaderInterface< ::milvus::grpc::Status>> PrepareAsyncDeleteByDate(::grpc::ClientContext* context, const ::milvus::grpc::DeleteByDateParam& request, ::grpc::CompletionQueue* cq) { + return std::unique_ptr< ::grpc::ClientAsyncResponseReaderInterface< ::milvus::grpc::Status>>(PrepareAsyncDeleteByDateRaw(context, request, cq)); + } + // * + // @brief This method is used to preload table + // + // @param TableName, target table name. + // + // @return Status + virtual ::grpc::Status PreloadTable(::grpc::ClientContext* context, const ::milvus::grpc::TableName& request, ::milvus::grpc::Status* response) = 0; + std::unique_ptr< ::grpc::ClientAsyncResponseReaderInterface< ::milvus::grpc::Status>> AsyncPreloadTable(::grpc::ClientContext* context, const ::milvus::grpc::TableName& request, ::grpc::CompletionQueue* cq) { + return std::unique_ptr< ::grpc::ClientAsyncResponseReaderInterface< ::milvus::grpc::Status>>(AsyncPreloadTableRaw(context, request, cq)); + } + std::unique_ptr< ::grpc::ClientAsyncResponseReaderInterface< ::milvus::grpc::Status>> PrepareAsyncPreloadTable(::grpc::ClientContext* context, const ::milvus::grpc::TableName& request, ::grpc::CompletionQueue* cq) { + return std::unique_ptr< ::grpc::ClientAsyncResponseReaderInterface< ::milvus::grpc::Status>>(PrepareAsyncPreloadTableRaw(context, request, cq)); + } class experimental_async_interface { public: virtual ~experimental_async_interface() {} // * - // @brief Create table method + // @brief This method is used to create table // - // This method is used to create table - // - // @param param, use to provide table information to be created. + // @param TableSchema, use to provide table information to be created. // + // @return Status virtual void CreateTable(::grpc::ClientContext* context, const ::milvus::grpc::TableSchema* request, ::milvus::grpc::Status* response, std::function) = 0; virtual void CreateTable(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::Status* response, std::function) = 0; virtual void CreateTable(::grpc::ClientContext* context, const ::milvus::grpc::TableSchema* request, ::milvus::grpc::Status* response, ::grpc::experimental::ClientUnaryReactor* reactor) = 0; virtual void CreateTable(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::Status* response, ::grpc::experimental::ClientUnaryReactor* reactor) = 0; // * - // @brief Test table existence method + // @brief This method is used to test table existence. // - // This method is used to test table existence. - // - // @param table_name, table name is going to be tested. + // @param TableName, table name is going to be tested. // + // @return BoolReply virtual void HasTable(::grpc::ClientContext* context, const ::milvus::grpc::TableName* request, ::milvus::grpc::BoolReply* response, std::function) = 0; virtual void HasTable(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::BoolReply* response, std::function) = 0; virtual void HasTable(::grpc::ClientContext* context, const ::milvus::grpc::TableName* request, ::milvus::grpc::BoolReply* response, ::grpc::experimental::ClientUnaryReactor* reactor) = 0; virtual void HasTable(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::BoolReply* response, ::grpc::experimental::ClientUnaryReactor* reactor) = 0; // * - // @brief Delete table method + // @brief This method is used to get table schema. // - // This method is used to delete table. + // @param TableName, target table name. // - // @param table_name, table name is going to be deleted. - // - virtual void DropTable(::grpc::ClientContext* context, const ::milvus::grpc::TableName* request, ::milvus::grpc::Status* response, std::function) = 0; - virtual void DropTable(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::Status* response, std::function) = 0; - virtual void DropTable(::grpc::ClientContext* context, const ::milvus::grpc::TableName* request, ::milvus::grpc::Status* response, ::grpc::experimental::ClientUnaryReactor* reactor) = 0; - virtual void DropTable(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::Status* response, ::grpc::experimental::ClientUnaryReactor* reactor) = 0; - // * - // @brief Build index by table method - // - // This method is used to build index by table in sync mode. - // - // @param table_name, table is going to be built index. - // - virtual void CreateIndex(::grpc::ClientContext* context, const ::milvus::grpc::IndexParam* request, ::milvus::grpc::Status* response, std::function) = 0; - virtual void CreateIndex(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::Status* response, std::function) = 0; - virtual void CreateIndex(::grpc::ClientContext* context, const ::milvus::grpc::IndexParam* request, ::milvus::grpc::Status* response, ::grpc::experimental::ClientUnaryReactor* reactor) = 0; - virtual void CreateIndex(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::Status* response, ::grpc::experimental::ClientUnaryReactor* reactor) = 0; - // * - // @brief Add vector array to table - // - // This method is used to add vector array to table. - // - // @param table_name, table_name is inserted. - // @param record_array, vector array is inserted. - // - // @return vector id array - virtual void Insert(::grpc::ClientContext* context, const ::milvus::grpc::InsertParam* request, ::milvus::grpc::VectorIds* response, std::function) = 0; - virtual void Insert(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::VectorIds* response, std::function) = 0; - virtual void Insert(::grpc::ClientContext* context, const ::milvus::grpc::InsertParam* request, ::milvus::grpc::VectorIds* response, ::grpc::experimental::ClientUnaryReactor* reactor) = 0; - virtual void Insert(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::VectorIds* response, ::grpc::experimental::ClientUnaryReactor* reactor) = 0; - // * - // @brief Query vector - // - // This method is used to query vector in table. - // - // @param table_name, table_name is queried. - // @param query_record_array, all vector are going to be queried. - // @param query_range_array, optional ranges for conditional search. If not specified, search whole table - // @param topk, how many similarity vectors will be searched. - // - // @return query result array. - virtual void Search(::grpc::ClientContext* context, const ::milvus::grpc::SearchParam* request, ::milvus::grpc::TopKQueryResult* response, std::function) = 0; - virtual void Search(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::TopKQueryResult* response, std::function) = 0; - virtual void Search(::grpc::ClientContext* context, const ::milvus::grpc::SearchParam* request, ::milvus::grpc::TopKQueryResult* response, ::grpc::experimental::ClientUnaryReactor* reactor) = 0; - virtual void Search(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::TopKQueryResult* response, ::grpc::experimental::ClientUnaryReactor* reactor) = 0; - // * - // @brief Internal use query interface - // - // This method is used to query vector in specified files. - // - // @param file_id_array, specified files id array, queried. - // @param query_record_array, all vector are going to be queried. - // @param query_range_array, optional ranges for conditional search. If not specified, search whole table - // @param topk, how many similarity vectors will be searched. - // - // @return query result array. - virtual void SearchInFiles(::grpc::ClientContext* context, const ::milvus::grpc::SearchInFilesParam* request, ::milvus::grpc::TopKQueryResult* response, std::function) = 0; - virtual void SearchInFiles(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::TopKQueryResult* response, std::function) = 0; - virtual void SearchInFiles(::grpc::ClientContext* context, const ::milvus::grpc::SearchInFilesParam* request, ::milvus::grpc::TopKQueryResult* response, ::grpc::experimental::ClientUnaryReactor* reactor) = 0; - virtual void SearchInFiles(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::TopKQueryResult* response, ::grpc::experimental::ClientUnaryReactor* reactor) = 0; - // * - // @brief Get table schema - // - // This method is used to get table schema. - // - // @param table_name, target table name. - // - // @return table schema + // @return TableSchema virtual void DescribeTable(::grpc::ClientContext* context, const ::milvus::grpc::TableName* request, ::milvus::grpc::TableSchema* response, std::function) = 0; virtual void DescribeTable(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::TableSchema* response, std::function) = 0; virtual void DescribeTable(::grpc::ClientContext* context, const ::milvus::grpc::TableName* request, ::milvus::grpc::TableSchema* response, ::grpc::experimental::ClientUnaryReactor* reactor) = 0; virtual void DescribeTable(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::TableSchema* response, ::grpc::experimental::ClientUnaryReactor* reactor) = 0; // * - // @brief Get table schema + // @brief This method is used to get table schema. // - // This method is used to get table schema. + // @param TableName, target table name. // - // @param table_name, target table name. - // - // @return table schema + // @return TableRowCount virtual void CountTable(::grpc::ClientContext* context, const ::milvus::grpc::TableName* request, ::milvus::grpc::TableRowCount* response, std::function) = 0; virtual void CountTable(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::TableRowCount* response, std::function) = 0; virtual void CountTable(::grpc::ClientContext* context, const ::milvus::grpc::TableName* request, ::milvus::grpc::TableRowCount* response, ::grpc::experimental::ClientUnaryReactor* reactor) = 0; virtual void CountTable(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::TableRowCount* response, ::grpc::experimental::ClientUnaryReactor* reactor) = 0; // * - // @brief List all tables in database + // @brief This method is used to list all tables. // - // This method is used to list all tables. + // @param Command, dummy parameter. // - // - // @return table names. + // @return TableNameList virtual void ShowTables(::grpc::ClientContext* context, const ::milvus::grpc::Command* request, ::milvus::grpc::TableNameList* response, std::function) = 0; virtual void ShowTables(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::TableNameList* response, std::function) = 0; virtual void ShowTables(::grpc::ClientContext* context, const ::milvus::grpc::Command* request, ::milvus::grpc::TableNameList* response, ::grpc::experimental::ClientUnaryReactor* reactor) = 0; virtual void ShowTables(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::TableNameList* response, ::grpc::experimental::ClientUnaryReactor* reactor) = 0; // * - // @brief Give the server status + // @brief This method is used to delete table. // - // This method is used to give the server status. + // @param TableName, table name is going to be deleted. // - // @return Server status. - virtual void Cmd(::grpc::ClientContext* context, const ::milvus::grpc::Command* request, ::milvus::grpc::StringReply* response, std::function) = 0; - virtual void Cmd(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::StringReply* response, std::function) = 0; - virtual void Cmd(::grpc::ClientContext* context, const ::milvus::grpc::Command* request, ::milvus::grpc::StringReply* response, ::grpc::experimental::ClientUnaryReactor* reactor) = 0; - virtual void Cmd(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::StringReply* response, ::grpc::experimental::ClientUnaryReactor* reactor) = 0; + // @return TableNameList + virtual void DropTable(::grpc::ClientContext* context, const ::milvus::grpc::TableName* request, ::milvus::grpc::Status* response, std::function) = 0; + virtual void DropTable(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::Status* response, std::function) = 0; + virtual void DropTable(::grpc::ClientContext* context, const ::milvus::grpc::TableName* request, ::milvus::grpc::Status* response, ::grpc::experimental::ClientUnaryReactor* reactor) = 0; + virtual void DropTable(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::Status* response, ::grpc::experimental::ClientUnaryReactor* reactor) = 0; // * - // @brief delete table by range + // @brief This method is used to build index by table in sync mode. // - // This method is used to delete vector by range + // @param IndexParam, index paramters. // - // @return rpc status. - virtual void DeleteByRange(::grpc::ClientContext* context, const ::milvus::grpc::DeleteByRangeParam* request, ::milvus::grpc::Status* response, std::function) = 0; - virtual void DeleteByRange(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::Status* response, std::function) = 0; - virtual void DeleteByRange(::grpc::ClientContext* context, const ::milvus::grpc::DeleteByRangeParam* request, ::milvus::grpc::Status* response, ::grpc::experimental::ClientUnaryReactor* reactor) = 0; - virtual void DeleteByRange(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::Status* response, ::grpc::experimental::ClientUnaryReactor* reactor) = 0; + // @return Status + virtual void CreateIndex(::grpc::ClientContext* context, const ::milvus::grpc::IndexParam* request, ::milvus::grpc::Status* response, std::function) = 0; + virtual void CreateIndex(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::Status* response, std::function) = 0; + virtual void CreateIndex(::grpc::ClientContext* context, const ::milvus::grpc::IndexParam* request, ::milvus::grpc::Status* response, ::grpc::experimental::ClientUnaryReactor* reactor) = 0; + virtual void CreateIndex(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::Status* response, ::grpc::experimental::ClientUnaryReactor* reactor) = 0; // * - // @brief preload table + // @brief This method is used to describe index // - // This method is used to preload table + // @param TableName, target table name. // - // @return Status. - virtual void PreloadTable(::grpc::ClientContext* context, const ::milvus::grpc::TableName* request, ::milvus::grpc::Status* response, std::function) = 0; - virtual void PreloadTable(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::Status* response, std::function) = 0; - virtual void PreloadTable(::grpc::ClientContext* context, const ::milvus::grpc::TableName* request, ::milvus::grpc::Status* response, ::grpc::experimental::ClientUnaryReactor* reactor) = 0; - virtual void PreloadTable(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::Status* response, ::grpc::experimental::ClientUnaryReactor* reactor) = 0; - // * - // @brief describe index - // - // This method is used to describe index - // - // @return Status. + // @return IndexParam virtual void DescribeIndex(::grpc::ClientContext* context, const ::milvus::grpc::TableName* request, ::milvus::grpc::IndexParam* response, std::function) = 0; virtual void DescribeIndex(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::IndexParam* response, std::function) = 0; virtual void DescribeIndex(::grpc::ClientContext* context, const ::milvus::grpc::TableName* request, ::milvus::grpc::IndexParam* response, ::grpc::experimental::ClientUnaryReactor* reactor) = 0; virtual void DescribeIndex(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::IndexParam* response, ::grpc::experimental::ClientUnaryReactor* reactor) = 0; // * - // @brief drop index + // @brief This method is used to drop index // - // This method is used to drop index + // @param TableName, target table name. // - // @return Status. + // @return Status virtual void DropIndex(::grpc::ClientContext* context, const ::milvus::grpc::TableName* request, ::milvus::grpc::Status* response, std::function) = 0; virtual void DropIndex(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::Status* response, std::function) = 0; virtual void DropIndex(::grpc::ClientContext* context, const ::milvus::grpc::TableName* request, ::milvus::grpc::Status* response, ::grpc::experimental::ClientUnaryReactor* reactor) = 0; virtual void DropIndex(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::Status* response, ::grpc::experimental::ClientUnaryReactor* reactor) = 0; + // * + // @brief This method is used to create partition + // + // @param PartitionParam, partition parameters. + // + // @return Status + virtual void CreatePartition(::grpc::ClientContext* context, const ::milvus::grpc::PartitionParam* request, ::milvus::grpc::Status* response, std::function) = 0; + virtual void CreatePartition(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::Status* response, std::function) = 0; + virtual void CreatePartition(::grpc::ClientContext* context, const ::milvus::grpc::PartitionParam* request, ::milvus::grpc::Status* response, ::grpc::experimental::ClientUnaryReactor* reactor) = 0; + virtual void CreatePartition(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::Status* response, ::grpc::experimental::ClientUnaryReactor* reactor) = 0; + // * + // @brief This method is used to show partition information + // + // @param TableName, target table name. + // + // @return PartitionList + virtual void ShowPartitions(::grpc::ClientContext* context, const ::milvus::grpc::TableName* request, ::milvus::grpc::PartitionList* response, std::function) = 0; + virtual void ShowPartitions(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::PartitionList* response, std::function) = 0; + virtual void ShowPartitions(::grpc::ClientContext* context, const ::milvus::grpc::TableName* request, ::milvus::grpc::PartitionList* response, ::grpc::experimental::ClientUnaryReactor* reactor) = 0; + virtual void ShowPartitions(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::PartitionList* response, ::grpc::experimental::ClientUnaryReactor* reactor) = 0; + // * + // @brief This method is used to drop partition + // + // @param PartitionParam, target partition. + // + // @return Status + virtual void DropPartition(::grpc::ClientContext* context, const ::milvus::grpc::PartitionParam* request, ::milvus::grpc::Status* response, std::function) = 0; + virtual void DropPartition(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::Status* response, std::function) = 0; + virtual void DropPartition(::grpc::ClientContext* context, const ::milvus::grpc::PartitionParam* request, ::milvus::grpc::Status* response, ::grpc::experimental::ClientUnaryReactor* reactor) = 0; + virtual void DropPartition(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::Status* response, ::grpc::experimental::ClientUnaryReactor* reactor) = 0; + // * + // @brief This method is used to add vector array to table. + // + // @param InsertParam, insert parameters. + // + // @return VectorIds + virtual void Insert(::grpc::ClientContext* context, const ::milvus::grpc::InsertParam* request, ::milvus::grpc::VectorIds* response, std::function) = 0; + virtual void Insert(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::VectorIds* response, std::function) = 0; + virtual void Insert(::grpc::ClientContext* context, const ::milvus::grpc::InsertParam* request, ::milvus::grpc::VectorIds* response, ::grpc::experimental::ClientUnaryReactor* reactor) = 0; + virtual void Insert(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::VectorIds* response, ::grpc::experimental::ClientUnaryReactor* reactor) = 0; + // * + // @brief This method is used to query vector in table. + // + // @param SearchParam, search parameters. + // + // @return TopKQueryResult + virtual void Search(::grpc::ClientContext* context, const ::milvus::grpc::SearchParam* request, ::milvus::grpc::TopKQueryResult* response, std::function) = 0; + virtual void Search(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::TopKQueryResult* response, std::function) = 0; + virtual void Search(::grpc::ClientContext* context, const ::milvus::grpc::SearchParam* request, ::milvus::grpc::TopKQueryResult* response, ::grpc::experimental::ClientUnaryReactor* reactor) = 0; + virtual void Search(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::TopKQueryResult* response, ::grpc::experimental::ClientUnaryReactor* reactor) = 0; + // * + // @brief This method is used to query vector in specified files. + // + // @param SearchInFilesParam, search in files paremeters. + // + // @return TopKQueryResult + virtual void SearchInFiles(::grpc::ClientContext* context, const ::milvus::grpc::SearchInFilesParam* request, ::milvus::grpc::TopKQueryResult* response, std::function) = 0; + virtual void SearchInFiles(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::TopKQueryResult* response, std::function) = 0; + virtual void SearchInFiles(::grpc::ClientContext* context, const ::milvus::grpc::SearchInFilesParam* request, ::milvus::grpc::TopKQueryResult* response, ::grpc::experimental::ClientUnaryReactor* reactor) = 0; + virtual void SearchInFiles(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::TopKQueryResult* response, ::grpc::experimental::ClientUnaryReactor* reactor) = 0; + // * + // @brief This method is used to give the server status. + // + // @param Command, command string + // + // @return StringReply + virtual void Cmd(::grpc::ClientContext* context, const ::milvus::grpc::Command* request, ::milvus::grpc::StringReply* response, std::function) = 0; + virtual void Cmd(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::StringReply* response, std::function) = 0; + virtual void Cmd(::grpc::ClientContext* context, const ::milvus::grpc::Command* request, ::milvus::grpc::StringReply* response, ::grpc::experimental::ClientUnaryReactor* reactor) = 0; + virtual void Cmd(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::StringReply* response, ::grpc::experimental::ClientUnaryReactor* reactor) = 0; + // * + // @brief This method is used to delete vector by date range + // + // @param DeleteByDateParam, delete parameters. + // + // @return status + virtual void DeleteByDate(::grpc::ClientContext* context, const ::milvus::grpc::DeleteByDateParam* request, ::milvus::grpc::Status* response, std::function) = 0; + virtual void DeleteByDate(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::Status* response, std::function) = 0; + virtual void DeleteByDate(::grpc::ClientContext* context, const ::milvus::grpc::DeleteByDateParam* request, ::milvus::grpc::Status* response, ::grpc::experimental::ClientUnaryReactor* reactor) = 0; + virtual void DeleteByDate(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::Status* response, ::grpc::experimental::ClientUnaryReactor* reactor) = 0; + // * + // @brief This method is used to preload table + // + // @param TableName, target table name. + // + // @return Status + virtual void PreloadTable(::grpc::ClientContext* context, const ::milvus::grpc::TableName* request, ::milvus::grpc::Status* response, std::function) = 0; + virtual void PreloadTable(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::Status* response, std::function) = 0; + virtual void PreloadTable(::grpc::ClientContext* context, const ::milvus::grpc::TableName* request, ::milvus::grpc::Status* response, ::grpc::experimental::ClientUnaryReactor* reactor) = 0; + virtual void PreloadTable(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::Status* response, ::grpc::experimental::ClientUnaryReactor* reactor) = 0; }; virtual class experimental_async_interface* experimental_async() { return nullptr; } private: @@ -446,32 +471,38 @@ class MilvusService final { virtual ::grpc::ClientAsyncResponseReaderInterface< ::milvus::grpc::Status>* PrepareAsyncCreateTableRaw(::grpc::ClientContext* context, const ::milvus::grpc::TableSchema& request, ::grpc::CompletionQueue* cq) = 0; virtual ::grpc::ClientAsyncResponseReaderInterface< ::milvus::grpc::BoolReply>* AsyncHasTableRaw(::grpc::ClientContext* context, const ::milvus::grpc::TableName& request, ::grpc::CompletionQueue* cq) = 0; virtual ::grpc::ClientAsyncResponseReaderInterface< ::milvus::grpc::BoolReply>* PrepareAsyncHasTableRaw(::grpc::ClientContext* context, const ::milvus::grpc::TableName& request, ::grpc::CompletionQueue* cq) = 0; - virtual ::grpc::ClientAsyncResponseReaderInterface< ::milvus::grpc::Status>* AsyncDropTableRaw(::grpc::ClientContext* context, const ::milvus::grpc::TableName& request, ::grpc::CompletionQueue* cq) = 0; - virtual ::grpc::ClientAsyncResponseReaderInterface< ::milvus::grpc::Status>* PrepareAsyncDropTableRaw(::grpc::ClientContext* context, const ::milvus::grpc::TableName& request, ::grpc::CompletionQueue* cq) = 0; - virtual ::grpc::ClientAsyncResponseReaderInterface< ::milvus::grpc::Status>* AsyncCreateIndexRaw(::grpc::ClientContext* context, const ::milvus::grpc::IndexParam& request, ::grpc::CompletionQueue* cq) = 0; - virtual ::grpc::ClientAsyncResponseReaderInterface< ::milvus::grpc::Status>* PrepareAsyncCreateIndexRaw(::grpc::ClientContext* context, const ::milvus::grpc::IndexParam& request, ::grpc::CompletionQueue* cq) = 0; - virtual ::grpc::ClientAsyncResponseReaderInterface< ::milvus::grpc::VectorIds>* AsyncInsertRaw(::grpc::ClientContext* context, const ::milvus::grpc::InsertParam& request, ::grpc::CompletionQueue* cq) = 0; - virtual ::grpc::ClientAsyncResponseReaderInterface< ::milvus::grpc::VectorIds>* PrepareAsyncInsertRaw(::grpc::ClientContext* context, const ::milvus::grpc::InsertParam& request, ::grpc::CompletionQueue* cq) = 0; - virtual ::grpc::ClientAsyncResponseReaderInterface< ::milvus::grpc::TopKQueryResult>* AsyncSearchRaw(::grpc::ClientContext* context, const ::milvus::grpc::SearchParam& request, ::grpc::CompletionQueue* cq) = 0; - virtual ::grpc::ClientAsyncResponseReaderInterface< ::milvus::grpc::TopKQueryResult>* PrepareAsyncSearchRaw(::grpc::ClientContext* context, const ::milvus::grpc::SearchParam& request, ::grpc::CompletionQueue* cq) = 0; - virtual ::grpc::ClientAsyncResponseReaderInterface< ::milvus::grpc::TopKQueryResult>* AsyncSearchInFilesRaw(::grpc::ClientContext* context, const ::milvus::grpc::SearchInFilesParam& request, ::grpc::CompletionQueue* cq) = 0; - virtual ::grpc::ClientAsyncResponseReaderInterface< ::milvus::grpc::TopKQueryResult>* PrepareAsyncSearchInFilesRaw(::grpc::ClientContext* context, const ::milvus::grpc::SearchInFilesParam& request, ::grpc::CompletionQueue* cq) = 0; virtual ::grpc::ClientAsyncResponseReaderInterface< ::milvus::grpc::TableSchema>* AsyncDescribeTableRaw(::grpc::ClientContext* context, const ::milvus::grpc::TableName& request, ::grpc::CompletionQueue* cq) = 0; virtual ::grpc::ClientAsyncResponseReaderInterface< ::milvus::grpc::TableSchema>* PrepareAsyncDescribeTableRaw(::grpc::ClientContext* context, const ::milvus::grpc::TableName& request, ::grpc::CompletionQueue* cq) = 0; virtual ::grpc::ClientAsyncResponseReaderInterface< ::milvus::grpc::TableRowCount>* AsyncCountTableRaw(::grpc::ClientContext* context, const ::milvus::grpc::TableName& request, ::grpc::CompletionQueue* cq) = 0; virtual ::grpc::ClientAsyncResponseReaderInterface< ::milvus::grpc::TableRowCount>* PrepareAsyncCountTableRaw(::grpc::ClientContext* context, const ::milvus::grpc::TableName& request, ::grpc::CompletionQueue* cq) = 0; virtual ::grpc::ClientAsyncResponseReaderInterface< ::milvus::grpc::TableNameList>* AsyncShowTablesRaw(::grpc::ClientContext* context, const ::milvus::grpc::Command& request, ::grpc::CompletionQueue* cq) = 0; virtual ::grpc::ClientAsyncResponseReaderInterface< ::milvus::grpc::TableNameList>* PrepareAsyncShowTablesRaw(::grpc::ClientContext* context, const ::milvus::grpc::Command& request, ::grpc::CompletionQueue* cq) = 0; - virtual ::grpc::ClientAsyncResponseReaderInterface< ::milvus::grpc::StringReply>* AsyncCmdRaw(::grpc::ClientContext* context, const ::milvus::grpc::Command& request, ::grpc::CompletionQueue* cq) = 0; - virtual ::grpc::ClientAsyncResponseReaderInterface< ::milvus::grpc::StringReply>* PrepareAsyncCmdRaw(::grpc::ClientContext* context, const ::milvus::grpc::Command& request, ::grpc::CompletionQueue* cq) = 0; - virtual ::grpc::ClientAsyncResponseReaderInterface< ::milvus::grpc::Status>* AsyncDeleteByRangeRaw(::grpc::ClientContext* context, const ::milvus::grpc::DeleteByRangeParam& request, ::grpc::CompletionQueue* cq) = 0; - virtual ::grpc::ClientAsyncResponseReaderInterface< ::milvus::grpc::Status>* PrepareAsyncDeleteByRangeRaw(::grpc::ClientContext* context, const ::milvus::grpc::DeleteByRangeParam& request, ::grpc::CompletionQueue* cq) = 0; - virtual ::grpc::ClientAsyncResponseReaderInterface< ::milvus::grpc::Status>* AsyncPreloadTableRaw(::grpc::ClientContext* context, const ::milvus::grpc::TableName& request, ::grpc::CompletionQueue* cq) = 0; - virtual ::grpc::ClientAsyncResponseReaderInterface< ::milvus::grpc::Status>* PrepareAsyncPreloadTableRaw(::grpc::ClientContext* context, const ::milvus::grpc::TableName& request, ::grpc::CompletionQueue* cq) = 0; + virtual ::grpc::ClientAsyncResponseReaderInterface< ::milvus::grpc::Status>* AsyncDropTableRaw(::grpc::ClientContext* context, const ::milvus::grpc::TableName& request, ::grpc::CompletionQueue* cq) = 0; + virtual ::grpc::ClientAsyncResponseReaderInterface< ::milvus::grpc::Status>* PrepareAsyncDropTableRaw(::grpc::ClientContext* context, const ::milvus::grpc::TableName& request, ::grpc::CompletionQueue* cq) = 0; + virtual ::grpc::ClientAsyncResponseReaderInterface< ::milvus::grpc::Status>* AsyncCreateIndexRaw(::grpc::ClientContext* context, const ::milvus::grpc::IndexParam& request, ::grpc::CompletionQueue* cq) = 0; + virtual ::grpc::ClientAsyncResponseReaderInterface< ::milvus::grpc::Status>* PrepareAsyncCreateIndexRaw(::grpc::ClientContext* context, const ::milvus::grpc::IndexParam& request, ::grpc::CompletionQueue* cq) = 0; virtual ::grpc::ClientAsyncResponseReaderInterface< ::milvus::grpc::IndexParam>* AsyncDescribeIndexRaw(::grpc::ClientContext* context, const ::milvus::grpc::TableName& request, ::grpc::CompletionQueue* cq) = 0; virtual ::grpc::ClientAsyncResponseReaderInterface< ::milvus::grpc::IndexParam>* PrepareAsyncDescribeIndexRaw(::grpc::ClientContext* context, const ::milvus::grpc::TableName& request, ::grpc::CompletionQueue* cq) = 0; virtual ::grpc::ClientAsyncResponseReaderInterface< ::milvus::grpc::Status>* AsyncDropIndexRaw(::grpc::ClientContext* context, const ::milvus::grpc::TableName& request, ::grpc::CompletionQueue* cq) = 0; virtual ::grpc::ClientAsyncResponseReaderInterface< ::milvus::grpc::Status>* PrepareAsyncDropIndexRaw(::grpc::ClientContext* context, const ::milvus::grpc::TableName& request, ::grpc::CompletionQueue* cq) = 0; + virtual ::grpc::ClientAsyncResponseReaderInterface< ::milvus::grpc::Status>* AsyncCreatePartitionRaw(::grpc::ClientContext* context, const ::milvus::grpc::PartitionParam& request, ::grpc::CompletionQueue* cq) = 0; + virtual ::grpc::ClientAsyncResponseReaderInterface< ::milvus::grpc::Status>* PrepareAsyncCreatePartitionRaw(::grpc::ClientContext* context, const ::milvus::grpc::PartitionParam& request, ::grpc::CompletionQueue* cq) = 0; + virtual ::grpc::ClientAsyncResponseReaderInterface< ::milvus::grpc::PartitionList>* AsyncShowPartitionsRaw(::grpc::ClientContext* context, const ::milvus::grpc::TableName& request, ::grpc::CompletionQueue* cq) = 0; + virtual ::grpc::ClientAsyncResponseReaderInterface< ::milvus::grpc::PartitionList>* PrepareAsyncShowPartitionsRaw(::grpc::ClientContext* context, const ::milvus::grpc::TableName& request, ::grpc::CompletionQueue* cq) = 0; + virtual ::grpc::ClientAsyncResponseReaderInterface< ::milvus::grpc::Status>* AsyncDropPartitionRaw(::grpc::ClientContext* context, const ::milvus::grpc::PartitionParam& request, ::grpc::CompletionQueue* cq) = 0; + virtual ::grpc::ClientAsyncResponseReaderInterface< ::milvus::grpc::Status>* PrepareAsyncDropPartitionRaw(::grpc::ClientContext* context, const ::milvus::grpc::PartitionParam& request, ::grpc::CompletionQueue* cq) = 0; + virtual ::grpc::ClientAsyncResponseReaderInterface< ::milvus::grpc::VectorIds>* AsyncInsertRaw(::grpc::ClientContext* context, const ::milvus::grpc::InsertParam& request, ::grpc::CompletionQueue* cq) = 0; + virtual ::grpc::ClientAsyncResponseReaderInterface< ::milvus::grpc::VectorIds>* PrepareAsyncInsertRaw(::grpc::ClientContext* context, const ::milvus::grpc::InsertParam& request, ::grpc::CompletionQueue* cq) = 0; + virtual ::grpc::ClientAsyncResponseReaderInterface< ::milvus::grpc::TopKQueryResult>* AsyncSearchRaw(::grpc::ClientContext* context, const ::milvus::grpc::SearchParam& request, ::grpc::CompletionQueue* cq) = 0; + virtual ::grpc::ClientAsyncResponseReaderInterface< ::milvus::grpc::TopKQueryResult>* PrepareAsyncSearchRaw(::grpc::ClientContext* context, const ::milvus::grpc::SearchParam& request, ::grpc::CompletionQueue* cq) = 0; + virtual ::grpc::ClientAsyncResponseReaderInterface< ::milvus::grpc::TopKQueryResult>* AsyncSearchInFilesRaw(::grpc::ClientContext* context, const ::milvus::grpc::SearchInFilesParam& request, ::grpc::CompletionQueue* cq) = 0; + virtual ::grpc::ClientAsyncResponseReaderInterface< ::milvus::grpc::TopKQueryResult>* PrepareAsyncSearchInFilesRaw(::grpc::ClientContext* context, const ::milvus::grpc::SearchInFilesParam& request, ::grpc::CompletionQueue* cq) = 0; + virtual ::grpc::ClientAsyncResponseReaderInterface< ::milvus::grpc::StringReply>* AsyncCmdRaw(::grpc::ClientContext* context, const ::milvus::grpc::Command& request, ::grpc::CompletionQueue* cq) = 0; + virtual ::grpc::ClientAsyncResponseReaderInterface< ::milvus::grpc::StringReply>* PrepareAsyncCmdRaw(::grpc::ClientContext* context, const ::milvus::grpc::Command& request, ::grpc::CompletionQueue* cq) = 0; + virtual ::grpc::ClientAsyncResponseReaderInterface< ::milvus::grpc::Status>* AsyncDeleteByDateRaw(::grpc::ClientContext* context, const ::milvus::grpc::DeleteByDateParam& request, ::grpc::CompletionQueue* cq) = 0; + virtual ::grpc::ClientAsyncResponseReaderInterface< ::milvus::grpc::Status>* PrepareAsyncDeleteByDateRaw(::grpc::ClientContext* context, const ::milvus::grpc::DeleteByDateParam& request, ::grpc::CompletionQueue* cq) = 0; + virtual ::grpc::ClientAsyncResponseReaderInterface< ::milvus::grpc::Status>* AsyncPreloadTableRaw(::grpc::ClientContext* context, const ::milvus::grpc::TableName& request, ::grpc::CompletionQueue* cq) = 0; + virtual ::grpc::ClientAsyncResponseReaderInterface< ::milvus::grpc::Status>* PrepareAsyncPreloadTableRaw(::grpc::ClientContext* context, const ::milvus::grpc::TableName& request, ::grpc::CompletionQueue* cq) = 0; }; class Stub final : public StubInterface { public: @@ -490,41 +521,6 @@ class MilvusService final { std::unique_ptr< ::grpc::ClientAsyncResponseReader< ::milvus::grpc::BoolReply>> PrepareAsyncHasTable(::grpc::ClientContext* context, const ::milvus::grpc::TableName& request, ::grpc::CompletionQueue* cq) { return std::unique_ptr< ::grpc::ClientAsyncResponseReader< ::milvus::grpc::BoolReply>>(PrepareAsyncHasTableRaw(context, request, cq)); } - ::grpc::Status DropTable(::grpc::ClientContext* context, const ::milvus::grpc::TableName& request, ::milvus::grpc::Status* response) override; - std::unique_ptr< ::grpc::ClientAsyncResponseReader< ::milvus::grpc::Status>> AsyncDropTable(::grpc::ClientContext* context, const ::milvus::grpc::TableName& request, ::grpc::CompletionQueue* cq) { - return std::unique_ptr< ::grpc::ClientAsyncResponseReader< ::milvus::grpc::Status>>(AsyncDropTableRaw(context, request, cq)); - } - std::unique_ptr< ::grpc::ClientAsyncResponseReader< ::milvus::grpc::Status>> PrepareAsyncDropTable(::grpc::ClientContext* context, const ::milvus::grpc::TableName& request, ::grpc::CompletionQueue* cq) { - return std::unique_ptr< ::grpc::ClientAsyncResponseReader< ::milvus::grpc::Status>>(PrepareAsyncDropTableRaw(context, request, cq)); - } - ::grpc::Status CreateIndex(::grpc::ClientContext* context, const ::milvus::grpc::IndexParam& request, ::milvus::grpc::Status* response) override; - std::unique_ptr< ::grpc::ClientAsyncResponseReader< ::milvus::grpc::Status>> AsyncCreateIndex(::grpc::ClientContext* context, const ::milvus::grpc::IndexParam& request, ::grpc::CompletionQueue* cq) { - return std::unique_ptr< ::grpc::ClientAsyncResponseReader< ::milvus::grpc::Status>>(AsyncCreateIndexRaw(context, request, cq)); - } - std::unique_ptr< ::grpc::ClientAsyncResponseReader< ::milvus::grpc::Status>> PrepareAsyncCreateIndex(::grpc::ClientContext* context, const ::milvus::grpc::IndexParam& request, ::grpc::CompletionQueue* cq) { - return std::unique_ptr< ::grpc::ClientAsyncResponseReader< ::milvus::grpc::Status>>(PrepareAsyncCreateIndexRaw(context, request, cq)); - } - ::grpc::Status Insert(::grpc::ClientContext* context, const ::milvus::grpc::InsertParam& request, ::milvus::grpc::VectorIds* response) override; - std::unique_ptr< ::grpc::ClientAsyncResponseReader< ::milvus::grpc::VectorIds>> AsyncInsert(::grpc::ClientContext* context, const ::milvus::grpc::InsertParam& request, ::grpc::CompletionQueue* cq) { - return std::unique_ptr< ::grpc::ClientAsyncResponseReader< ::milvus::grpc::VectorIds>>(AsyncInsertRaw(context, request, cq)); - } - std::unique_ptr< ::grpc::ClientAsyncResponseReader< ::milvus::grpc::VectorIds>> PrepareAsyncInsert(::grpc::ClientContext* context, const ::milvus::grpc::InsertParam& request, ::grpc::CompletionQueue* cq) { - return std::unique_ptr< ::grpc::ClientAsyncResponseReader< ::milvus::grpc::VectorIds>>(PrepareAsyncInsertRaw(context, request, cq)); - } - ::grpc::Status Search(::grpc::ClientContext* context, const ::milvus::grpc::SearchParam& request, ::milvus::grpc::TopKQueryResult* response) override; - std::unique_ptr< ::grpc::ClientAsyncResponseReader< ::milvus::grpc::TopKQueryResult>> AsyncSearch(::grpc::ClientContext* context, const ::milvus::grpc::SearchParam& request, ::grpc::CompletionQueue* cq) { - return std::unique_ptr< ::grpc::ClientAsyncResponseReader< ::milvus::grpc::TopKQueryResult>>(AsyncSearchRaw(context, request, cq)); - } - std::unique_ptr< ::grpc::ClientAsyncResponseReader< ::milvus::grpc::TopKQueryResult>> PrepareAsyncSearch(::grpc::ClientContext* context, const ::milvus::grpc::SearchParam& request, ::grpc::CompletionQueue* cq) { - return std::unique_ptr< ::grpc::ClientAsyncResponseReader< ::milvus::grpc::TopKQueryResult>>(PrepareAsyncSearchRaw(context, request, cq)); - } - ::grpc::Status SearchInFiles(::grpc::ClientContext* context, const ::milvus::grpc::SearchInFilesParam& request, ::milvus::grpc::TopKQueryResult* response) override; - std::unique_ptr< ::grpc::ClientAsyncResponseReader< ::milvus::grpc::TopKQueryResult>> AsyncSearchInFiles(::grpc::ClientContext* context, const ::milvus::grpc::SearchInFilesParam& request, ::grpc::CompletionQueue* cq) { - return std::unique_ptr< ::grpc::ClientAsyncResponseReader< ::milvus::grpc::TopKQueryResult>>(AsyncSearchInFilesRaw(context, request, cq)); - } - std::unique_ptr< ::grpc::ClientAsyncResponseReader< ::milvus::grpc::TopKQueryResult>> PrepareAsyncSearchInFiles(::grpc::ClientContext* context, const ::milvus::grpc::SearchInFilesParam& request, ::grpc::CompletionQueue* cq) { - return std::unique_ptr< ::grpc::ClientAsyncResponseReader< ::milvus::grpc::TopKQueryResult>>(PrepareAsyncSearchInFilesRaw(context, request, cq)); - } ::grpc::Status DescribeTable(::grpc::ClientContext* context, const ::milvus::grpc::TableName& request, ::milvus::grpc::TableSchema* response) override; std::unique_ptr< ::grpc::ClientAsyncResponseReader< ::milvus::grpc::TableSchema>> AsyncDescribeTable(::grpc::ClientContext* context, const ::milvus::grpc::TableName& request, ::grpc::CompletionQueue* cq) { return std::unique_ptr< ::grpc::ClientAsyncResponseReader< ::milvus::grpc::TableSchema>>(AsyncDescribeTableRaw(context, request, cq)); @@ -546,26 +542,19 @@ class MilvusService final { std::unique_ptr< ::grpc::ClientAsyncResponseReader< ::milvus::grpc::TableNameList>> PrepareAsyncShowTables(::grpc::ClientContext* context, const ::milvus::grpc::Command& request, ::grpc::CompletionQueue* cq) { return std::unique_ptr< ::grpc::ClientAsyncResponseReader< ::milvus::grpc::TableNameList>>(PrepareAsyncShowTablesRaw(context, request, cq)); } - ::grpc::Status Cmd(::grpc::ClientContext* context, const ::milvus::grpc::Command& request, ::milvus::grpc::StringReply* response) override; - std::unique_ptr< ::grpc::ClientAsyncResponseReader< ::milvus::grpc::StringReply>> AsyncCmd(::grpc::ClientContext* context, const ::milvus::grpc::Command& request, ::grpc::CompletionQueue* cq) { - return std::unique_ptr< ::grpc::ClientAsyncResponseReader< ::milvus::grpc::StringReply>>(AsyncCmdRaw(context, request, cq)); + ::grpc::Status DropTable(::grpc::ClientContext* context, const ::milvus::grpc::TableName& request, ::milvus::grpc::Status* response) override; + std::unique_ptr< ::grpc::ClientAsyncResponseReader< ::milvus::grpc::Status>> AsyncDropTable(::grpc::ClientContext* context, const ::milvus::grpc::TableName& request, ::grpc::CompletionQueue* cq) { + return std::unique_ptr< ::grpc::ClientAsyncResponseReader< ::milvus::grpc::Status>>(AsyncDropTableRaw(context, request, cq)); } - std::unique_ptr< ::grpc::ClientAsyncResponseReader< ::milvus::grpc::StringReply>> PrepareAsyncCmd(::grpc::ClientContext* context, const ::milvus::grpc::Command& request, ::grpc::CompletionQueue* cq) { - return std::unique_ptr< ::grpc::ClientAsyncResponseReader< ::milvus::grpc::StringReply>>(PrepareAsyncCmdRaw(context, request, cq)); + std::unique_ptr< ::grpc::ClientAsyncResponseReader< ::milvus::grpc::Status>> PrepareAsyncDropTable(::grpc::ClientContext* context, const ::milvus::grpc::TableName& request, ::grpc::CompletionQueue* cq) { + return std::unique_ptr< ::grpc::ClientAsyncResponseReader< ::milvus::grpc::Status>>(PrepareAsyncDropTableRaw(context, request, cq)); } - ::grpc::Status DeleteByRange(::grpc::ClientContext* context, const ::milvus::grpc::DeleteByRangeParam& request, ::milvus::grpc::Status* response) override; - std::unique_ptr< ::grpc::ClientAsyncResponseReader< ::milvus::grpc::Status>> AsyncDeleteByRange(::grpc::ClientContext* context, const ::milvus::grpc::DeleteByRangeParam& request, ::grpc::CompletionQueue* cq) { - return std::unique_ptr< ::grpc::ClientAsyncResponseReader< ::milvus::grpc::Status>>(AsyncDeleteByRangeRaw(context, request, cq)); + ::grpc::Status CreateIndex(::grpc::ClientContext* context, const ::milvus::grpc::IndexParam& request, ::milvus::grpc::Status* response) override; + std::unique_ptr< ::grpc::ClientAsyncResponseReader< ::milvus::grpc::Status>> AsyncCreateIndex(::grpc::ClientContext* context, const ::milvus::grpc::IndexParam& request, ::grpc::CompletionQueue* cq) { + return std::unique_ptr< ::grpc::ClientAsyncResponseReader< ::milvus::grpc::Status>>(AsyncCreateIndexRaw(context, request, cq)); } - std::unique_ptr< ::grpc::ClientAsyncResponseReader< ::milvus::grpc::Status>> PrepareAsyncDeleteByRange(::grpc::ClientContext* context, const ::milvus::grpc::DeleteByRangeParam& request, ::grpc::CompletionQueue* cq) { - return std::unique_ptr< ::grpc::ClientAsyncResponseReader< ::milvus::grpc::Status>>(PrepareAsyncDeleteByRangeRaw(context, request, cq)); - } - ::grpc::Status PreloadTable(::grpc::ClientContext* context, const ::milvus::grpc::TableName& request, ::milvus::grpc::Status* response) override; - std::unique_ptr< ::grpc::ClientAsyncResponseReader< ::milvus::grpc::Status>> AsyncPreloadTable(::grpc::ClientContext* context, const ::milvus::grpc::TableName& request, ::grpc::CompletionQueue* cq) { - return std::unique_ptr< ::grpc::ClientAsyncResponseReader< ::milvus::grpc::Status>>(AsyncPreloadTableRaw(context, request, cq)); - } - std::unique_ptr< ::grpc::ClientAsyncResponseReader< ::milvus::grpc::Status>> PrepareAsyncPreloadTable(::grpc::ClientContext* context, const ::milvus::grpc::TableName& request, ::grpc::CompletionQueue* cq) { - return std::unique_ptr< ::grpc::ClientAsyncResponseReader< ::milvus::grpc::Status>>(PrepareAsyncPreloadTableRaw(context, request, cq)); + std::unique_ptr< ::grpc::ClientAsyncResponseReader< ::milvus::grpc::Status>> PrepareAsyncCreateIndex(::grpc::ClientContext* context, const ::milvus::grpc::IndexParam& request, ::grpc::CompletionQueue* cq) { + return std::unique_ptr< ::grpc::ClientAsyncResponseReader< ::milvus::grpc::Status>>(PrepareAsyncCreateIndexRaw(context, request, cq)); } ::grpc::Status DescribeIndex(::grpc::ClientContext* context, const ::milvus::grpc::TableName& request, ::milvus::grpc::IndexParam* response) override; std::unique_ptr< ::grpc::ClientAsyncResponseReader< ::milvus::grpc::IndexParam>> AsyncDescribeIndex(::grpc::ClientContext* context, const ::milvus::grpc::TableName& request, ::grpc::CompletionQueue* cq) { @@ -581,6 +570,69 @@ class MilvusService final { std::unique_ptr< ::grpc::ClientAsyncResponseReader< ::milvus::grpc::Status>> PrepareAsyncDropIndex(::grpc::ClientContext* context, const ::milvus::grpc::TableName& request, ::grpc::CompletionQueue* cq) { return std::unique_ptr< ::grpc::ClientAsyncResponseReader< ::milvus::grpc::Status>>(PrepareAsyncDropIndexRaw(context, request, cq)); } + ::grpc::Status CreatePartition(::grpc::ClientContext* context, const ::milvus::grpc::PartitionParam& request, ::milvus::grpc::Status* response) override; + std::unique_ptr< ::grpc::ClientAsyncResponseReader< ::milvus::grpc::Status>> AsyncCreatePartition(::grpc::ClientContext* context, const ::milvus::grpc::PartitionParam& request, ::grpc::CompletionQueue* cq) { + return std::unique_ptr< ::grpc::ClientAsyncResponseReader< ::milvus::grpc::Status>>(AsyncCreatePartitionRaw(context, request, cq)); + } + std::unique_ptr< ::grpc::ClientAsyncResponseReader< ::milvus::grpc::Status>> PrepareAsyncCreatePartition(::grpc::ClientContext* context, const ::milvus::grpc::PartitionParam& request, ::grpc::CompletionQueue* cq) { + return std::unique_ptr< ::grpc::ClientAsyncResponseReader< ::milvus::grpc::Status>>(PrepareAsyncCreatePartitionRaw(context, request, cq)); + } + ::grpc::Status ShowPartitions(::grpc::ClientContext* context, const ::milvus::grpc::TableName& request, ::milvus::grpc::PartitionList* response) override; + std::unique_ptr< ::grpc::ClientAsyncResponseReader< ::milvus::grpc::PartitionList>> AsyncShowPartitions(::grpc::ClientContext* context, const ::milvus::grpc::TableName& request, ::grpc::CompletionQueue* cq) { + return std::unique_ptr< ::grpc::ClientAsyncResponseReader< ::milvus::grpc::PartitionList>>(AsyncShowPartitionsRaw(context, request, cq)); + } + std::unique_ptr< ::grpc::ClientAsyncResponseReader< ::milvus::grpc::PartitionList>> PrepareAsyncShowPartitions(::grpc::ClientContext* context, const ::milvus::grpc::TableName& request, ::grpc::CompletionQueue* cq) { + return std::unique_ptr< ::grpc::ClientAsyncResponseReader< ::milvus::grpc::PartitionList>>(PrepareAsyncShowPartitionsRaw(context, request, cq)); + } + ::grpc::Status DropPartition(::grpc::ClientContext* context, const ::milvus::grpc::PartitionParam& request, ::milvus::grpc::Status* response) override; + std::unique_ptr< ::grpc::ClientAsyncResponseReader< ::milvus::grpc::Status>> AsyncDropPartition(::grpc::ClientContext* context, const ::milvus::grpc::PartitionParam& request, ::grpc::CompletionQueue* cq) { + return std::unique_ptr< ::grpc::ClientAsyncResponseReader< ::milvus::grpc::Status>>(AsyncDropPartitionRaw(context, request, cq)); + } + std::unique_ptr< ::grpc::ClientAsyncResponseReader< ::milvus::grpc::Status>> PrepareAsyncDropPartition(::grpc::ClientContext* context, const ::milvus::grpc::PartitionParam& request, ::grpc::CompletionQueue* cq) { + return std::unique_ptr< ::grpc::ClientAsyncResponseReader< ::milvus::grpc::Status>>(PrepareAsyncDropPartitionRaw(context, request, cq)); + } + ::grpc::Status Insert(::grpc::ClientContext* context, const ::milvus::grpc::InsertParam& request, ::milvus::grpc::VectorIds* response) override; + std::unique_ptr< ::grpc::ClientAsyncResponseReader< ::milvus::grpc::VectorIds>> AsyncInsert(::grpc::ClientContext* context, const ::milvus::grpc::InsertParam& request, ::grpc::CompletionQueue* cq) { + return std::unique_ptr< ::grpc::ClientAsyncResponseReader< ::milvus::grpc::VectorIds>>(AsyncInsertRaw(context, request, cq)); + } + std::unique_ptr< ::grpc::ClientAsyncResponseReader< ::milvus::grpc::VectorIds>> PrepareAsyncInsert(::grpc::ClientContext* context, const ::milvus::grpc::InsertParam& request, ::grpc::CompletionQueue* cq) { + return std::unique_ptr< ::grpc::ClientAsyncResponseReader< ::milvus::grpc::VectorIds>>(PrepareAsyncInsertRaw(context, request, cq)); + } + ::grpc::Status Search(::grpc::ClientContext* context, const ::milvus::grpc::SearchParam& request, ::milvus::grpc::TopKQueryResult* response) override; + std::unique_ptr< ::grpc::ClientAsyncResponseReader< ::milvus::grpc::TopKQueryResult>> AsyncSearch(::grpc::ClientContext* context, const ::milvus::grpc::SearchParam& request, ::grpc::CompletionQueue* cq) { + return std::unique_ptr< ::grpc::ClientAsyncResponseReader< ::milvus::grpc::TopKQueryResult>>(AsyncSearchRaw(context, request, cq)); + } + std::unique_ptr< ::grpc::ClientAsyncResponseReader< ::milvus::grpc::TopKQueryResult>> PrepareAsyncSearch(::grpc::ClientContext* context, const ::milvus::grpc::SearchParam& request, ::grpc::CompletionQueue* cq) { + return std::unique_ptr< ::grpc::ClientAsyncResponseReader< ::milvus::grpc::TopKQueryResult>>(PrepareAsyncSearchRaw(context, request, cq)); + } + ::grpc::Status SearchInFiles(::grpc::ClientContext* context, const ::milvus::grpc::SearchInFilesParam& request, ::milvus::grpc::TopKQueryResult* response) override; + std::unique_ptr< ::grpc::ClientAsyncResponseReader< ::milvus::grpc::TopKQueryResult>> AsyncSearchInFiles(::grpc::ClientContext* context, const ::milvus::grpc::SearchInFilesParam& request, ::grpc::CompletionQueue* cq) { + return std::unique_ptr< ::grpc::ClientAsyncResponseReader< ::milvus::grpc::TopKQueryResult>>(AsyncSearchInFilesRaw(context, request, cq)); + } + std::unique_ptr< ::grpc::ClientAsyncResponseReader< ::milvus::grpc::TopKQueryResult>> PrepareAsyncSearchInFiles(::grpc::ClientContext* context, const ::milvus::grpc::SearchInFilesParam& request, ::grpc::CompletionQueue* cq) { + return std::unique_ptr< ::grpc::ClientAsyncResponseReader< ::milvus::grpc::TopKQueryResult>>(PrepareAsyncSearchInFilesRaw(context, request, cq)); + } + ::grpc::Status Cmd(::grpc::ClientContext* context, const ::milvus::grpc::Command& request, ::milvus::grpc::StringReply* response) override; + std::unique_ptr< ::grpc::ClientAsyncResponseReader< ::milvus::grpc::StringReply>> AsyncCmd(::grpc::ClientContext* context, const ::milvus::grpc::Command& request, ::grpc::CompletionQueue* cq) { + return std::unique_ptr< ::grpc::ClientAsyncResponseReader< ::milvus::grpc::StringReply>>(AsyncCmdRaw(context, request, cq)); + } + std::unique_ptr< ::grpc::ClientAsyncResponseReader< ::milvus::grpc::StringReply>> PrepareAsyncCmd(::grpc::ClientContext* context, const ::milvus::grpc::Command& request, ::grpc::CompletionQueue* cq) { + return std::unique_ptr< ::grpc::ClientAsyncResponseReader< ::milvus::grpc::StringReply>>(PrepareAsyncCmdRaw(context, request, cq)); + } + ::grpc::Status DeleteByDate(::grpc::ClientContext* context, const ::milvus::grpc::DeleteByDateParam& request, ::milvus::grpc::Status* response) override; + std::unique_ptr< ::grpc::ClientAsyncResponseReader< ::milvus::grpc::Status>> AsyncDeleteByDate(::grpc::ClientContext* context, const ::milvus::grpc::DeleteByDateParam& request, ::grpc::CompletionQueue* cq) { + return std::unique_ptr< ::grpc::ClientAsyncResponseReader< ::milvus::grpc::Status>>(AsyncDeleteByDateRaw(context, request, cq)); + } + std::unique_ptr< ::grpc::ClientAsyncResponseReader< ::milvus::grpc::Status>> PrepareAsyncDeleteByDate(::grpc::ClientContext* context, const ::milvus::grpc::DeleteByDateParam& request, ::grpc::CompletionQueue* cq) { + return std::unique_ptr< ::grpc::ClientAsyncResponseReader< ::milvus::grpc::Status>>(PrepareAsyncDeleteByDateRaw(context, request, cq)); + } + ::grpc::Status PreloadTable(::grpc::ClientContext* context, const ::milvus::grpc::TableName& request, ::milvus::grpc::Status* response) override; + std::unique_ptr< ::grpc::ClientAsyncResponseReader< ::milvus::grpc::Status>> AsyncPreloadTable(::grpc::ClientContext* context, const ::milvus::grpc::TableName& request, ::grpc::CompletionQueue* cq) { + return std::unique_ptr< ::grpc::ClientAsyncResponseReader< ::milvus::grpc::Status>>(AsyncPreloadTableRaw(context, request, cq)); + } + std::unique_ptr< ::grpc::ClientAsyncResponseReader< ::milvus::grpc::Status>> PrepareAsyncPreloadTable(::grpc::ClientContext* context, const ::milvus::grpc::TableName& request, ::grpc::CompletionQueue* cq) { + return std::unique_ptr< ::grpc::ClientAsyncResponseReader< ::milvus::grpc::Status>>(PrepareAsyncPreloadTableRaw(context, request, cq)); + } class experimental_async final : public StubInterface::experimental_async_interface { public: @@ -592,26 +644,6 @@ class MilvusService final { void HasTable(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::BoolReply* response, std::function) override; void HasTable(::grpc::ClientContext* context, const ::milvus::grpc::TableName* request, ::milvus::grpc::BoolReply* response, ::grpc::experimental::ClientUnaryReactor* reactor) override; void HasTable(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::BoolReply* response, ::grpc::experimental::ClientUnaryReactor* reactor) override; - void DropTable(::grpc::ClientContext* context, const ::milvus::grpc::TableName* request, ::milvus::grpc::Status* response, std::function) override; - void DropTable(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::Status* response, std::function) override; - void DropTable(::grpc::ClientContext* context, const ::milvus::grpc::TableName* request, ::milvus::grpc::Status* response, ::grpc::experimental::ClientUnaryReactor* reactor) override; - void DropTable(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::Status* response, ::grpc::experimental::ClientUnaryReactor* reactor) override; - void CreateIndex(::grpc::ClientContext* context, const ::milvus::grpc::IndexParam* request, ::milvus::grpc::Status* response, std::function) override; - void CreateIndex(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::Status* response, std::function) override; - void CreateIndex(::grpc::ClientContext* context, const ::milvus::grpc::IndexParam* request, ::milvus::grpc::Status* response, ::grpc::experimental::ClientUnaryReactor* reactor) override; - void CreateIndex(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::Status* response, ::grpc::experimental::ClientUnaryReactor* reactor) override; - void Insert(::grpc::ClientContext* context, const ::milvus::grpc::InsertParam* request, ::milvus::grpc::VectorIds* response, std::function) override; - void Insert(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::VectorIds* response, std::function) override; - void Insert(::grpc::ClientContext* context, const ::milvus::grpc::InsertParam* request, ::milvus::grpc::VectorIds* response, ::grpc::experimental::ClientUnaryReactor* reactor) override; - void Insert(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::VectorIds* response, ::grpc::experimental::ClientUnaryReactor* reactor) override; - void Search(::grpc::ClientContext* context, const ::milvus::grpc::SearchParam* request, ::milvus::grpc::TopKQueryResult* response, std::function) override; - void Search(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::TopKQueryResult* response, std::function) override; - void Search(::grpc::ClientContext* context, const ::milvus::grpc::SearchParam* request, ::milvus::grpc::TopKQueryResult* response, ::grpc::experimental::ClientUnaryReactor* reactor) override; - void Search(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::TopKQueryResult* response, ::grpc::experimental::ClientUnaryReactor* reactor) override; - void SearchInFiles(::grpc::ClientContext* context, const ::milvus::grpc::SearchInFilesParam* request, ::milvus::grpc::TopKQueryResult* response, std::function) override; - void SearchInFiles(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::TopKQueryResult* response, std::function) override; - void SearchInFiles(::grpc::ClientContext* context, const ::milvus::grpc::SearchInFilesParam* request, ::milvus::grpc::TopKQueryResult* response, ::grpc::experimental::ClientUnaryReactor* reactor) override; - void SearchInFiles(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::TopKQueryResult* response, ::grpc::experimental::ClientUnaryReactor* reactor) override; void DescribeTable(::grpc::ClientContext* context, const ::milvus::grpc::TableName* request, ::milvus::grpc::TableSchema* response, std::function) override; void DescribeTable(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::TableSchema* response, std::function) override; void DescribeTable(::grpc::ClientContext* context, const ::milvus::grpc::TableName* request, ::milvus::grpc::TableSchema* response, ::grpc::experimental::ClientUnaryReactor* reactor) override; @@ -624,18 +656,14 @@ class MilvusService final { void ShowTables(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::TableNameList* response, std::function) override; void ShowTables(::grpc::ClientContext* context, const ::milvus::grpc::Command* request, ::milvus::grpc::TableNameList* response, ::grpc::experimental::ClientUnaryReactor* reactor) override; void ShowTables(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::TableNameList* response, ::grpc::experimental::ClientUnaryReactor* reactor) override; - void Cmd(::grpc::ClientContext* context, const ::milvus::grpc::Command* request, ::milvus::grpc::StringReply* response, std::function) override; - void Cmd(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::StringReply* response, std::function) override; - void Cmd(::grpc::ClientContext* context, const ::milvus::grpc::Command* request, ::milvus::grpc::StringReply* response, ::grpc::experimental::ClientUnaryReactor* reactor) override; - void Cmd(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::StringReply* response, ::grpc::experimental::ClientUnaryReactor* reactor) override; - void DeleteByRange(::grpc::ClientContext* context, const ::milvus::grpc::DeleteByRangeParam* request, ::milvus::grpc::Status* response, std::function) override; - void DeleteByRange(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::Status* response, std::function) override; - void DeleteByRange(::grpc::ClientContext* context, const ::milvus::grpc::DeleteByRangeParam* request, ::milvus::grpc::Status* response, ::grpc::experimental::ClientUnaryReactor* reactor) override; - void DeleteByRange(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::Status* response, ::grpc::experimental::ClientUnaryReactor* reactor) override; - void PreloadTable(::grpc::ClientContext* context, const ::milvus::grpc::TableName* request, ::milvus::grpc::Status* response, std::function) override; - void PreloadTable(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::Status* response, std::function) override; - void PreloadTable(::grpc::ClientContext* context, const ::milvus::grpc::TableName* request, ::milvus::grpc::Status* response, ::grpc::experimental::ClientUnaryReactor* reactor) override; - void PreloadTable(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::Status* response, ::grpc::experimental::ClientUnaryReactor* reactor) override; + void DropTable(::grpc::ClientContext* context, const ::milvus::grpc::TableName* request, ::milvus::grpc::Status* response, std::function) override; + void DropTable(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::Status* response, std::function) override; + void DropTable(::grpc::ClientContext* context, const ::milvus::grpc::TableName* request, ::milvus::grpc::Status* response, ::grpc::experimental::ClientUnaryReactor* reactor) override; + void DropTable(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::Status* response, ::grpc::experimental::ClientUnaryReactor* reactor) override; + void CreateIndex(::grpc::ClientContext* context, const ::milvus::grpc::IndexParam* request, ::milvus::grpc::Status* response, std::function) override; + void CreateIndex(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::Status* response, std::function) override; + void CreateIndex(::grpc::ClientContext* context, const ::milvus::grpc::IndexParam* request, ::milvus::grpc::Status* response, ::grpc::experimental::ClientUnaryReactor* reactor) override; + void CreateIndex(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::Status* response, ::grpc::experimental::ClientUnaryReactor* reactor) override; void DescribeIndex(::grpc::ClientContext* context, const ::milvus::grpc::TableName* request, ::milvus::grpc::IndexParam* response, std::function) override; void DescribeIndex(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::IndexParam* response, std::function) override; void DescribeIndex(::grpc::ClientContext* context, const ::milvus::grpc::TableName* request, ::milvus::grpc::IndexParam* response, ::grpc::experimental::ClientUnaryReactor* reactor) override; @@ -644,6 +672,42 @@ class MilvusService final { void DropIndex(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::Status* response, std::function) override; void DropIndex(::grpc::ClientContext* context, const ::milvus::grpc::TableName* request, ::milvus::grpc::Status* response, ::grpc::experimental::ClientUnaryReactor* reactor) override; void DropIndex(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::Status* response, ::grpc::experimental::ClientUnaryReactor* reactor) override; + void CreatePartition(::grpc::ClientContext* context, const ::milvus::grpc::PartitionParam* request, ::milvus::grpc::Status* response, std::function) override; + void CreatePartition(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::Status* response, std::function) override; + void CreatePartition(::grpc::ClientContext* context, const ::milvus::grpc::PartitionParam* request, ::milvus::grpc::Status* response, ::grpc::experimental::ClientUnaryReactor* reactor) override; + void CreatePartition(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::Status* response, ::grpc::experimental::ClientUnaryReactor* reactor) override; + void ShowPartitions(::grpc::ClientContext* context, const ::milvus::grpc::TableName* request, ::milvus::grpc::PartitionList* response, std::function) override; + void ShowPartitions(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::PartitionList* response, std::function) override; + void ShowPartitions(::grpc::ClientContext* context, const ::milvus::grpc::TableName* request, ::milvus::grpc::PartitionList* response, ::grpc::experimental::ClientUnaryReactor* reactor) override; + void ShowPartitions(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::PartitionList* response, ::grpc::experimental::ClientUnaryReactor* reactor) override; + void DropPartition(::grpc::ClientContext* context, const ::milvus::grpc::PartitionParam* request, ::milvus::grpc::Status* response, std::function) override; + void DropPartition(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::Status* response, std::function) override; + void DropPartition(::grpc::ClientContext* context, const ::milvus::grpc::PartitionParam* request, ::milvus::grpc::Status* response, ::grpc::experimental::ClientUnaryReactor* reactor) override; + void DropPartition(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::Status* response, ::grpc::experimental::ClientUnaryReactor* reactor) override; + void Insert(::grpc::ClientContext* context, const ::milvus::grpc::InsertParam* request, ::milvus::grpc::VectorIds* response, std::function) override; + void Insert(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::VectorIds* response, std::function) override; + void Insert(::grpc::ClientContext* context, const ::milvus::grpc::InsertParam* request, ::milvus::grpc::VectorIds* response, ::grpc::experimental::ClientUnaryReactor* reactor) override; + void Insert(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::VectorIds* response, ::grpc::experimental::ClientUnaryReactor* reactor) override; + void Search(::grpc::ClientContext* context, const ::milvus::grpc::SearchParam* request, ::milvus::grpc::TopKQueryResult* response, std::function) override; + void Search(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::TopKQueryResult* response, std::function) override; + void Search(::grpc::ClientContext* context, const ::milvus::grpc::SearchParam* request, ::milvus::grpc::TopKQueryResult* response, ::grpc::experimental::ClientUnaryReactor* reactor) override; + void Search(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::TopKQueryResult* response, ::grpc::experimental::ClientUnaryReactor* reactor) override; + void SearchInFiles(::grpc::ClientContext* context, const ::milvus::grpc::SearchInFilesParam* request, ::milvus::grpc::TopKQueryResult* response, std::function) override; + void SearchInFiles(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::TopKQueryResult* response, std::function) override; + void SearchInFiles(::grpc::ClientContext* context, const ::milvus::grpc::SearchInFilesParam* request, ::milvus::grpc::TopKQueryResult* response, ::grpc::experimental::ClientUnaryReactor* reactor) override; + void SearchInFiles(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::TopKQueryResult* response, ::grpc::experimental::ClientUnaryReactor* reactor) override; + void Cmd(::grpc::ClientContext* context, const ::milvus::grpc::Command* request, ::milvus::grpc::StringReply* response, std::function) override; + void Cmd(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::StringReply* response, std::function) override; + void Cmd(::grpc::ClientContext* context, const ::milvus::grpc::Command* request, ::milvus::grpc::StringReply* response, ::grpc::experimental::ClientUnaryReactor* reactor) override; + void Cmd(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::StringReply* response, ::grpc::experimental::ClientUnaryReactor* reactor) override; + void DeleteByDate(::grpc::ClientContext* context, const ::milvus::grpc::DeleteByDateParam* request, ::milvus::grpc::Status* response, std::function) override; + void DeleteByDate(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::Status* response, std::function) override; + void DeleteByDate(::grpc::ClientContext* context, const ::milvus::grpc::DeleteByDateParam* request, ::milvus::grpc::Status* response, ::grpc::experimental::ClientUnaryReactor* reactor) override; + void DeleteByDate(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::Status* response, ::grpc::experimental::ClientUnaryReactor* reactor) override; + void PreloadTable(::grpc::ClientContext* context, const ::milvus::grpc::TableName* request, ::milvus::grpc::Status* response, std::function) override; + void PreloadTable(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::Status* response, std::function) override; + void PreloadTable(::grpc::ClientContext* context, const ::milvus::grpc::TableName* request, ::milvus::grpc::Status* response, ::grpc::experimental::ClientUnaryReactor* reactor) override; + void PreloadTable(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::Status* response, ::grpc::experimental::ClientUnaryReactor* reactor) override; private: friend class Stub; explicit experimental_async(Stub* stub): stub_(stub) { } @@ -659,47 +723,56 @@ class MilvusService final { ::grpc::ClientAsyncResponseReader< ::milvus::grpc::Status>* PrepareAsyncCreateTableRaw(::grpc::ClientContext* context, const ::milvus::grpc::TableSchema& request, ::grpc::CompletionQueue* cq) override; ::grpc::ClientAsyncResponseReader< ::milvus::grpc::BoolReply>* AsyncHasTableRaw(::grpc::ClientContext* context, const ::milvus::grpc::TableName& request, ::grpc::CompletionQueue* cq) override; ::grpc::ClientAsyncResponseReader< ::milvus::grpc::BoolReply>* PrepareAsyncHasTableRaw(::grpc::ClientContext* context, const ::milvus::grpc::TableName& request, ::grpc::CompletionQueue* cq) override; - ::grpc::ClientAsyncResponseReader< ::milvus::grpc::Status>* AsyncDropTableRaw(::grpc::ClientContext* context, const ::milvus::grpc::TableName& request, ::grpc::CompletionQueue* cq) override; - ::grpc::ClientAsyncResponseReader< ::milvus::grpc::Status>* PrepareAsyncDropTableRaw(::grpc::ClientContext* context, const ::milvus::grpc::TableName& request, ::grpc::CompletionQueue* cq) override; - ::grpc::ClientAsyncResponseReader< ::milvus::grpc::Status>* AsyncCreateIndexRaw(::grpc::ClientContext* context, const ::milvus::grpc::IndexParam& request, ::grpc::CompletionQueue* cq) override; - ::grpc::ClientAsyncResponseReader< ::milvus::grpc::Status>* PrepareAsyncCreateIndexRaw(::grpc::ClientContext* context, const ::milvus::grpc::IndexParam& request, ::grpc::CompletionQueue* cq) override; - ::grpc::ClientAsyncResponseReader< ::milvus::grpc::VectorIds>* AsyncInsertRaw(::grpc::ClientContext* context, const ::milvus::grpc::InsertParam& request, ::grpc::CompletionQueue* cq) override; - ::grpc::ClientAsyncResponseReader< ::milvus::grpc::VectorIds>* PrepareAsyncInsertRaw(::grpc::ClientContext* context, const ::milvus::grpc::InsertParam& request, ::grpc::CompletionQueue* cq) override; - ::grpc::ClientAsyncResponseReader< ::milvus::grpc::TopKQueryResult>* AsyncSearchRaw(::grpc::ClientContext* context, const ::milvus::grpc::SearchParam& request, ::grpc::CompletionQueue* cq) override; - ::grpc::ClientAsyncResponseReader< ::milvus::grpc::TopKQueryResult>* PrepareAsyncSearchRaw(::grpc::ClientContext* context, const ::milvus::grpc::SearchParam& request, ::grpc::CompletionQueue* cq) override; - ::grpc::ClientAsyncResponseReader< ::milvus::grpc::TopKQueryResult>* AsyncSearchInFilesRaw(::grpc::ClientContext* context, const ::milvus::grpc::SearchInFilesParam& request, ::grpc::CompletionQueue* cq) override; - ::grpc::ClientAsyncResponseReader< ::milvus::grpc::TopKQueryResult>* PrepareAsyncSearchInFilesRaw(::grpc::ClientContext* context, const ::milvus::grpc::SearchInFilesParam& request, ::grpc::CompletionQueue* cq) override; ::grpc::ClientAsyncResponseReader< ::milvus::grpc::TableSchema>* AsyncDescribeTableRaw(::grpc::ClientContext* context, const ::milvus::grpc::TableName& request, ::grpc::CompletionQueue* cq) override; ::grpc::ClientAsyncResponseReader< ::milvus::grpc::TableSchema>* PrepareAsyncDescribeTableRaw(::grpc::ClientContext* context, const ::milvus::grpc::TableName& request, ::grpc::CompletionQueue* cq) override; ::grpc::ClientAsyncResponseReader< ::milvus::grpc::TableRowCount>* AsyncCountTableRaw(::grpc::ClientContext* context, const ::milvus::grpc::TableName& request, ::grpc::CompletionQueue* cq) override; ::grpc::ClientAsyncResponseReader< ::milvus::grpc::TableRowCount>* PrepareAsyncCountTableRaw(::grpc::ClientContext* context, const ::milvus::grpc::TableName& request, ::grpc::CompletionQueue* cq) override; ::grpc::ClientAsyncResponseReader< ::milvus::grpc::TableNameList>* AsyncShowTablesRaw(::grpc::ClientContext* context, const ::milvus::grpc::Command& request, ::grpc::CompletionQueue* cq) override; ::grpc::ClientAsyncResponseReader< ::milvus::grpc::TableNameList>* PrepareAsyncShowTablesRaw(::grpc::ClientContext* context, const ::milvus::grpc::Command& request, ::grpc::CompletionQueue* cq) override; - ::grpc::ClientAsyncResponseReader< ::milvus::grpc::StringReply>* AsyncCmdRaw(::grpc::ClientContext* context, const ::milvus::grpc::Command& request, ::grpc::CompletionQueue* cq) override; - ::grpc::ClientAsyncResponseReader< ::milvus::grpc::StringReply>* PrepareAsyncCmdRaw(::grpc::ClientContext* context, const ::milvus::grpc::Command& request, ::grpc::CompletionQueue* cq) override; - ::grpc::ClientAsyncResponseReader< ::milvus::grpc::Status>* AsyncDeleteByRangeRaw(::grpc::ClientContext* context, const ::milvus::grpc::DeleteByRangeParam& request, ::grpc::CompletionQueue* cq) override; - ::grpc::ClientAsyncResponseReader< ::milvus::grpc::Status>* PrepareAsyncDeleteByRangeRaw(::grpc::ClientContext* context, const ::milvus::grpc::DeleteByRangeParam& request, ::grpc::CompletionQueue* cq) override; - ::grpc::ClientAsyncResponseReader< ::milvus::grpc::Status>* AsyncPreloadTableRaw(::grpc::ClientContext* context, const ::milvus::grpc::TableName& request, ::grpc::CompletionQueue* cq) override; - ::grpc::ClientAsyncResponseReader< ::milvus::grpc::Status>* PrepareAsyncPreloadTableRaw(::grpc::ClientContext* context, const ::milvus::grpc::TableName& request, ::grpc::CompletionQueue* cq) override; + ::grpc::ClientAsyncResponseReader< ::milvus::grpc::Status>* AsyncDropTableRaw(::grpc::ClientContext* context, const ::milvus::grpc::TableName& request, ::grpc::CompletionQueue* cq) override; + ::grpc::ClientAsyncResponseReader< ::milvus::grpc::Status>* PrepareAsyncDropTableRaw(::grpc::ClientContext* context, const ::milvus::grpc::TableName& request, ::grpc::CompletionQueue* cq) override; + ::grpc::ClientAsyncResponseReader< ::milvus::grpc::Status>* AsyncCreateIndexRaw(::grpc::ClientContext* context, const ::milvus::grpc::IndexParam& request, ::grpc::CompletionQueue* cq) override; + ::grpc::ClientAsyncResponseReader< ::milvus::grpc::Status>* PrepareAsyncCreateIndexRaw(::grpc::ClientContext* context, const ::milvus::grpc::IndexParam& request, ::grpc::CompletionQueue* cq) override; ::grpc::ClientAsyncResponseReader< ::milvus::grpc::IndexParam>* AsyncDescribeIndexRaw(::grpc::ClientContext* context, const ::milvus::grpc::TableName& request, ::grpc::CompletionQueue* cq) override; ::grpc::ClientAsyncResponseReader< ::milvus::grpc::IndexParam>* PrepareAsyncDescribeIndexRaw(::grpc::ClientContext* context, const ::milvus::grpc::TableName& request, ::grpc::CompletionQueue* cq) override; ::grpc::ClientAsyncResponseReader< ::milvus::grpc::Status>* AsyncDropIndexRaw(::grpc::ClientContext* context, const ::milvus::grpc::TableName& request, ::grpc::CompletionQueue* cq) override; ::grpc::ClientAsyncResponseReader< ::milvus::grpc::Status>* PrepareAsyncDropIndexRaw(::grpc::ClientContext* context, const ::milvus::grpc::TableName& request, ::grpc::CompletionQueue* cq) override; + ::grpc::ClientAsyncResponseReader< ::milvus::grpc::Status>* AsyncCreatePartitionRaw(::grpc::ClientContext* context, const ::milvus::grpc::PartitionParam& request, ::grpc::CompletionQueue* cq) override; + ::grpc::ClientAsyncResponseReader< ::milvus::grpc::Status>* PrepareAsyncCreatePartitionRaw(::grpc::ClientContext* context, const ::milvus::grpc::PartitionParam& request, ::grpc::CompletionQueue* cq) override; + ::grpc::ClientAsyncResponseReader< ::milvus::grpc::PartitionList>* AsyncShowPartitionsRaw(::grpc::ClientContext* context, const ::milvus::grpc::TableName& request, ::grpc::CompletionQueue* cq) override; + ::grpc::ClientAsyncResponseReader< ::milvus::grpc::PartitionList>* PrepareAsyncShowPartitionsRaw(::grpc::ClientContext* context, const ::milvus::grpc::TableName& request, ::grpc::CompletionQueue* cq) override; + ::grpc::ClientAsyncResponseReader< ::milvus::grpc::Status>* AsyncDropPartitionRaw(::grpc::ClientContext* context, const ::milvus::grpc::PartitionParam& request, ::grpc::CompletionQueue* cq) override; + ::grpc::ClientAsyncResponseReader< ::milvus::grpc::Status>* PrepareAsyncDropPartitionRaw(::grpc::ClientContext* context, const ::milvus::grpc::PartitionParam& request, ::grpc::CompletionQueue* cq) override; + ::grpc::ClientAsyncResponseReader< ::milvus::grpc::VectorIds>* AsyncInsertRaw(::grpc::ClientContext* context, const ::milvus::grpc::InsertParam& request, ::grpc::CompletionQueue* cq) override; + ::grpc::ClientAsyncResponseReader< ::milvus::grpc::VectorIds>* PrepareAsyncInsertRaw(::grpc::ClientContext* context, const ::milvus::grpc::InsertParam& request, ::grpc::CompletionQueue* cq) override; + ::grpc::ClientAsyncResponseReader< ::milvus::grpc::TopKQueryResult>* AsyncSearchRaw(::grpc::ClientContext* context, const ::milvus::grpc::SearchParam& request, ::grpc::CompletionQueue* cq) override; + ::grpc::ClientAsyncResponseReader< ::milvus::grpc::TopKQueryResult>* PrepareAsyncSearchRaw(::grpc::ClientContext* context, const ::milvus::grpc::SearchParam& request, ::grpc::CompletionQueue* cq) override; + ::grpc::ClientAsyncResponseReader< ::milvus::grpc::TopKQueryResult>* AsyncSearchInFilesRaw(::grpc::ClientContext* context, const ::milvus::grpc::SearchInFilesParam& request, ::grpc::CompletionQueue* cq) override; + ::grpc::ClientAsyncResponseReader< ::milvus::grpc::TopKQueryResult>* PrepareAsyncSearchInFilesRaw(::grpc::ClientContext* context, const ::milvus::grpc::SearchInFilesParam& request, ::grpc::CompletionQueue* cq) override; + ::grpc::ClientAsyncResponseReader< ::milvus::grpc::StringReply>* AsyncCmdRaw(::grpc::ClientContext* context, const ::milvus::grpc::Command& request, ::grpc::CompletionQueue* cq) override; + ::grpc::ClientAsyncResponseReader< ::milvus::grpc::StringReply>* PrepareAsyncCmdRaw(::grpc::ClientContext* context, const ::milvus::grpc::Command& request, ::grpc::CompletionQueue* cq) override; + ::grpc::ClientAsyncResponseReader< ::milvus::grpc::Status>* AsyncDeleteByDateRaw(::grpc::ClientContext* context, const ::milvus::grpc::DeleteByDateParam& request, ::grpc::CompletionQueue* cq) override; + ::grpc::ClientAsyncResponseReader< ::milvus::grpc::Status>* PrepareAsyncDeleteByDateRaw(::grpc::ClientContext* context, const ::milvus::grpc::DeleteByDateParam& request, ::grpc::CompletionQueue* cq) override; + ::grpc::ClientAsyncResponseReader< ::milvus::grpc::Status>* AsyncPreloadTableRaw(::grpc::ClientContext* context, const ::milvus::grpc::TableName& request, ::grpc::CompletionQueue* cq) override; + ::grpc::ClientAsyncResponseReader< ::milvus::grpc::Status>* PrepareAsyncPreloadTableRaw(::grpc::ClientContext* context, const ::milvus::grpc::TableName& request, ::grpc::CompletionQueue* cq) override; const ::grpc::internal::RpcMethod rpcmethod_CreateTable_; const ::grpc::internal::RpcMethod rpcmethod_HasTable_; - const ::grpc::internal::RpcMethod rpcmethod_DropTable_; - const ::grpc::internal::RpcMethod rpcmethod_CreateIndex_; - const ::grpc::internal::RpcMethod rpcmethod_Insert_; - const ::grpc::internal::RpcMethod rpcmethod_Search_; - const ::grpc::internal::RpcMethod rpcmethod_SearchInFiles_; const ::grpc::internal::RpcMethod rpcmethod_DescribeTable_; const ::grpc::internal::RpcMethod rpcmethod_CountTable_; const ::grpc::internal::RpcMethod rpcmethod_ShowTables_; - const ::grpc::internal::RpcMethod rpcmethod_Cmd_; - const ::grpc::internal::RpcMethod rpcmethod_DeleteByRange_; - const ::grpc::internal::RpcMethod rpcmethod_PreloadTable_; + const ::grpc::internal::RpcMethod rpcmethod_DropTable_; + const ::grpc::internal::RpcMethod rpcmethod_CreateIndex_; const ::grpc::internal::RpcMethod rpcmethod_DescribeIndex_; const ::grpc::internal::RpcMethod rpcmethod_DropIndex_; + const ::grpc::internal::RpcMethod rpcmethod_CreatePartition_; + const ::grpc::internal::RpcMethod rpcmethod_ShowPartitions_; + const ::grpc::internal::RpcMethod rpcmethod_DropPartition_; + const ::grpc::internal::RpcMethod rpcmethod_Insert_; + const ::grpc::internal::RpcMethod rpcmethod_Search_; + const ::grpc::internal::RpcMethod rpcmethod_SearchInFiles_; + const ::grpc::internal::RpcMethod rpcmethod_Cmd_; + const ::grpc::internal::RpcMethod rpcmethod_DeleteByDate_; + const ::grpc::internal::RpcMethod rpcmethod_PreloadTable_; }; static std::unique_ptr NewStub(const std::shared_ptr< ::grpc::ChannelInterface>& channel, const ::grpc::StubOptions& options = ::grpc::StubOptions()); @@ -708,132 +781,131 @@ class MilvusService final { Service(); virtual ~Service(); // * - // @brief Create table method + // @brief This method is used to create table // - // This method is used to create table - // - // @param param, use to provide table information to be created. + // @param TableSchema, use to provide table information to be created. // + // @return Status virtual ::grpc::Status CreateTable(::grpc::ServerContext* context, const ::milvus::grpc::TableSchema* request, ::milvus::grpc::Status* response); // * - // @brief Test table existence method + // @brief This method is used to test table existence. // - // This method is used to test table existence. - // - // @param table_name, table name is going to be tested. + // @param TableName, table name is going to be tested. // + // @return BoolReply virtual ::grpc::Status HasTable(::grpc::ServerContext* context, const ::milvus::grpc::TableName* request, ::milvus::grpc::BoolReply* response); // * - // @brief Delete table method + // @brief This method is used to get table schema. // - // This method is used to delete table. + // @param TableName, target table name. // - // @param table_name, table name is going to be deleted. - // - virtual ::grpc::Status DropTable(::grpc::ServerContext* context, const ::milvus::grpc::TableName* request, ::milvus::grpc::Status* response); - // * - // @brief Build index by table method - // - // This method is used to build index by table in sync mode. - // - // @param table_name, table is going to be built index. - // - virtual ::grpc::Status CreateIndex(::grpc::ServerContext* context, const ::milvus::grpc::IndexParam* request, ::milvus::grpc::Status* response); - // * - // @brief Add vector array to table - // - // This method is used to add vector array to table. - // - // @param table_name, table_name is inserted. - // @param record_array, vector array is inserted. - // - // @return vector id array - virtual ::grpc::Status Insert(::grpc::ServerContext* context, const ::milvus::grpc::InsertParam* request, ::milvus::grpc::VectorIds* response); - // * - // @brief Query vector - // - // This method is used to query vector in table. - // - // @param table_name, table_name is queried. - // @param query_record_array, all vector are going to be queried. - // @param query_range_array, optional ranges for conditional search. If not specified, search whole table - // @param topk, how many similarity vectors will be searched. - // - // @return query result array. - virtual ::grpc::Status Search(::grpc::ServerContext* context, const ::milvus::grpc::SearchParam* request, ::milvus::grpc::TopKQueryResult* response); - // * - // @brief Internal use query interface - // - // This method is used to query vector in specified files. - // - // @param file_id_array, specified files id array, queried. - // @param query_record_array, all vector are going to be queried. - // @param query_range_array, optional ranges for conditional search. If not specified, search whole table - // @param topk, how many similarity vectors will be searched. - // - // @return query result array. - virtual ::grpc::Status SearchInFiles(::grpc::ServerContext* context, const ::milvus::grpc::SearchInFilesParam* request, ::milvus::grpc::TopKQueryResult* response); - // * - // @brief Get table schema - // - // This method is used to get table schema. - // - // @param table_name, target table name. - // - // @return table schema + // @return TableSchema virtual ::grpc::Status DescribeTable(::grpc::ServerContext* context, const ::milvus::grpc::TableName* request, ::milvus::grpc::TableSchema* response); // * - // @brief Get table schema + // @brief This method is used to get table schema. // - // This method is used to get table schema. + // @param TableName, target table name. // - // @param table_name, target table name. - // - // @return table schema + // @return TableRowCount virtual ::grpc::Status CountTable(::grpc::ServerContext* context, const ::milvus::grpc::TableName* request, ::milvus::grpc::TableRowCount* response); // * - // @brief List all tables in database + // @brief This method is used to list all tables. // - // This method is used to list all tables. + // @param Command, dummy parameter. // - // - // @return table names. + // @return TableNameList virtual ::grpc::Status ShowTables(::grpc::ServerContext* context, const ::milvus::grpc::Command* request, ::milvus::grpc::TableNameList* response); // * - // @brief Give the server status + // @brief This method is used to delete table. // - // This method is used to give the server status. + // @param TableName, table name is going to be deleted. // - // @return Server status. - virtual ::grpc::Status Cmd(::grpc::ServerContext* context, const ::milvus::grpc::Command* request, ::milvus::grpc::StringReply* response); + // @return TableNameList + virtual ::grpc::Status DropTable(::grpc::ServerContext* context, const ::milvus::grpc::TableName* request, ::milvus::grpc::Status* response); // * - // @brief delete table by range + // @brief This method is used to build index by table in sync mode. // - // This method is used to delete vector by range + // @param IndexParam, index paramters. // - // @return rpc status. - virtual ::grpc::Status DeleteByRange(::grpc::ServerContext* context, const ::milvus::grpc::DeleteByRangeParam* request, ::milvus::grpc::Status* response); + // @return Status + virtual ::grpc::Status CreateIndex(::grpc::ServerContext* context, const ::milvus::grpc::IndexParam* request, ::milvus::grpc::Status* response); // * - // @brief preload table + // @brief This method is used to describe index // - // This method is used to preload table + // @param TableName, target table name. // - // @return Status. - virtual ::grpc::Status PreloadTable(::grpc::ServerContext* context, const ::milvus::grpc::TableName* request, ::milvus::grpc::Status* response); - // * - // @brief describe index - // - // This method is used to describe index - // - // @return Status. + // @return IndexParam virtual ::grpc::Status DescribeIndex(::grpc::ServerContext* context, const ::milvus::grpc::TableName* request, ::milvus::grpc::IndexParam* response); // * - // @brief drop index + // @brief This method is used to drop index // - // This method is used to drop index + // @param TableName, target table name. // - // @return Status. + // @return Status virtual ::grpc::Status DropIndex(::grpc::ServerContext* context, const ::milvus::grpc::TableName* request, ::milvus::grpc::Status* response); + // * + // @brief This method is used to create partition + // + // @param PartitionParam, partition parameters. + // + // @return Status + virtual ::grpc::Status CreatePartition(::grpc::ServerContext* context, const ::milvus::grpc::PartitionParam* request, ::milvus::grpc::Status* response); + // * + // @brief This method is used to show partition information + // + // @param TableName, target table name. + // + // @return PartitionList + virtual ::grpc::Status ShowPartitions(::grpc::ServerContext* context, const ::milvus::grpc::TableName* request, ::milvus::grpc::PartitionList* response); + // * + // @brief This method is used to drop partition + // + // @param PartitionParam, target partition. + // + // @return Status + virtual ::grpc::Status DropPartition(::grpc::ServerContext* context, const ::milvus::grpc::PartitionParam* request, ::milvus::grpc::Status* response); + // * + // @brief This method is used to add vector array to table. + // + // @param InsertParam, insert parameters. + // + // @return VectorIds + virtual ::grpc::Status Insert(::grpc::ServerContext* context, const ::milvus::grpc::InsertParam* request, ::milvus::grpc::VectorIds* response); + // * + // @brief This method is used to query vector in table. + // + // @param SearchParam, search parameters. + // + // @return TopKQueryResult + virtual ::grpc::Status Search(::grpc::ServerContext* context, const ::milvus::grpc::SearchParam* request, ::milvus::grpc::TopKQueryResult* response); + // * + // @brief This method is used to query vector in specified files. + // + // @param SearchInFilesParam, search in files paremeters. + // + // @return TopKQueryResult + virtual ::grpc::Status SearchInFiles(::grpc::ServerContext* context, const ::milvus::grpc::SearchInFilesParam* request, ::milvus::grpc::TopKQueryResult* response); + // * + // @brief This method is used to give the server status. + // + // @param Command, command string + // + // @return StringReply + virtual ::grpc::Status Cmd(::grpc::ServerContext* context, const ::milvus::grpc::Command* request, ::milvus::grpc::StringReply* response); + // * + // @brief This method is used to delete vector by date range + // + // @param DeleteByDateParam, delete parameters. + // + // @return status + virtual ::grpc::Status DeleteByDate(::grpc::ServerContext* context, const ::milvus::grpc::DeleteByDateParam* request, ::milvus::grpc::Status* response); + // * + // @brief This method is used to preload table + // + // @param TableName, target table name. + // + // @return Status + virtual ::grpc::Status PreloadTable(::grpc::ServerContext* context, const ::milvus::grpc::TableName* request, ::milvus::grpc::Status* response); }; template class WithAsyncMethod_CreateTable : public BaseClass { @@ -876,112 +948,12 @@ class MilvusService final { } }; template - class WithAsyncMethod_DropTable : public BaseClass { - private: - void BaseClassMustBeDerivedFromService(const Service* /*service*/) {} - public: - WithAsyncMethod_DropTable() { - ::grpc::Service::MarkMethodAsync(2); - } - ~WithAsyncMethod_DropTable() override { - BaseClassMustBeDerivedFromService(this); - } - // disable synchronous version of this method - ::grpc::Status DropTable(::grpc::ServerContext* /*context*/, const ::milvus::grpc::TableName* /*request*/, ::milvus::grpc::Status* /*response*/) override { - abort(); - return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, ""); - } - void RequestDropTable(::grpc::ServerContext* context, ::milvus::grpc::TableName* request, ::grpc::ServerAsyncResponseWriter< ::milvus::grpc::Status>* response, ::grpc::CompletionQueue* new_call_cq, ::grpc::ServerCompletionQueue* notification_cq, void *tag) { - ::grpc::Service::RequestAsyncUnary(2, context, request, response, new_call_cq, notification_cq, tag); - } - }; - template - class WithAsyncMethod_CreateIndex : public BaseClass { - private: - void BaseClassMustBeDerivedFromService(const Service* /*service*/) {} - public: - WithAsyncMethod_CreateIndex() { - ::grpc::Service::MarkMethodAsync(3); - } - ~WithAsyncMethod_CreateIndex() override { - BaseClassMustBeDerivedFromService(this); - } - // disable synchronous version of this method - ::grpc::Status CreateIndex(::grpc::ServerContext* /*context*/, const ::milvus::grpc::IndexParam* /*request*/, ::milvus::grpc::Status* /*response*/) override { - abort(); - return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, ""); - } - void RequestCreateIndex(::grpc::ServerContext* context, ::milvus::grpc::IndexParam* request, ::grpc::ServerAsyncResponseWriter< ::milvus::grpc::Status>* response, ::grpc::CompletionQueue* new_call_cq, ::grpc::ServerCompletionQueue* notification_cq, void *tag) { - ::grpc::Service::RequestAsyncUnary(3, context, request, response, new_call_cq, notification_cq, tag); - } - }; - template - class WithAsyncMethod_Insert : public BaseClass { - private: - void BaseClassMustBeDerivedFromService(const Service* /*service*/) {} - public: - WithAsyncMethod_Insert() { - ::grpc::Service::MarkMethodAsync(4); - } - ~WithAsyncMethod_Insert() override { - BaseClassMustBeDerivedFromService(this); - } - // disable synchronous version of this method - ::grpc::Status Insert(::grpc::ServerContext* /*context*/, const ::milvus::grpc::InsertParam* /*request*/, ::milvus::grpc::VectorIds* /*response*/) override { - abort(); - return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, ""); - } - void RequestInsert(::grpc::ServerContext* context, ::milvus::grpc::InsertParam* request, ::grpc::ServerAsyncResponseWriter< ::milvus::grpc::VectorIds>* response, ::grpc::CompletionQueue* new_call_cq, ::grpc::ServerCompletionQueue* notification_cq, void *tag) { - ::grpc::Service::RequestAsyncUnary(4, context, request, response, new_call_cq, notification_cq, tag); - } - }; - template - class WithAsyncMethod_Search : public BaseClass { - private: - void BaseClassMustBeDerivedFromService(const Service* /*service*/) {} - public: - WithAsyncMethod_Search() { - ::grpc::Service::MarkMethodAsync(5); - } - ~WithAsyncMethod_Search() override { - BaseClassMustBeDerivedFromService(this); - } - // disable synchronous version of this method - ::grpc::Status Search(::grpc::ServerContext* /*context*/, const ::milvus::grpc::SearchParam* /*request*/, ::milvus::grpc::TopKQueryResult* /*response*/) override { - abort(); - return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, ""); - } - void RequestSearch(::grpc::ServerContext* context, ::milvus::grpc::SearchParam* request, ::grpc::ServerAsyncResponseWriter< ::milvus::grpc::TopKQueryResult>* response, ::grpc::CompletionQueue* new_call_cq, ::grpc::ServerCompletionQueue* notification_cq, void *tag) { - ::grpc::Service::RequestAsyncUnary(5, context, request, response, new_call_cq, notification_cq, tag); - } - }; - template - class WithAsyncMethod_SearchInFiles : public BaseClass { - private: - void BaseClassMustBeDerivedFromService(const Service* /*service*/) {} - public: - WithAsyncMethod_SearchInFiles() { - ::grpc::Service::MarkMethodAsync(6); - } - ~WithAsyncMethod_SearchInFiles() override { - BaseClassMustBeDerivedFromService(this); - } - // disable synchronous version of this method - ::grpc::Status SearchInFiles(::grpc::ServerContext* /*context*/, const ::milvus::grpc::SearchInFilesParam* /*request*/, ::milvus::grpc::TopKQueryResult* /*response*/) override { - abort(); - return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, ""); - } - void RequestSearchInFiles(::grpc::ServerContext* context, ::milvus::grpc::SearchInFilesParam* request, ::grpc::ServerAsyncResponseWriter< ::milvus::grpc::TopKQueryResult>* response, ::grpc::CompletionQueue* new_call_cq, ::grpc::ServerCompletionQueue* notification_cq, void *tag) { - ::grpc::Service::RequestAsyncUnary(6, context, request, response, new_call_cq, notification_cq, tag); - } - }; - template class WithAsyncMethod_DescribeTable : public BaseClass { private: void BaseClassMustBeDerivedFromService(const Service* /*service*/) {} public: WithAsyncMethod_DescribeTable() { - ::grpc::Service::MarkMethodAsync(7); + ::grpc::Service::MarkMethodAsync(2); } ~WithAsyncMethod_DescribeTable() override { BaseClassMustBeDerivedFromService(this); @@ -992,7 +964,7 @@ class MilvusService final { return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, ""); } void RequestDescribeTable(::grpc::ServerContext* context, ::milvus::grpc::TableName* request, ::grpc::ServerAsyncResponseWriter< ::milvus::grpc::TableSchema>* response, ::grpc::CompletionQueue* new_call_cq, ::grpc::ServerCompletionQueue* notification_cq, void *tag) { - ::grpc::Service::RequestAsyncUnary(7, context, request, response, new_call_cq, notification_cq, tag); + ::grpc::Service::RequestAsyncUnary(2, context, request, response, new_call_cq, notification_cq, tag); } }; template @@ -1001,7 +973,7 @@ class MilvusService final { void BaseClassMustBeDerivedFromService(const Service* /*service*/) {} public: WithAsyncMethod_CountTable() { - ::grpc::Service::MarkMethodAsync(8); + ::grpc::Service::MarkMethodAsync(3); } ~WithAsyncMethod_CountTable() override { BaseClassMustBeDerivedFromService(this); @@ -1012,7 +984,7 @@ class MilvusService final { return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, ""); } void RequestCountTable(::grpc::ServerContext* context, ::milvus::grpc::TableName* request, ::grpc::ServerAsyncResponseWriter< ::milvus::grpc::TableRowCount>* response, ::grpc::CompletionQueue* new_call_cq, ::grpc::ServerCompletionQueue* notification_cq, void *tag) { - ::grpc::Service::RequestAsyncUnary(8, context, request, response, new_call_cq, notification_cq, tag); + ::grpc::Service::RequestAsyncUnary(3, context, request, response, new_call_cq, notification_cq, tag); } }; template @@ -1021,7 +993,7 @@ class MilvusService final { void BaseClassMustBeDerivedFromService(const Service* /*service*/) {} public: WithAsyncMethod_ShowTables() { - ::grpc::Service::MarkMethodAsync(9); + ::grpc::Service::MarkMethodAsync(4); } ~WithAsyncMethod_ShowTables() override { BaseClassMustBeDerivedFromService(this); @@ -1032,67 +1004,47 @@ class MilvusService final { return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, ""); } void RequestShowTables(::grpc::ServerContext* context, ::milvus::grpc::Command* request, ::grpc::ServerAsyncResponseWriter< ::milvus::grpc::TableNameList>* response, ::grpc::CompletionQueue* new_call_cq, ::grpc::ServerCompletionQueue* notification_cq, void *tag) { - ::grpc::Service::RequestAsyncUnary(9, context, request, response, new_call_cq, notification_cq, tag); + ::grpc::Service::RequestAsyncUnary(4, context, request, response, new_call_cq, notification_cq, tag); } }; template - class WithAsyncMethod_Cmd : public BaseClass { + class WithAsyncMethod_DropTable : public BaseClass { private: void BaseClassMustBeDerivedFromService(const Service* /*service*/) {} public: - WithAsyncMethod_Cmd() { - ::grpc::Service::MarkMethodAsync(10); + WithAsyncMethod_DropTable() { + ::grpc::Service::MarkMethodAsync(5); } - ~WithAsyncMethod_Cmd() override { + ~WithAsyncMethod_DropTable() override { BaseClassMustBeDerivedFromService(this); } // disable synchronous version of this method - ::grpc::Status Cmd(::grpc::ServerContext* /*context*/, const ::milvus::grpc::Command* /*request*/, ::milvus::grpc::StringReply* /*response*/) override { + ::grpc::Status DropTable(::grpc::ServerContext* /*context*/, const ::milvus::grpc::TableName* /*request*/, ::milvus::grpc::Status* /*response*/) override { abort(); return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, ""); } - void RequestCmd(::grpc::ServerContext* context, ::milvus::grpc::Command* request, ::grpc::ServerAsyncResponseWriter< ::milvus::grpc::StringReply>* response, ::grpc::CompletionQueue* new_call_cq, ::grpc::ServerCompletionQueue* notification_cq, void *tag) { - ::grpc::Service::RequestAsyncUnary(10, context, request, response, new_call_cq, notification_cq, tag); + void RequestDropTable(::grpc::ServerContext* context, ::milvus::grpc::TableName* request, ::grpc::ServerAsyncResponseWriter< ::milvus::grpc::Status>* response, ::grpc::CompletionQueue* new_call_cq, ::grpc::ServerCompletionQueue* notification_cq, void *tag) { + ::grpc::Service::RequestAsyncUnary(5, context, request, response, new_call_cq, notification_cq, tag); } }; template - class WithAsyncMethod_DeleteByRange : public BaseClass { + class WithAsyncMethod_CreateIndex : public BaseClass { private: void BaseClassMustBeDerivedFromService(const Service* /*service*/) {} public: - WithAsyncMethod_DeleteByRange() { - ::grpc::Service::MarkMethodAsync(11); + WithAsyncMethod_CreateIndex() { + ::grpc::Service::MarkMethodAsync(6); } - ~WithAsyncMethod_DeleteByRange() override { + ~WithAsyncMethod_CreateIndex() override { BaseClassMustBeDerivedFromService(this); } // disable synchronous version of this method - ::grpc::Status DeleteByRange(::grpc::ServerContext* /*context*/, const ::milvus::grpc::DeleteByRangeParam* /*request*/, ::milvus::grpc::Status* /*response*/) override { + ::grpc::Status CreateIndex(::grpc::ServerContext* /*context*/, const ::milvus::grpc::IndexParam* /*request*/, ::milvus::grpc::Status* /*response*/) override { abort(); return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, ""); } - void RequestDeleteByRange(::grpc::ServerContext* context, ::milvus::grpc::DeleteByRangeParam* request, ::grpc::ServerAsyncResponseWriter< ::milvus::grpc::Status>* response, ::grpc::CompletionQueue* new_call_cq, ::grpc::ServerCompletionQueue* notification_cq, void *tag) { - ::grpc::Service::RequestAsyncUnary(11, context, request, response, new_call_cq, notification_cq, tag); - } - }; - template - class WithAsyncMethod_PreloadTable : public BaseClass { - private: - void BaseClassMustBeDerivedFromService(const Service* /*service*/) {} - public: - WithAsyncMethod_PreloadTable() { - ::grpc::Service::MarkMethodAsync(12); - } - ~WithAsyncMethod_PreloadTable() override { - BaseClassMustBeDerivedFromService(this); - } - // disable synchronous version of this method - ::grpc::Status PreloadTable(::grpc::ServerContext* /*context*/, const ::milvus::grpc::TableName* /*request*/, ::milvus::grpc::Status* /*response*/) override { - abort(); - return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, ""); - } - void RequestPreloadTable(::grpc::ServerContext* context, ::milvus::grpc::TableName* request, ::grpc::ServerAsyncResponseWriter< ::milvus::grpc::Status>* response, ::grpc::CompletionQueue* new_call_cq, ::grpc::ServerCompletionQueue* notification_cq, void *tag) { - ::grpc::Service::RequestAsyncUnary(12, context, request, response, new_call_cq, notification_cq, tag); + void RequestCreateIndex(::grpc::ServerContext* context, ::milvus::grpc::IndexParam* request, ::grpc::ServerAsyncResponseWriter< ::milvus::grpc::Status>* response, ::grpc::CompletionQueue* new_call_cq, ::grpc::ServerCompletionQueue* notification_cq, void *tag) { + ::grpc::Service::RequestAsyncUnary(6, context, request, response, new_call_cq, notification_cq, tag); } }; template @@ -1101,7 +1053,7 @@ class MilvusService final { void BaseClassMustBeDerivedFromService(const Service* /*service*/) {} public: WithAsyncMethod_DescribeIndex() { - ::grpc::Service::MarkMethodAsync(13); + ::grpc::Service::MarkMethodAsync(7); } ~WithAsyncMethod_DescribeIndex() override { BaseClassMustBeDerivedFromService(this); @@ -1112,7 +1064,7 @@ class MilvusService final { return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, ""); } void RequestDescribeIndex(::grpc::ServerContext* context, ::milvus::grpc::TableName* request, ::grpc::ServerAsyncResponseWriter< ::milvus::grpc::IndexParam>* response, ::grpc::CompletionQueue* new_call_cq, ::grpc::ServerCompletionQueue* notification_cq, void *tag) { - ::grpc::Service::RequestAsyncUnary(13, context, request, response, new_call_cq, notification_cq, tag); + ::grpc::Service::RequestAsyncUnary(7, context, request, response, new_call_cq, notification_cq, tag); } }; template @@ -1121,7 +1073,7 @@ class MilvusService final { void BaseClassMustBeDerivedFromService(const Service* /*service*/) {} public: WithAsyncMethod_DropIndex() { - ::grpc::Service::MarkMethodAsync(14); + ::grpc::Service::MarkMethodAsync(8); } ~WithAsyncMethod_DropIndex() override { BaseClassMustBeDerivedFromService(this); @@ -1132,10 +1084,190 @@ class MilvusService final { return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, ""); } void RequestDropIndex(::grpc::ServerContext* context, ::milvus::grpc::TableName* request, ::grpc::ServerAsyncResponseWriter< ::milvus::grpc::Status>* response, ::grpc::CompletionQueue* new_call_cq, ::grpc::ServerCompletionQueue* notification_cq, void *tag) { + ::grpc::Service::RequestAsyncUnary(8, context, request, response, new_call_cq, notification_cq, tag); + } + }; + template + class WithAsyncMethod_CreatePartition : public BaseClass { + private: + void BaseClassMustBeDerivedFromService(const Service* /*service*/) {} + public: + WithAsyncMethod_CreatePartition() { + ::grpc::Service::MarkMethodAsync(9); + } + ~WithAsyncMethod_CreatePartition() override { + BaseClassMustBeDerivedFromService(this); + } + // disable synchronous version of this method + ::grpc::Status CreatePartition(::grpc::ServerContext* /*context*/, const ::milvus::grpc::PartitionParam* /*request*/, ::milvus::grpc::Status* /*response*/) override { + abort(); + return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, ""); + } + void RequestCreatePartition(::grpc::ServerContext* context, ::milvus::grpc::PartitionParam* request, ::grpc::ServerAsyncResponseWriter< ::milvus::grpc::Status>* response, ::grpc::CompletionQueue* new_call_cq, ::grpc::ServerCompletionQueue* notification_cq, void *tag) { + ::grpc::Service::RequestAsyncUnary(9, context, request, response, new_call_cq, notification_cq, tag); + } + }; + template + class WithAsyncMethod_ShowPartitions : public BaseClass { + private: + void BaseClassMustBeDerivedFromService(const Service* /*service*/) {} + public: + WithAsyncMethod_ShowPartitions() { + ::grpc::Service::MarkMethodAsync(10); + } + ~WithAsyncMethod_ShowPartitions() override { + BaseClassMustBeDerivedFromService(this); + } + // disable synchronous version of this method + ::grpc::Status ShowPartitions(::grpc::ServerContext* /*context*/, const ::milvus::grpc::TableName* /*request*/, ::milvus::grpc::PartitionList* /*response*/) override { + abort(); + return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, ""); + } + void RequestShowPartitions(::grpc::ServerContext* context, ::milvus::grpc::TableName* request, ::grpc::ServerAsyncResponseWriter< ::milvus::grpc::PartitionList>* response, ::grpc::CompletionQueue* new_call_cq, ::grpc::ServerCompletionQueue* notification_cq, void *tag) { + ::grpc::Service::RequestAsyncUnary(10, context, request, response, new_call_cq, notification_cq, tag); + } + }; + template + class WithAsyncMethod_DropPartition : public BaseClass { + private: + void BaseClassMustBeDerivedFromService(const Service* /*service*/) {} + public: + WithAsyncMethod_DropPartition() { + ::grpc::Service::MarkMethodAsync(11); + } + ~WithAsyncMethod_DropPartition() override { + BaseClassMustBeDerivedFromService(this); + } + // disable synchronous version of this method + ::grpc::Status DropPartition(::grpc::ServerContext* /*context*/, const ::milvus::grpc::PartitionParam* /*request*/, ::milvus::grpc::Status* /*response*/) override { + abort(); + return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, ""); + } + void RequestDropPartition(::grpc::ServerContext* context, ::milvus::grpc::PartitionParam* request, ::grpc::ServerAsyncResponseWriter< ::milvus::grpc::Status>* response, ::grpc::CompletionQueue* new_call_cq, ::grpc::ServerCompletionQueue* notification_cq, void *tag) { + ::grpc::Service::RequestAsyncUnary(11, context, request, response, new_call_cq, notification_cq, tag); + } + }; + template + class WithAsyncMethod_Insert : public BaseClass { + private: + void BaseClassMustBeDerivedFromService(const Service* /*service*/) {} + public: + WithAsyncMethod_Insert() { + ::grpc::Service::MarkMethodAsync(12); + } + ~WithAsyncMethod_Insert() override { + BaseClassMustBeDerivedFromService(this); + } + // disable synchronous version of this method + ::grpc::Status Insert(::grpc::ServerContext* /*context*/, const ::milvus::grpc::InsertParam* /*request*/, ::milvus::grpc::VectorIds* /*response*/) override { + abort(); + return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, ""); + } + void RequestInsert(::grpc::ServerContext* context, ::milvus::grpc::InsertParam* request, ::grpc::ServerAsyncResponseWriter< ::milvus::grpc::VectorIds>* response, ::grpc::CompletionQueue* new_call_cq, ::grpc::ServerCompletionQueue* notification_cq, void *tag) { + ::grpc::Service::RequestAsyncUnary(12, context, request, response, new_call_cq, notification_cq, tag); + } + }; + template + class WithAsyncMethod_Search : public BaseClass { + private: + void BaseClassMustBeDerivedFromService(const Service* /*service*/) {} + public: + WithAsyncMethod_Search() { + ::grpc::Service::MarkMethodAsync(13); + } + ~WithAsyncMethod_Search() override { + BaseClassMustBeDerivedFromService(this); + } + // disable synchronous version of this method + ::grpc::Status Search(::grpc::ServerContext* /*context*/, const ::milvus::grpc::SearchParam* /*request*/, ::milvus::grpc::TopKQueryResult* /*response*/) override { + abort(); + return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, ""); + } + void RequestSearch(::grpc::ServerContext* context, ::milvus::grpc::SearchParam* request, ::grpc::ServerAsyncResponseWriter< ::milvus::grpc::TopKQueryResult>* response, ::grpc::CompletionQueue* new_call_cq, ::grpc::ServerCompletionQueue* notification_cq, void *tag) { + ::grpc::Service::RequestAsyncUnary(13, context, request, response, new_call_cq, notification_cq, tag); + } + }; + template + class WithAsyncMethod_SearchInFiles : public BaseClass { + private: + void BaseClassMustBeDerivedFromService(const Service* /*service*/) {} + public: + WithAsyncMethod_SearchInFiles() { + ::grpc::Service::MarkMethodAsync(14); + } + ~WithAsyncMethod_SearchInFiles() override { + BaseClassMustBeDerivedFromService(this); + } + // disable synchronous version of this method + ::grpc::Status SearchInFiles(::grpc::ServerContext* /*context*/, const ::milvus::grpc::SearchInFilesParam* /*request*/, ::milvus::grpc::TopKQueryResult* /*response*/) override { + abort(); + return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, ""); + } + void RequestSearchInFiles(::grpc::ServerContext* context, ::milvus::grpc::SearchInFilesParam* request, ::grpc::ServerAsyncResponseWriter< ::milvus::grpc::TopKQueryResult>* response, ::grpc::CompletionQueue* new_call_cq, ::grpc::ServerCompletionQueue* notification_cq, void *tag) { ::grpc::Service::RequestAsyncUnary(14, context, request, response, new_call_cq, notification_cq, tag); } }; - typedef WithAsyncMethod_CreateTable > > > > > > > > > > > > > > AsyncService; + template + class WithAsyncMethod_Cmd : public BaseClass { + private: + void BaseClassMustBeDerivedFromService(const Service* /*service*/) {} + public: + WithAsyncMethod_Cmd() { + ::grpc::Service::MarkMethodAsync(15); + } + ~WithAsyncMethod_Cmd() override { + BaseClassMustBeDerivedFromService(this); + } + // disable synchronous version of this method + ::grpc::Status Cmd(::grpc::ServerContext* /*context*/, const ::milvus::grpc::Command* /*request*/, ::milvus::grpc::StringReply* /*response*/) override { + abort(); + return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, ""); + } + void RequestCmd(::grpc::ServerContext* context, ::milvus::grpc::Command* request, ::grpc::ServerAsyncResponseWriter< ::milvus::grpc::StringReply>* response, ::grpc::CompletionQueue* new_call_cq, ::grpc::ServerCompletionQueue* notification_cq, void *tag) { + ::grpc::Service::RequestAsyncUnary(15, context, request, response, new_call_cq, notification_cq, tag); + } + }; + template + class WithAsyncMethod_DeleteByDate : public BaseClass { + private: + void BaseClassMustBeDerivedFromService(const Service* /*service*/) {} + public: + WithAsyncMethod_DeleteByDate() { + ::grpc::Service::MarkMethodAsync(16); + } + ~WithAsyncMethod_DeleteByDate() override { + BaseClassMustBeDerivedFromService(this); + } + // disable synchronous version of this method + ::grpc::Status DeleteByDate(::grpc::ServerContext* /*context*/, const ::milvus::grpc::DeleteByDateParam* /*request*/, ::milvus::grpc::Status* /*response*/) override { + abort(); + return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, ""); + } + void RequestDeleteByDate(::grpc::ServerContext* context, ::milvus::grpc::DeleteByDateParam* request, ::grpc::ServerAsyncResponseWriter< ::milvus::grpc::Status>* response, ::grpc::CompletionQueue* new_call_cq, ::grpc::ServerCompletionQueue* notification_cq, void *tag) { + ::grpc::Service::RequestAsyncUnary(16, context, request, response, new_call_cq, notification_cq, tag); + } + }; + template + class WithAsyncMethod_PreloadTable : public BaseClass { + private: + void BaseClassMustBeDerivedFromService(const Service* /*service*/) {} + public: + WithAsyncMethod_PreloadTable() { + ::grpc::Service::MarkMethodAsync(17); + } + ~WithAsyncMethod_PreloadTable() override { + BaseClassMustBeDerivedFromService(this); + } + // disable synchronous version of this method + ::grpc::Status PreloadTable(::grpc::ServerContext* /*context*/, const ::milvus::grpc::TableName* /*request*/, ::milvus::grpc::Status* /*response*/) override { + abort(); + return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, ""); + } + void RequestPreloadTable(::grpc::ServerContext* context, ::milvus::grpc::TableName* request, ::grpc::ServerAsyncResponseWriter< ::milvus::grpc::Status>* response, ::grpc::CompletionQueue* new_call_cq, ::grpc::ServerCompletionQueue* notification_cq, void *tag) { + ::grpc::Service::RequestAsyncUnary(17, context, request, response, new_call_cq, notification_cq, tag); + } + }; + typedef WithAsyncMethod_CreateTable > > > > > > > > > > > > > > > > > AsyncService; template class ExperimentalWithCallbackMethod_CreateTable : public BaseClass { private: @@ -1199,167 +1331,12 @@ class MilvusService final { virtual void HasTable(::grpc::ServerContext* /*context*/, const ::milvus::grpc::TableName* /*request*/, ::milvus::grpc::BoolReply* /*response*/, ::grpc::experimental::ServerCallbackRpcController* controller) { controller->Finish(::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, "")); } }; template - class ExperimentalWithCallbackMethod_DropTable : public BaseClass { - private: - void BaseClassMustBeDerivedFromService(const Service* /*service*/) {} - public: - ExperimentalWithCallbackMethod_DropTable() { - ::grpc::Service::experimental().MarkMethodCallback(2, - new ::grpc_impl::internal::CallbackUnaryHandler< ::milvus::grpc::TableName, ::milvus::grpc::Status>( - [this](::grpc::ServerContext* context, - const ::milvus::grpc::TableName* request, - ::milvus::grpc::Status* response, - ::grpc::experimental::ServerCallbackRpcController* controller) { - return this->DropTable(context, request, response, controller); - })); - } - void SetMessageAllocatorFor_DropTable( - ::grpc::experimental::MessageAllocator< ::milvus::grpc::TableName, ::milvus::grpc::Status>* allocator) { - static_cast<::grpc_impl::internal::CallbackUnaryHandler< ::milvus::grpc::TableName, ::milvus::grpc::Status>*>( - ::grpc::Service::experimental().GetHandler(2)) - ->SetMessageAllocator(allocator); - } - ~ExperimentalWithCallbackMethod_DropTable() override { - BaseClassMustBeDerivedFromService(this); - } - // disable synchronous version of this method - ::grpc::Status DropTable(::grpc::ServerContext* /*context*/, const ::milvus::grpc::TableName* /*request*/, ::milvus::grpc::Status* /*response*/) override { - abort(); - return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, ""); - } - virtual void DropTable(::grpc::ServerContext* /*context*/, const ::milvus::grpc::TableName* /*request*/, ::milvus::grpc::Status* /*response*/, ::grpc::experimental::ServerCallbackRpcController* controller) { controller->Finish(::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, "")); } - }; - template - class ExperimentalWithCallbackMethod_CreateIndex : public BaseClass { - private: - void BaseClassMustBeDerivedFromService(const Service* /*service*/) {} - public: - ExperimentalWithCallbackMethod_CreateIndex() { - ::grpc::Service::experimental().MarkMethodCallback(3, - new ::grpc_impl::internal::CallbackUnaryHandler< ::milvus::grpc::IndexParam, ::milvus::grpc::Status>( - [this](::grpc::ServerContext* context, - const ::milvus::grpc::IndexParam* request, - ::milvus::grpc::Status* response, - ::grpc::experimental::ServerCallbackRpcController* controller) { - return this->CreateIndex(context, request, response, controller); - })); - } - void SetMessageAllocatorFor_CreateIndex( - ::grpc::experimental::MessageAllocator< ::milvus::grpc::IndexParam, ::milvus::grpc::Status>* allocator) { - static_cast<::grpc_impl::internal::CallbackUnaryHandler< ::milvus::grpc::IndexParam, ::milvus::grpc::Status>*>( - ::grpc::Service::experimental().GetHandler(3)) - ->SetMessageAllocator(allocator); - } - ~ExperimentalWithCallbackMethod_CreateIndex() override { - BaseClassMustBeDerivedFromService(this); - } - // disable synchronous version of this method - ::grpc::Status CreateIndex(::grpc::ServerContext* /*context*/, const ::milvus::grpc::IndexParam* /*request*/, ::milvus::grpc::Status* /*response*/) override { - abort(); - return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, ""); - } - virtual void CreateIndex(::grpc::ServerContext* /*context*/, const ::milvus::grpc::IndexParam* /*request*/, ::milvus::grpc::Status* /*response*/, ::grpc::experimental::ServerCallbackRpcController* controller) { controller->Finish(::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, "")); } - }; - template - class ExperimentalWithCallbackMethod_Insert : public BaseClass { - private: - void BaseClassMustBeDerivedFromService(const Service* /*service*/) {} - public: - ExperimentalWithCallbackMethod_Insert() { - ::grpc::Service::experimental().MarkMethodCallback(4, - new ::grpc_impl::internal::CallbackUnaryHandler< ::milvus::grpc::InsertParam, ::milvus::grpc::VectorIds>( - [this](::grpc::ServerContext* context, - const ::milvus::grpc::InsertParam* request, - ::milvus::grpc::VectorIds* response, - ::grpc::experimental::ServerCallbackRpcController* controller) { - return this->Insert(context, request, response, controller); - })); - } - void SetMessageAllocatorFor_Insert( - ::grpc::experimental::MessageAllocator< ::milvus::grpc::InsertParam, ::milvus::grpc::VectorIds>* allocator) { - static_cast<::grpc_impl::internal::CallbackUnaryHandler< ::milvus::grpc::InsertParam, ::milvus::grpc::VectorIds>*>( - ::grpc::Service::experimental().GetHandler(4)) - ->SetMessageAllocator(allocator); - } - ~ExperimentalWithCallbackMethod_Insert() override { - BaseClassMustBeDerivedFromService(this); - } - // disable synchronous version of this method - ::grpc::Status Insert(::grpc::ServerContext* /*context*/, const ::milvus::grpc::InsertParam* /*request*/, ::milvus::grpc::VectorIds* /*response*/) override { - abort(); - return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, ""); - } - virtual void Insert(::grpc::ServerContext* /*context*/, const ::milvus::grpc::InsertParam* /*request*/, ::milvus::grpc::VectorIds* /*response*/, ::grpc::experimental::ServerCallbackRpcController* controller) { controller->Finish(::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, "")); } - }; - template - class ExperimentalWithCallbackMethod_Search : public BaseClass { - private: - void BaseClassMustBeDerivedFromService(const Service* /*service*/) {} - public: - ExperimentalWithCallbackMethod_Search() { - ::grpc::Service::experimental().MarkMethodCallback(5, - new ::grpc_impl::internal::CallbackUnaryHandler< ::milvus::grpc::SearchParam, ::milvus::grpc::TopKQueryResult>( - [this](::grpc::ServerContext* context, - const ::milvus::grpc::SearchParam* request, - ::milvus::grpc::TopKQueryResult* response, - ::grpc::experimental::ServerCallbackRpcController* controller) { - return this->Search(context, request, response, controller); - })); - } - void SetMessageAllocatorFor_Search( - ::grpc::experimental::MessageAllocator< ::milvus::grpc::SearchParam, ::milvus::grpc::TopKQueryResult>* allocator) { - static_cast<::grpc_impl::internal::CallbackUnaryHandler< ::milvus::grpc::SearchParam, ::milvus::grpc::TopKQueryResult>*>( - ::grpc::Service::experimental().GetHandler(5)) - ->SetMessageAllocator(allocator); - } - ~ExperimentalWithCallbackMethod_Search() override { - BaseClassMustBeDerivedFromService(this); - } - // disable synchronous version of this method - ::grpc::Status Search(::grpc::ServerContext* /*context*/, const ::milvus::grpc::SearchParam* /*request*/, ::milvus::grpc::TopKQueryResult* /*response*/) override { - abort(); - return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, ""); - } - virtual void Search(::grpc::ServerContext* /*context*/, const ::milvus::grpc::SearchParam* /*request*/, ::milvus::grpc::TopKQueryResult* /*response*/, ::grpc::experimental::ServerCallbackRpcController* controller) { controller->Finish(::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, "")); } - }; - template - class ExperimentalWithCallbackMethod_SearchInFiles : public BaseClass { - private: - void BaseClassMustBeDerivedFromService(const Service* /*service*/) {} - public: - ExperimentalWithCallbackMethod_SearchInFiles() { - ::grpc::Service::experimental().MarkMethodCallback(6, - new ::grpc_impl::internal::CallbackUnaryHandler< ::milvus::grpc::SearchInFilesParam, ::milvus::grpc::TopKQueryResult>( - [this](::grpc::ServerContext* context, - const ::milvus::grpc::SearchInFilesParam* request, - ::milvus::grpc::TopKQueryResult* response, - ::grpc::experimental::ServerCallbackRpcController* controller) { - return this->SearchInFiles(context, request, response, controller); - })); - } - void SetMessageAllocatorFor_SearchInFiles( - ::grpc::experimental::MessageAllocator< ::milvus::grpc::SearchInFilesParam, ::milvus::grpc::TopKQueryResult>* allocator) { - static_cast<::grpc_impl::internal::CallbackUnaryHandler< ::milvus::grpc::SearchInFilesParam, ::milvus::grpc::TopKQueryResult>*>( - ::grpc::Service::experimental().GetHandler(6)) - ->SetMessageAllocator(allocator); - } - ~ExperimentalWithCallbackMethod_SearchInFiles() override { - BaseClassMustBeDerivedFromService(this); - } - // disable synchronous version of this method - ::grpc::Status SearchInFiles(::grpc::ServerContext* /*context*/, const ::milvus::grpc::SearchInFilesParam* /*request*/, ::milvus::grpc::TopKQueryResult* /*response*/) override { - abort(); - return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, ""); - } - virtual void SearchInFiles(::grpc::ServerContext* /*context*/, const ::milvus::grpc::SearchInFilesParam* /*request*/, ::milvus::grpc::TopKQueryResult* /*response*/, ::grpc::experimental::ServerCallbackRpcController* controller) { controller->Finish(::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, "")); } - }; - template class ExperimentalWithCallbackMethod_DescribeTable : public BaseClass { private: void BaseClassMustBeDerivedFromService(const Service* /*service*/) {} public: ExperimentalWithCallbackMethod_DescribeTable() { - ::grpc::Service::experimental().MarkMethodCallback(7, + ::grpc::Service::experimental().MarkMethodCallback(2, new ::grpc_impl::internal::CallbackUnaryHandler< ::milvus::grpc::TableName, ::milvus::grpc::TableSchema>( [this](::grpc::ServerContext* context, const ::milvus::grpc::TableName* request, @@ -1371,7 +1348,7 @@ class MilvusService final { void SetMessageAllocatorFor_DescribeTable( ::grpc::experimental::MessageAllocator< ::milvus::grpc::TableName, ::milvus::grpc::TableSchema>* allocator) { static_cast<::grpc_impl::internal::CallbackUnaryHandler< ::milvus::grpc::TableName, ::milvus::grpc::TableSchema>*>( - ::grpc::Service::experimental().GetHandler(7)) + ::grpc::Service::experimental().GetHandler(2)) ->SetMessageAllocator(allocator); } ~ExperimentalWithCallbackMethod_DescribeTable() override { @@ -1390,7 +1367,7 @@ class MilvusService final { void BaseClassMustBeDerivedFromService(const Service* /*service*/) {} public: ExperimentalWithCallbackMethod_CountTable() { - ::grpc::Service::experimental().MarkMethodCallback(8, + ::grpc::Service::experimental().MarkMethodCallback(3, new ::grpc_impl::internal::CallbackUnaryHandler< ::milvus::grpc::TableName, ::milvus::grpc::TableRowCount>( [this](::grpc::ServerContext* context, const ::milvus::grpc::TableName* request, @@ -1402,7 +1379,7 @@ class MilvusService final { void SetMessageAllocatorFor_CountTable( ::grpc::experimental::MessageAllocator< ::milvus::grpc::TableName, ::milvus::grpc::TableRowCount>* allocator) { static_cast<::grpc_impl::internal::CallbackUnaryHandler< ::milvus::grpc::TableName, ::milvus::grpc::TableRowCount>*>( - ::grpc::Service::experimental().GetHandler(8)) + ::grpc::Service::experimental().GetHandler(3)) ->SetMessageAllocator(allocator); } ~ExperimentalWithCallbackMethod_CountTable() override { @@ -1421,7 +1398,7 @@ class MilvusService final { void BaseClassMustBeDerivedFromService(const Service* /*service*/) {} public: ExperimentalWithCallbackMethod_ShowTables() { - ::grpc::Service::experimental().MarkMethodCallback(9, + ::grpc::Service::experimental().MarkMethodCallback(4, new ::grpc_impl::internal::CallbackUnaryHandler< ::milvus::grpc::Command, ::milvus::grpc::TableNameList>( [this](::grpc::ServerContext* context, const ::milvus::grpc::Command* request, @@ -1433,7 +1410,7 @@ class MilvusService final { void SetMessageAllocatorFor_ShowTables( ::grpc::experimental::MessageAllocator< ::milvus::grpc::Command, ::milvus::grpc::TableNameList>* allocator) { static_cast<::grpc_impl::internal::CallbackUnaryHandler< ::milvus::grpc::Command, ::milvus::grpc::TableNameList>*>( - ::grpc::Service::experimental().GetHandler(9)) + ::grpc::Service::experimental().GetHandler(4)) ->SetMessageAllocator(allocator); } ~ExperimentalWithCallbackMethod_ShowTables() override { @@ -1447,97 +1424,66 @@ class MilvusService final { virtual void ShowTables(::grpc::ServerContext* /*context*/, const ::milvus::grpc::Command* /*request*/, ::milvus::grpc::TableNameList* /*response*/, ::grpc::experimental::ServerCallbackRpcController* controller) { controller->Finish(::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, "")); } }; template - class ExperimentalWithCallbackMethod_Cmd : public BaseClass { + class ExperimentalWithCallbackMethod_DropTable : public BaseClass { private: void BaseClassMustBeDerivedFromService(const Service* /*service*/) {} public: - ExperimentalWithCallbackMethod_Cmd() { - ::grpc::Service::experimental().MarkMethodCallback(10, - new ::grpc_impl::internal::CallbackUnaryHandler< ::milvus::grpc::Command, ::milvus::grpc::StringReply>( - [this](::grpc::ServerContext* context, - const ::milvus::grpc::Command* request, - ::milvus::grpc::StringReply* response, - ::grpc::experimental::ServerCallbackRpcController* controller) { - return this->Cmd(context, request, response, controller); - })); - } - void SetMessageAllocatorFor_Cmd( - ::grpc::experimental::MessageAllocator< ::milvus::grpc::Command, ::milvus::grpc::StringReply>* allocator) { - static_cast<::grpc_impl::internal::CallbackUnaryHandler< ::milvus::grpc::Command, ::milvus::grpc::StringReply>*>( - ::grpc::Service::experimental().GetHandler(10)) - ->SetMessageAllocator(allocator); - } - ~ExperimentalWithCallbackMethod_Cmd() override { - BaseClassMustBeDerivedFromService(this); - } - // disable synchronous version of this method - ::grpc::Status Cmd(::grpc::ServerContext* /*context*/, const ::milvus::grpc::Command* /*request*/, ::milvus::grpc::StringReply* /*response*/) override { - abort(); - return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, ""); - } - virtual void Cmd(::grpc::ServerContext* /*context*/, const ::milvus::grpc::Command* /*request*/, ::milvus::grpc::StringReply* /*response*/, ::grpc::experimental::ServerCallbackRpcController* controller) { controller->Finish(::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, "")); } - }; - template - class ExperimentalWithCallbackMethod_DeleteByRange : public BaseClass { - private: - void BaseClassMustBeDerivedFromService(const Service* /*service*/) {} - public: - ExperimentalWithCallbackMethod_DeleteByRange() { - ::grpc::Service::experimental().MarkMethodCallback(11, - new ::grpc_impl::internal::CallbackUnaryHandler< ::milvus::grpc::DeleteByRangeParam, ::milvus::grpc::Status>( - [this](::grpc::ServerContext* context, - const ::milvus::grpc::DeleteByRangeParam* request, - ::milvus::grpc::Status* response, - ::grpc::experimental::ServerCallbackRpcController* controller) { - return this->DeleteByRange(context, request, response, controller); - })); - } - void SetMessageAllocatorFor_DeleteByRange( - ::grpc::experimental::MessageAllocator< ::milvus::grpc::DeleteByRangeParam, ::milvus::grpc::Status>* allocator) { - static_cast<::grpc_impl::internal::CallbackUnaryHandler< ::milvus::grpc::DeleteByRangeParam, ::milvus::grpc::Status>*>( - ::grpc::Service::experimental().GetHandler(11)) - ->SetMessageAllocator(allocator); - } - ~ExperimentalWithCallbackMethod_DeleteByRange() override { - BaseClassMustBeDerivedFromService(this); - } - // disable synchronous version of this method - ::grpc::Status DeleteByRange(::grpc::ServerContext* /*context*/, const ::milvus::grpc::DeleteByRangeParam* /*request*/, ::milvus::grpc::Status* /*response*/) override { - abort(); - return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, ""); - } - virtual void DeleteByRange(::grpc::ServerContext* /*context*/, const ::milvus::grpc::DeleteByRangeParam* /*request*/, ::milvus::grpc::Status* /*response*/, ::grpc::experimental::ServerCallbackRpcController* controller) { controller->Finish(::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, "")); } - }; - template - class ExperimentalWithCallbackMethod_PreloadTable : public BaseClass { - private: - void BaseClassMustBeDerivedFromService(const Service* /*service*/) {} - public: - ExperimentalWithCallbackMethod_PreloadTable() { - ::grpc::Service::experimental().MarkMethodCallback(12, + ExperimentalWithCallbackMethod_DropTable() { + ::grpc::Service::experimental().MarkMethodCallback(5, new ::grpc_impl::internal::CallbackUnaryHandler< ::milvus::grpc::TableName, ::milvus::grpc::Status>( [this](::grpc::ServerContext* context, const ::milvus::grpc::TableName* request, ::milvus::grpc::Status* response, ::grpc::experimental::ServerCallbackRpcController* controller) { - return this->PreloadTable(context, request, response, controller); + return this->DropTable(context, request, response, controller); })); } - void SetMessageAllocatorFor_PreloadTable( + void SetMessageAllocatorFor_DropTable( ::grpc::experimental::MessageAllocator< ::milvus::grpc::TableName, ::milvus::grpc::Status>* allocator) { static_cast<::grpc_impl::internal::CallbackUnaryHandler< ::milvus::grpc::TableName, ::milvus::grpc::Status>*>( - ::grpc::Service::experimental().GetHandler(12)) + ::grpc::Service::experimental().GetHandler(5)) ->SetMessageAllocator(allocator); } - ~ExperimentalWithCallbackMethod_PreloadTable() override { + ~ExperimentalWithCallbackMethod_DropTable() override { BaseClassMustBeDerivedFromService(this); } // disable synchronous version of this method - ::grpc::Status PreloadTable(::grpc::ServerContext* /*context*/, const ::milvus::grpc::TableName* /*request*/, ::milvus::grpc::Status* /*response*/) override { + ::grpc::Status DropTable(::grpc::ServerContext* /*context*/, const ::milvus::grpc::TableName* /*request*/, ::milvus::grpc::Status* /*response*/) override { abort(); return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, ""); } - virtual void PreloadTable(::grpc::ServerContext* /*context*/, const ::milvus::grpc::TableName* /*request*/, ::milvus::grpc::Status* /*response*/, ::grpc::experimental::ServerCallbackRpcController* controller) { controller->Finish(::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, "")); } + virtual void DropTable(::grpc::ServerContext* /*context*/, const ::milvus::grpc::TableName* /*request*/, ::milvus::grpc::Status* /*response*/, ::grpc::experimental::ServerCallbackRpcController* controller) { controller->Finish(::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, "")); } + }; + template + class ExperimentalWithCallbackMethod_CreateIndex : public BaseClass { + private: + void BaseClassMustBeDerivedFromService(const Service* /*service*/) {} + public: + ExperimentalWithCallbackMethod_CreateIndex() { + ::grpc::Service::experimental().MarkMethodCallback(6, + new ::grpc_impl::internal::CallbackUnaryHandler< ::milvus::grpc::IndexParam, ::milvus::grpc::Status>( + [this](::grpc::ServerContext* context, + const ::milvus::grpc::IndexParam* request, + ::milvus::grpc::Status* response, + ::grpc::experimental::ServerCallbackRpcController* controller) { + return this->CreateIndex(context, request, response, controller); + })); + } + void SetMessageAllocatorFor_CreateIndex( + ::grpc::experimental::MessageAllocator< ::milvus::grpc::IndexParam, ::milvus::grpc::Status>* allocator) { + static_cast<::grpc_impl::internal::CallbackUnaryHandler< ::milvus::grpc::IndexParam, ::milvus::grpc::Status>*>( + ::grpc::Service::experimental().GetHandler(6)) + ->SetMessageAllocator(allocator); + } + ~ExperimentalWithCallbackMethod_CreateIndex() override { + BaseClassMustBeDerivedFromService(this); + } + // disable synchronous version of this method + ::grpc::Status CreateIndex(::grpc::ServerContext* /*context*/, const ::milvus::grpc::IndexParam* /*request*/, ::milvus::grpc::Status* /*response*/) override { + abort(); + return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, ""); + } + virtual void CreateIndex(::grpc::ServerContext* /*context*/, const ::milvus::grpc::IndexParam* /*request*/, ::milvus::grpc::Status* /*response*/, ::grpc::experimental::ServerCallbackRpcController* controller) { controller->Finish(::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, "")); } }; template class ExperimentalWithCallbackMethod_DescribeIndex : public BaseClass { @@ -1545,7 +1491,7 @@ class MilvusService final { void BaseClassMustBeDerivedFromService(const Service* /*service*/) {} public: ExperimentalWithCallbackMethod_DescribeIndex() { - ::grpc::Service::experimental().MarkMethodCallback(13, + ::grpc::Service::experimental().MarkMethodCallback(7, new ::grpc_impl::internal::CallbackUnaryHandler< ::milvus::grpc::TableName, ::milvus::grpc::IndexParam>( [this](::grpc::ServerContext* context, const ::milvus::grpc::TableName* request, @@ -1557,7 +1503,7 @@ class MilvusService final { void SetMessageAllocatorFor_DescribeIndex( ::grpc::experimental::MessageAllocator< ::milvus::grpc::TableName, ::milvus::grpc::IndexParam>* allocator) { static_cast<::grpc_impl::internal::CallbackUnaryHandler< ::milvus::grpc::TableName, ::milvus::grpc::IndexParam>*>( - ::grpc::Service::experimental().GetHandler(13)) + ::grpc::Service::experimental().GetHandler(7)) ->SetMessageAllocator(allocator); } ~ExperimentalWithCallbackMethod_DescribeIndex() override { @@ -1576,7 +1522,7 @@ class MilvusService final { void BaseClassMustBeDerivedFromService(const Service* /*service*/) {} public: ExperimentalWithCallbackMethod_DropIndex() { - ::grpc::Service::experimental().MarkMethodCallback(14, + ::grpc::Service::experimental().MarkMethodCallback(8, new ::grpc_impl::internal::CallbackUnaryHandler< ::milvus::grpc::TableName, ::milvus::grpc::Status>( [this](::grpc::ServerContext* context, const ::milvus::grpc::TableName* request, @@ -1588,7 +1534,7 @@ class MilvusService final { void SetMessageAllocatorFor_DropIndex( ::grpc::experimental::MessageAllocator< ::milvus::grpc::TableName, ::milvus::grpc::Status>* allocator) { static_cast<::grpc_impl::internal::CallbackUnaryHandler< ::milvus::grpc::TableName, ::milvus::grpc::Status>*>( - ::grpc::Service::experimental().GetHandler(14)) + ::grpc::Service::experimental().GetHandler(8)) ->SetMessageAllocator(allocator); } ~ExperimentalWithCallbackMethod_DropIndex() override { @@ -1601,7 +1547,286 @@ class MilvusService final { } virtual void DropIndex(::grpc::ServerContext* /*context*/, const ::milvus::grpc::TableName* /*request*/, ::milvus::grpc::Status* /*response*/, ::grpc::experimental::ServerCallbackRpcController* controller) { controller->Finish(::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, "")); } }; - typedef ExperimentalWithCallbackMethod_CreateTable > > > > > > > > > > > > > > ExperimentalCallbackService; + template + class ExperimentalWithCallbackMethod_CreatePartition : public BaseClass { + private: + void BaseClassMustBeDerivedFromService(const Service* /*service*/) {} + public: + ExperimentalWithCallbackMethod_CreatePartition() { + ::grpc::Service::experimental().MarkMethodCallback(9, + new ::grpc_impl::internal::CallbackUnaryHandler< ::milvus::grpc::PartitionParam, ::milvus::grpc::Status>( + [this](::grpc::ServerContext* context, + const ::milvus::grpc::PartitionParam* request, + ::milvus::grpc::Status* response, + ::grpc::experimental::ServerCallbackRpcController* controller) { + return this->CreatePartition(context, request, response, controller); + })); + } + void SetMessageAllocatorFor_CreatePartition( + ::grpc::experimental::MessageAllocator< ::milvus::grpc::PartitionParam, ::milvus::grpc::Status>* allocator) { + static_cast<::grpc_impl::internal::CallbackUnaryHandler< ::milvus::grpc::PartitionParam, ::milvus::grpc::Status>*>( + ::grpc::Service::experimental().GetHandler(9)) + ->SetMessageAllocator(allocator); + } + ~ExperimentalWithCallbackMethod_CreatePartition() override { + BaseClassMustBeDerivedFromService(this); + } + // disable synchronous version of this method + ::grpc::Status CreatePartition(::grpc::ServerContext* /*context*/, const ::milvus::grpc::PartitionParam* /*request*/, ::milvus::grpc::Status* /*response*/) override { + abort(); + return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, ""); + } + virtual void CreatePartition(::grpc::ServerContext* /*context*/, const ::milvus::grpc::PartitionParam* /*request*/, ::milvus::grpc::Status* /*response*/, ::grpc::experimental::ServerCallbackRpcController* controller) { controller->Finish(::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, "")); } + }; + template + class ExperimentalWithCallbackMethod_ShowPartitions : public BaseClass { + private: + void BaseClassMustBeDerivedFromService(const Service* /*service*/) {} + public: + ExperimentalWithCallbackMethod_ShowPartitions() { + ::grpc::Service::experimental().MarkMethodCallback(10, + new ::grpc_impl::internal::CallbackUnaryHandler< ::milvus::grpc::TableName, ::milvus::grpc::PartitionList>( + [this](::grpc::ServerContext* context, + const ::milvus::grpc::TableName* request, + ::milvus::grpc::PartitionList* response, + ::grpc::experimental::ServerCallbackRpcController* controller) { + return this->ShowPartitions(context, request, response, controller); + })); + } + void SetMessageAllocatorFor_ShowPartitions( + ::grpc::experimental::MessageAllocator< ::milvus::grpc::TableName, ::milvus::grpc::PartitionList>* allocator) { + static_cast<::grpc_impl::internal::CallbackUnaryHandler< ::milvus::grpc::TableName, ::milvus::grpc::PartitionList>*>( + ::grpc::Service::experimental().GetHandler(10)) + ->SetMessageAllocator(allocator); + } + ~ExperimentalWithCallbackMethod_ShowPartitions() override { + BaseClassMustBeDerivedFromService(this); + } + // disable synchronous version of this method + ::grpc::Status ShowPartitions(::grpc::ServerContext* /*context*/, const ::milvus::grpc::TableName* /*request*/, ::milvus::grpc::PartitionList* /*response*/) override { + abort(); + return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, ""); + } + virtual void ShowPartitions(::grpc::ServerContext* /*context*/, const ::milvus::grpc::TableName* /*request*/, ::milvus::grpc::PartitionList* /*response*/, ::grpc::experimental::ServerCallbackRpcController* controller) { controller->Finish(::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, "")); } + }; + template + class ExperimentalWithCallbackMethod_DropPartition : public BaseClass { + private: + void BaseClassMustBeDerivedFromService(const Service* /*service*/) {} + public: + ExperimentalWithCallbackMethod_DropPartition() { + ::grpc::Service::experimental().MarkMethodCallback(11, + new ::grpc_impl::internal::CallbackUnaryHandler< ::milvus::grpc::PartitionParam, ::milvus::grpc::Status>( + [this](::grpc::ServerContext* context, + const ::milvus::grpc::PartitionParam* request, + ::milvus::grpc::Status* response, + ::grpc::experimental::ServerCallbackRpcController* controller) { + return this->DropPartition(context, request, response, controller); + })); + } + void SetMessageAllocatorFor_DropPartition( + ::grpc::experimental::MessageAllocator< ::milvus::grpc::PartitionParam, ::milvus::grpc::Status>* allocator) { + static_cast<::grpc_impl::internal::CallbackUnaryHandler< ::milvus::grpc::PartitionParam, ::milvus::grpc::Status>*>( + ::grpc::Service::experimental().GetHandler(11)) + ->SetMessageAllocator(allocator); + } + ~ExperimentalWithCallbackMethod_DropPartition() override { + BaseClassMustBeDerivedFromService(this); + } + // disable synchronous version of this method + ::grpc::Status DropPartition(::grpc::ServerContext* /*context*/, const ::milvus::grpc::PartitionParam* /*request*/, ::milvus::grpc::Status* /*response*/) override { + abort(); + return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, ""); + } + virtual void DropPartition(::grpc::ServerContext* /*context*/, const ::milvus::grpc::PartitionParam* /*request*/, ::milvus::grpc::Status* /*response*/, ::grpc::experimental::ServerCallbackRpcController* controller) { controller->Finish(::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, "")); } + }; + template + class ExperimentalWithCallbackMethod_Insert : public BaseClass { + private: + void BaseClassMustBeDerivedFromService(const Service* /*service*/) {} + public: + ExperimentalWithCallbackMethod_Insert() { + ::grpc::Service::experimental().MarkMethodCallback(12, + new ::grpc_impl::internal::CallbackUnaryHandler< ::milvus::grpc::InsertParam, ::milvus::grpc::VectorIds>( + [this](::grpc::ServerContext* context, + const ::milvus::grpc::InsertParam* request, + ::milvus::grpc::VectorIds* response, + ::grpc::experimental::ServerCallbackRpcController* controller) { + return this->Insert(context, request, response, controller); + })); + } + void SetMessageAllocatorFor_Insert( + ::grpc::experimental::MessageAllocator< ::milvus::grpc::InsertParam, ::milvus::grpc::VectorIds>* allocator) { + static_cast<::grpc_impl::internal::CallbackUnaryHandler< ::milvus::grpc::InsertParam, ::milvus::grpc::VectorIds>*>( + ::grpc::Service::experimental().GetHandler(12)) + ->SetMessageAllocator(allocator); + } + ~ExperimentalWithCallbackMethod_Insert() override { + BaseClassMustBeDerivedFromService(this); + } + // disable synchronous version of this method + ::grpc::Status Insert(::grpc::ServerContext* /*context*/, const ::milvus::grpc::InsertParam* /*request*/, ::milvus::grpc::VectorIds* /*response*/) override { + abort(); + return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, ""); + } + virtual void Insert(::grpc::ServerContext* /*context*/, const ::milvus::grpc::InsertParam* /*request*/, ::milvus::grpc::VectorIds* /*response*/, ::grpc::experimental::ServerCallbackRpcController* controller) { controller->Finish(::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, "")); } + }; + template + class ExperimentalWithCallbackMethod_Search : public BaseClass { + private: + void BaseClassMustBeDerivedFromService(const Service* /*service*/) {} + public: + ExperimentalWithCallbackMethod_Search() { + ::grpc::Service::experimental().MarkMethodCallback(13, + new ::grpc_impl::internal::CallbackUnaryHandler< ::milvus::grpc::SearchParam, ::milvus::grpc::TopKQueryResult>( + [this](::grpc::ServerContext* context, + const ::milvus::grpc::SearchParam* request, + ::milvus::grpc::TopKQueryResult* response, + ::grpc::experimental::ServerCallbackRpcController* controller) { + return this->Search(context, request, response, controller); + })); + } + void SetMessageAllocatorFor_Search( + ::grpc::experimental::MessageAllocator< ::milvus::grpc::SearchParam, ::milvus::grpc::TopKQueryResult>* allocator) { + static_cast<::grpc_impl::internal::CallbackUnaryHandler< ::milvus::grpc::SearchParam, ::milvus::grpc::TopKQueryResult>*>( + ::grpc::Service::experimental().GetHandler(13)) + ->SetMessageAllocator(allocator); + } + ~ExperimentalWithCallbackMethod_Search() override { + BaseClassMustBeDerivedFromService(this); + } + // disable synchronous version of this method + ::grpc::Status Search(::grpc::ServerContext* /*context*/, const ::milvus::grpc::SearchParam* /*request*/, ::milvus::grpc::TopKQueryResult* /*response*/) override { + abort(); + return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, ""); + } + virtual void Search(::grpc::ServerContext* /*context*/, const ::milvus::grpc::SearchParam* /*request*/, ::milvus::grpc::TopKQueryResult* /*response*/, ::grpc::experimental::ServerCallbackRpcController* controller) { controller->Finish(::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, "")); } + }; + template + class ExperimentalWithCallbackMethod_SearchInFiles : public BaseClass { + private: + void BaseClassMustBeDerivedFromService(const Service* /*service*/) {} + public: + ExperimentalWithCallbackMethod_SearchInFiles() { + ::grpc::Service::experimental().MarkMethodCallback(14, + new ::grpc_impl::internal::CallbackUnaryHandler< ::milvus::grpc::SearchInFilesParam, ::milvus::grpc::TopKQueryResult>( + [this](::grpc::ServerContext* context, + const ::milvus::grpc::SearchInFilesParam* request, + ::milvus::grpc::TopKQueryResult* response, + ::grpc::experimental::ServerCallbackRpcController* controller) { + return this->SearchInFiles(context, request, response, controller); + })); + } + void SetMessageAllocatorFor_SearchInFiles( + ::grpc::experimental::MessageAllocator< ::milvus::grpc::SearchInFilesParam, ::milvus::grpc::TopKQueryResult>* allocator) { + static_cast<::grpc_impl::internal::CallbackUnaryHandler< ::milvus::grpc::SearchInFilesParam, ::milvus::grpc::TopKQueryResult>*>( + ::grpc::Service::experimental().GetHandler(14)) + ->SetMessageAllocator(allocator); + } + ~ExperimentalWithCallbackMethod_SearchInFiles() override { + BaseClassMustBeDerivedFromService(this); + } + // disable synchronous version of this method + ::grpc::Status SearchInFiles(::grpc::ServerContext* /*context*/, const ::milvus::grpc::SearchInFilesParam* /*request*/, ::milvus::grpc::TopKQueryResult* /*response*/) override { + abort(); + return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, ""); + } + virtual void SearchInFiles(::grpc::ServerContext* /*context*/, const ::milvus::grpc::SearchInFilesParam* /*request*/, ::milvus::grpc::TopKQueryResult* /*response*/, ::grpc::experimental::ServerCallbackRpcController* controller) { controller->Finish(::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, "")); } + }; + template + class ExperimentalWithCallbackMethod_Cmd : public BaseClass { + private: + void BaseClassMustBeDerivedFromService(const Service* /*service*/) {} + public: + ExperimentalWithCallbackMethod_Cmd() { + ::grpc::Service::experimental().MarkMethodCallback(15, + new ::grpc_impl::internal::CallbackUnaryHandler< ::milvus::grpc::Command, ::milvus::grpc::StringReply>( + [this](::grpc::ServerContext* context, + const ::milvus::grpc::Command* request, + ::milvus::grpc::StringReply* response, + ::grpc::experimental::ServerCallbackRpcController* controller) { + return this->Cmd(context, request, response, controller); + })); + } + void SetMessageAllocatorFor_Cmd( + ::grpc::experimental::MessageAllocator< ::milvus::grpc::Command, ::milvus::grpc::StringReply>* allocator) { + static_cast<::grpc_impl::internal::CallbackUnaryHandler< ::milvus::grpc::Command, ::milvus::grpc::StringReply>*>( + ::grpc::Service::experimental().GetHandler(15)) + ->SetMessageAllocator(allocator); + } + ~ExperimentalWithCallbackMethod_Cmd() override { + BaseClassMustBeDerivedFromService(this); + } + // disable synchronous version of this method + ::grpc::Status Cmd(::grpc::ServerContext* /*context*/, const ::milvus::grpc::Command* /*request*/, ::milvus::grpc::StringReply* /*response*/) override { + abort(); + return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, ""); + } + virtual void Cmd(::grpc::ServerContext* /*context*/, const ::milvus::grpc::Command* /*request*/, ::milvus::grpc::StringReply* /*response*/, ::grpc::experimental::ServerCallbackRpcController* controller) { controller->Finish(::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, "")); } + }; + template + class ExperimentalWithCallbackMethod_DeleteByDate : public BaseClass { + private: + void BaseClassMustBeDerivedFromService(const Service* /*service*/) {} + public: + ExperimentalWithCallbackMethod_DeleteByDate() { + ::grpc::Service::experimental().MarkMethodCallback(16, + new ::grpc_impl::internal::CallbackUnaryHandler< ::milvus::grpc::DeleteByDateParam, ::milvus::grpc::Status>( + [this](::grpc::ServerContext* context, + const ::milvus::grpc::DeleteByDateParam* request, + ::milvus::grpc::Status* response, + ::grpc::experimental::ServerCallbackRpcController* controller) { + return this->DeleteByDate(context, request, response, controller); + })); + } + void SetMessageAllocatorFor_DeleteByDate( + ::grpc::experimental::MessageAllocator< ::milvus::grpc::DeleteByDateParam, ::milvus::grpc::Status>* allocator) { + static_cast<::grpc_impl::internal::CallbackUnaryHandler< ::milvus::grpc::DeleteByDateParam, ::milvus::grpc::Status>*>( + ::grpc::Service::experimental().GetHandler(16)) + ->SetMessageAllocator(allocator); + } + ~ExperimentalWithCallbackMethod_DeleteByDate() override { + BaseClassMustBeDerivedFromService(this); + } + // disable synchronous version of this method + ::grpc::Status DeleteByDate(::grpc::ServerContext* /*context*/, const ::milvus::grpc::DeleteByDateParam* /*request*/, ::milvus::grpc::Status* /*response*/) override { + abort(); + return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, ""); + } + virtual void DeleteByDate(::grpc::ServerContext* /*context*/, const ::milvus::grpc::DeleteByDateParam* /*request*/, ::milvus::grpc::Status* /*response*/, ::grpc::experimental::ServerCallbackRpcController* controller) { controller->Finish(::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, "")); } + }; + template + class ExperimentalWithCallbackMethod_PreloadTable : public BaseClass { + private: + void BaseClassMustBeDerivedFromService(const Service* /*service*/) {} + public: + ExperimentalWithCallbackMethod_PreloadTable() { + ::grpc::Service::experimental().MarkMethodCallback(17, + new ::grpc_impl::internal::CallbackUnaryHandler< ::milvus::grpc::TableName, ::milvus::grpc::Status>( + [this](::grpc::ServerContext* context, + const ::milvus::grpc::TableName* request, + ::milvus::grpc::Status* response, + ::grpc::experimental::ServerCallbackRpcController* controller) { + return this->PreloadTable(context, request, response, controller); + })); + } + void SetMessageAllocatorFor_PreloadTable( + ::grpc::experimental::MessageAllocator< ::milvus::grpc::TableName, ::milvus::grpc::Status>* allocator) { + static_cast<::grpc_impl::internal::CallbackUnaryHandler< ::milvus::grpc::TableName, ::milvus::grpc::Status>*>( + ::grpc::Service::experimental().GetHandler(17)) + ->SetMessageAllocator(allocator); + } + ~ExperimentalWithCallbackMethod_PreloadTable() override { + BaseClassMustBeDerivedFromService(this); + } + // disable synchronous version of this method + ::grpc::Status PreloadTable(::grpc::ServerContext* /*context*/, const ::milvus::grpc::TableName* /*request*/, ::milvus::grpc::Status* /*response*/) override { + abort(); + return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, ""); + } + virtual void PreloadTable(::grpc::ServerContext* /*context*/, const ::milvus::grpc::TableName* /*request*/, ::milvus::grpc::Status* /*response*/, ::grpc::experimental::ServerCallbackRpcController* controller) { controller->Finish(::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, "")); } + }; + typedef ExperimentalWithCallbackMethod_CreateTable > > > > > > > > > > > > > > > > > ExperimentalCallbackService; template class WithGenericMethod_CreateTable : public BaseClass { private: @@ -1637,97 +1862,12 @@ class MilvusService final { } }; template - class WithGenericMethod_DropTable : public BaseClass { - private: - void BaseClassMustBeDerivedFromService(const Service* /*service*/) {} - public: - WithGenericMethod_DropTable() { - ::grpc::Service::MarkMethodGeneric(2); - } - ~WithGenericMethod_DropTable() override { - BaseClassMustBeDerivedFromService(this); - } - // disable synchronous version of this method - ::grpc::Status DropTable(::grpc::ServerContext* /*context*/, const ::milvus::grpc::TableName* /*request*/, ::milvus::grpc::Status* /*response*/) override { - abort(); - return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, ""); - } - }; - template - class WithGenericMethod_CreateIndex : public BaseClass { - private: - void BaseClassMustBeDerivedFromService(const Service* /*service*/) {} - public: - WithGenericMethod_CreateIndex() { - ::grpc::Service::MarkMethodGeneric(3); - } - ~WithGenericMethod_CreateIndex() override { - BaseClassMustBeDerivedFromService(this); - } - // disable synchronous version of this method - ::grpc::Status CreateIndex(::grpc::ServerContext* /*context*/, const ::milvus::grpc::IndexParam* /*request*/, ::milvus::grpc::Status* /*response*/) override { - abort(); - return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, ""); - } - }; - template - class WithGenericMethod_Insert : public BaseClass { - private: - void BaseClassMustBeDerivedFromService(const Service* /*service*/) {} - public: - WithGenericMethod_Insert() { - ::grpc::Service::MarkMethodGeneric(4); - } - ~WithGenericMethod_Insert() override { - BaseClassMustBeDerivedFromService(this); - } - // disable synchronous version of this method - ::grpc::Status Insert(::grpc::ServerContext* /*context*/, const ::milvus::grpc::InsertParam* /*request*/, ::milvus::grpc::VectorIds* /*response*/) override { - abort(); - return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, ""); - } - }; - template - class WithGenericMethod_Search : public BaseClass { - private: - void BaseClassMustBeDerivedFromService(const Service* /*service*/) {} - public: - WithGenericMethod_Search() { - ::grpc::Service::MarkMethodGeneric(5); - } - ~WithGenericMethod_Search() override { - BaseClassMustBeDerivedFromService(this); - } - // disable synchronous version of this method - ::grpc::Status Search(::grpc::ServerContext* /*context*/, const ::milvus::grpc::SearchParam* /*request*/, ::milvus::grpc::TopKQueryResult* /*response*/) override { - abort(); - return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, ""); - } - }; - template - class WithGenericMethod_SearchInFiles : public BaseClass { - private: - void BaseClassMustBeDerivedFromService(const Service* /*service*/) {} - public: - WithGenericMethod_SearchInFiles() { - ::grpc::Service::MarkMethodGeneric(6); - } - ~WithGenericMethod_SearchInFiles() override { - BaseClassMustBeDerivedFromService(this); - } - // disable synchronous version of this method - ::grpc::Status SearchInFiles(::grpc::ServerContext* /*context*/, const ::milvus::grpc::SearchInFilesParam* /*request*/, ::milvus::grpc::TopKQueryResult* /*response*/) override { - abort(); - return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, ""); - } - }; - template class WithGenericMethod_DescribeTable : public BaseClass { private: void BaseClassMustBeDerivedFromService(const Service* /*service*/) {} public: WithGenericMethod_DescribeTable() { - ::grpc::Service::MarkMethodGeneric(7); + ::grpc::Service::MarkMethodGeneric(2); } ~WithGenericMethod_DescribeTable() override { BaseClassMustBeDerivedFromService(this); @@ -1744,7 +1884,7 @@ class MilvusService final { void BaseClassMustBeDerivedFromService(const Service* /*service*/) {} public: WithGenericMethod_CountTable() { - ::grpc::Service::MarkMethodGeneric(8); + ::grpc::Service::MarkMethodGeneric(3); } ~WithGenericMethod_CountTable() override { BaseClassMustBeDerivedFromService(this); @@ -1761,7 +1901,7 @@ class MilvusService final { void BaseClassMustBeDerivedFromService(const Service* /*service*/) {} public: WithGenericMethod_ShowTables() { - ::grpc::Service::MarkMethodGeneric(9); + ::grpc::Service::MarkMethodGeneric(4); } ~WithGenericMethod_ShowTables() override { BaseClassMustBeDerivedFromService(this); @@ -1773,52 +1913,35 @@ class MilvusService final { } }; template - class WithGenericMethod_Cmd : public BaseClass { + class WithGenericMethod_DropTable : public BaseClass { private: void BaseClassMustBeDerivedFromService(const Service* /*service*/) {} public: - WithGenericMethod_Cmd() { - ::grpc::Service::MarkMethodGeneric(10); + WithGenericMethod_DropTable() { + ::grpc::Service::MarkMethodGeneric(5); } - ~WithGenericMethod_Cmd() override { + ~WithGenericMethod_DropTable() override { BaseClassMustBeDerivedFromService(this); } // disable synchronous version of this method - ::grpc::Status Cmd(::grpc::ServerContext* /*context*/, const ::milvus::grpc::Command* /*request*/, ::milvus::grpc::StringReply* /*response*/) override { + ::grpc::Status DropTable(::grpc::ServerContext* /*context*/, const ::milvus::grpc::TableName* /*request*/, ::milvus::grpc::Status* /*response*/) override { abort(); return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, ""); } }; template - class WithGenericMethod_DeleteByRange : public BaseClass { + class WithGenericMethod_CreateIndex : public BaseClass { private: void BaseClassMustBeDerivedFromService(const Service* /*service*/) {} public: - WithGenericMethod_DeleteByRange() { - ::grpc::Service::MarkMethodGeneric(11); + WithGenericMethod_CreateIndex() { + ::grpc::Service::MarkMethodGeneric(6); } - ~WithGenericMethod_DeleteByRange() override { + ~WithGenericMethod_CreateIndex() override { BaseClassMustBeDerivedFromService(this); } // disable synchronous version of this method - ::grpc::Status DeleteByRange(::grpc::ServerContext* /*context*/, const ::milvus::grpc::DeleteByRangeParam* /*request*/, ::milvus::grpc::Status* /*response*/) override { - abort(); - return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, ""); - } - }; - template - class WithGenericMethod_PreloadTable : public BaseClass { - private: - void BaseClassMustBeDerivedFromService(const Service* /*service*/) {} - public: - WithGenericMethod_PreloadTable() { - ::grpc::Service::MarkMethodGeneric(12); - } - ~WithGenericMethod_PreloadTable() override { - BaseClassMustBeDerivedFromService(this); - } - // disable synchronous version of this method - ::grpc::Status PreloadTable(::grpc::ServerContext* /*context*/, const ::milvus::grpc::TableName* /*request*/, ::milvus::grpc::Status* /*response*/) override { + ::grpc::Status CreateIndex(::grpc::ServerContext* /*context*/, const ::milvus::grpc::IndexParam* /*request*/, ::milvus::grpc::Status* /*response*/) override { abort(); return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, ""); } @@ -1829,7 +1952,7 @@ class MilvusService final { void BaseClassMustBeDerivedFromService(const Service* /*service*/) {} public: WithGenericMethod_DescribeIndex() { - ::grpc::Service::MarkMethodGeneric(13); + ::grpc::Service::MarkMethodGeneric(7); } ~WithGenericMethod_DescribeIndex() override { BaseClassMustBeDerivedFromService(this); @@ -1846,7 +1969,7 @@ class MilvusService final { void BaseClassMustBeDerivedFromService(const Service* /*service*/) {} public: WithGenericMethod_DropIndex() { - ::grpc::Service::MarkMethodGeneric(14); + ::grpc::Service::MarkMethodGeneric(8); } ~WithGenericMethod_DropIndex() override { BaseClassMustBeDerivedFromService(this); @@ -1858,6 +1981,159 @@ class MilvusService final { } }; template + class WithGenericMethod_CreatePartition : public BaseClass { + private: + void BaseClassMustBeDerivedFromService(const Service* /*service*/) {} + public: + WithGenericMethod_CreatePartition() { + ::grpc::Service::MarkMethodGeneric(9); + } + ~WithGenericMethod_CreatePartition() override { + BaseClassMustBeDerivedFromService(this); + } + // disable synchronous version of this method + ::grpc::Status CreatePartition(::grpc::ServerContext* /*context*/, const ::milvus::grpc::PartitionParam* /*request*/, ::milvus::grpc::Status* /*response*/) override { + abort(); + return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, ""); + } + }; + template + class WithGenericMethod_ShowPartitions : public BaseClass { + private: + void BaseClassMustBeDerivedFromService(const Service* /*service*/) {} + public: + WithGenericMethod_ShowPartitions() { + ::grpc::Service::MarkMethodGeneric(10); + } + ~WithGenericMethod_ShowPartitions() override { + BaseClassMustBeDerivedFromService(this); + } + // disable synchronous version of this method + ::grpc::Status ShowPartitions(::grpc::ServerContext* /*context*/, const ::milvus::grpc::TableName* /*request*/, ::milvus::grpc::PartitionList* /*response*/) override { + abort(); + return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, ""); + } + }; + template + class WithGenericMethod_DropPartition : public BaseClass { + private: + void BaseClassMustBeDerivedFromService(const Service* /*service*/) {} + public: + WithGenericMethod_DropPartition() { + ::grpc::Service::MarkMethodGeneric(11); + } + ~WithGenericMethod_DropPartition() override { + BaseClassMustBeDerivedFromService(this); + } + // disable synchronous version of this method + ::grpc::Status DropPartition(::grpc::ServerContext* /*context*/, const ::milvus::grpc::PartitionParam* /*request*/, ::milvus::grpc::Status* /*response*/) override { + abort(); + return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, ""); + } + }; + template + class WithGenericMethod_Insert : public BaseClass { + private: + void BaseClassMustBeDerivedFromService(const Service* /*service*/) {} + public: + WithGenericMethod_Insert() { + ::grpc::Service::MarkMethodGeneric(12); + } + ~WithGenericMethod_Insert() override { + BaseClassMustBeDerivedFromService(this); + } + // disable synchronous version of this method + ::grpc::Status Insert(::grpc::ServerContext* /*context*/, const ::milvus::grpc::InsertParam* /*request*/, ::milvus::grpc::VectorIds* /*response*/) override { + abort(); + return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, ""); + } + }; + template + class WithGenericMethod_Search : public BaseClass { + private: + void BaseClassMustBeDerivedFromService(const Service* /*service*/) {} + public: + WithGenericMethod_Search() { + ::grpc::Service::MarkMethodGeneric(13); + } + ~WithGenericMethod_Search() override { + BaseClassMustBeDerivedFromService(this); + } + // disable synchronous version of this method + ::grpc::Status Search(::grpc::ServerContext* /*context*/, const ::milvus::grpc::SearchParam* /*request*/, ::milvus::grpc::TopKQueryResult* /*response*/) override { + abort(); + return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, ""); + } + }; + template + class WithGenericMethod_SearchInFiles : public BaseClass { + private: + void BaseClassMustBeDerivedFromService(const Service* /*service*/) {} + public: + WithGenericMethod_SearchInFiles() { + ::grpc::Service::MarkMethodGeneric(14); + } + ~WithGenericMethod_SearchInFiles() override { + BaseClassMustBeDerivedFromService(this); + } + // disable synchronous version of this method + ::grpc::Status SearchInFiles(::grpc::ServerContext* /*context*/, const ::milvus::grpc::SearchInFilesParam* /*request*/, ::milvus::grpc::TopKQueryResult* /*response*/) override { + abort(); + return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, ""); + } + }; + template + class WithGenericMethod_Cmd : public BaseClass { + private: + void BaseClassMustBeDerivedFromService(const Service* /*service*/) {} + public: + WithGenericMethod_Cmd() { + ::grpc::Service::MarkMethodGeneric(15); + } + ~WithGenericMethod_Cmd() override { + BaseClassMustBeDerivedFromService(this); + } + // disable synchronous version of this method + ::grpc::Status Cmd(::grpc::ServerContext* /*context*/, const ::milvus::grpc::Command* /*request*/, ::milvus::grpc::StringReply* /*response*/) override { + abort(); + return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, ""); + } + }; + template + class WithGenericMethod_DeleteByDate : public BaseClass { + private: + void BaseClassMustBeDerivedFromService(const Service* /*service*/) {} + public: + WithGenericMethod_DeleteByDate() { + ::grpc::Service::MarkMethodGeneric(16); + } + ~WithGenericMethod_DeleteByDate() override { + BaseClassMustBeDerivedFromService(this); + } + // disable synchronous version of this method + ::grpc::Status DeleteByDate(::grpc::ServerContext* /*context*/, const ::milvus::grpc::DeleteByDateParam* /*request*/, ::milvus::grpc::Status* /*response*/) override { + abort(); + return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, ""); + } + }; + template + class WithGenericMethod_PreloadTable : public BaseClass { + private: + void BaseClassMustBeDerivedFromService(const Service* /*service*/) {} + public: + WithGenericMethod_PreloadTable() { + ::grpc::Service::MarkMethodGeneric(17); + } + ~WithGenericMethod_PreloadTable() override { + BaseClassMustBeDerivedFromService(this); + } + // disable synchronous version of this method + ::grpc::Status PreloadTable(::grpc::ServerContext* /*context*/, const ::milvus::grpc::TableName* /*request*/, ::milvus::grpc::Status* /*response*/) override { + abort(); + return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, ""); + } + }; + template class WithRawMethod_CreateTable : public BaseClass { private: void BaseClassMustBeDerivedFromService(const Service* /*service*/) {} @@ -1898,112 +2174,12 @@ class MilvusService final { } }; template - class WithRawMethod_DropTable : public BaseClass { - private: - void BaseClassMustBeDerivedFromService(const Service* /*service*/) {} - public: - WithRawMethod_DropTable() { - ::grpc::Service::MarkMethodRaw(2); - } - ~WithRawMethod_DropTable() override { - BaseClassMustBeDerivedFromService(this); - } - // disable synchronous version of this method - ::grpc::Status DropTable(::grpc::ServerContext* /*context*/, const ::milvus::grpc::TableName* /*request*/, ::milvus::grpc::Status* /*response*/) override { - abort(); - return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, ""); - } - void RequestDropTable(::grpc::ServerContext* context, ::grpc::ByteBuffer* request, ::grpc::ServerAsyncResponseWriter< ::grpc::ByteBuffer>* response, ::grpc::CompletionQueue* new_call_cq, ::grpc::ServerCompletionQueue* notification_cq, void *tag) { - ::grpc::Service::RequestAsyncUnary(2, context, request, response, new_call_cq, notification_cq, tag); - } - }; - template - class WithRawMethod_CreateIndex : public BaseClass { - private: - void BaseClassMustBeDerivedFromService(const Service* /*service*/) {} - public: - WithRawMethod_CreateIndex() { - ::grpc::Service::MarkMethodRaw(3); - } - ~WithRawMethod_CreateIndex() override { - BaseClassMustBeDerivedFromService(this); - } - // disable synchronous version of this method - ::grpc::Status CreateIndex(::grpc::ServerContext* /*context*/, const ::milvus::grpc::IndexParam* /*request*/, ::milvus::grpc::Status* /*response*/) override { - abort(); - return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, ""); - } - void RequestCreateIndex(::grpc::ServerContext* context, ::grpc::ByteBuffer* request, ::grpc::ServerAsyncResponseWriter< ::grpc::ByteBuffer>* response, ::grpc::CompletionQueue* new_call_cq, ::grpc::ServerCompletionQueue* notification_cq, void *tag) { - ::grpc::Service::RequestAsyncUnary(3, context, request, response, new_call_cq, notification_cq, tag); - } - }; - template - class WithRawMethod_Insert : public BaseClass { - private: - void BaseClassMustBeDerivedFromService(const Service* /*service*/) {} - public: - WithRawMethod_Insert() { - ::grpc::Service::MarkMethodRaw(4); - } - ~WithRawMethod_Insert() override { - BaseClassMustBeDerivedFromService(this); - } - // disable synchronous version of this method - ::grpc::Status Insert(::grpc::ServerContext* /*context*/, const ::milvus::grpc::InsertParam* /*request*/, ::milvus::grpc::VectorIds* /*response*/) override { - abort(); - return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, ""); - } - void RequestInsert(::grpc::ServerContext* context, ::grpc::ByteBuffer* request, ::grpc::ServerAsyncResponseWriter< ::grpc::ByteBuffer>* response, ::grpc::CompletionQueue* new_call_cq, ::grpc::ServerCompletionQueue* notification_cq, void *tag) { - ::grpc::Service::RequestAsyncUnary(4, context, request, response, new_call_cq, notification_cq, tag); - } - }; - template - class WithRawMethod_Search : public BaseClass { - private: - void BaseClassMustBeDerivedFromService(const Service* /*service*/) {} - public: - WithRawMethod_Search() { - ::grpc::Service::MarkMethodRaw(5); - } - ~WithRawMethod_Search() override { - BaseClassMustBeDerivedFromService(this); - } - // disable synchronous version of this method - ::grpc::Status Search(::grpc::ServerContext* /*context*/, const ::milvus::grpc::SearchParam* /*request*/, ::milvus::grpc::TopKQueryResult* /*response*/) override { - abort(); - return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, ""); - } - void RequestSearch(::grpc::ServerContext* context, ::grpc::ByteBuffer* request, ::grpc::ServerAsyncResponseWriter< ::grpc::ByteBuffer>* response, ::grpc::CompletionQueue* new_call_cq, ::grpc::ServerCompletionQueue* notification_cq, void *tag) { - ::grpc::Service::RequestAsyncUnary(5, context, request, response, new_call_cq, notification_cq, tag); - } - }; - template - class WithRawMethod_SearchInFiles : public BaseClass { - private: - void BaseClassMustBeDerivedFromService(const Service* /*service*/) {} - public: - WithRawMethod_SearchInFiles() { - ::grpc::Service::MarkMethodRaw(6); - } - ~WithRawMethod_SearchInFiles() override { - BaseClassMustBeDerivedFromService(this); - } - // disable synchronous version of this method - ::grpc::Status SearchInFiles(::grpc::ServerContext* /*context*/, const ::milvus::grpc::SearchInFilesParam* /*request*/, ::milvus::grpc::TopKQueryResult* /*response*/) override { - abort(); - return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, ""); - } - void RequestSearchInFiles(::grpc::ServerContext* context, ::grpc::ByteBuffer* request, ::grpc::ServerAsyncResponseWriter< ::grpc::ByteBuffer>* response, ::grpc::CompletionQueue* new_call_cq, ::grpc::ServerCompletionQueue* notification_cq, void *tag) { - ::grpc::Service::RequestAsyncUnary(6, context, request, response, new_call_cq, notification_cq, tag); - } - }; - template class WithRawMethod_DescribeTable : public BaseClass { private: void BaseClassMustBeDerivedFromService(const Service* /*service*/) {} public: WithRawMethod_DescribeTable() { - ::grpc::Service::MarkMethodRaw(7); + ::grpc::Service::MarkMethodRaw(2); } ~WithRawMethod_DescribeTable() override { BaseClassMustBeDerivedFromService(this); @@ -2014,7 +2190,7 @@ class MilvusService final { return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, ""); } void RequestDescribeTable(::grpc::ServerContext* context, ::grpc::ByteBuffer* request, ::grpc::ServerAsyncResponseWriter< ::grpc::ByteBuffer>* response, ::grpc::CompletionQueue* new_call_cq, ::grpc::ServerCompletionQueue* notification_cq, void *tag) { - ::grpc::Service::RequestAsyncUnary(7, context, request, response, new_call_cq, notification_cq, tag); + ::grpc::Service::RequestAsyncUnary(2, context, request, response, new_call_cq, notification_cq, tag); } }; template @@ -2023,7 +2199,7 @@ class MilvusService final { void BaseClassMustBeDerivedFromService(const Service* /*service*/) {} public: WithRawMethod_CountTable() { - ::grpc::Service::MarkMethodRaw(8); + ::grpc::Service::MarkMethodRaw(3); } ~WithRawMethod_CountTable() override { BaseClassMustBeDerivedFromService(this); @@ -2034,7 +2210,7 @@ class MilvusService final { return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, ""); } void RequestCountTable(::grpc::ServerContext* context, ::grpc::ByteBuffer* request, ::grpc::ServerAsyncResponseWriter< ::grpc::ByteBuffer>* response, ::grpc::CompletionQueue* new_call_cq, ::grpc::ServerCompletionQueue* notification_cq, void *tag) { - ::grpc::Service::RequestAsyncUnary(8, context, request, response, new_call_cq, notification_cq, tag); + ::grpc::Service::RequestAsyncUnary(3, context, request, response, new_call_cq, notification_cq, tag); } }; template @@ -2043,7 +2219,7 @@ class MilvusService final { void BaseClassMustBeDerivedFromService(const Service* /*service*/) {} public: WithRawMethod_ShowTables() { - ::grpc::Service::MarkMethodRaw(9); + ::grpc::Service::MarkMethodRaw(4); } ~WithRawMethod_ShowTables() override { BaseClassMustBeDerivedFromService(this); @@ -2054,67 +2230,47 @@ class MilvusService final { return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, ""); } void RequestShowTables(::grpc::ServerContext* context, ::grpc::ByteBuffer* request, ::grpc::ServerAsyncResponseWriter< ::grpc::ByteBuffer>* response, ::grpc::CompletionQueue* new_call_cq, ::grpc::ServerCompletionQueue* notification_cq, void *tag) { - ::grpc::Service::RequestAsyncUnary(9, context, request, response, new_call_cq, notification_cq, tag); + ::grpc::Service::RequestAsyncUnary(4, context, request, response, new_call_cq, notification_cq, tag); } }; template - class WithRawMethod_Cmd : public BaseClass { + class WithRawMethod_DropTable : public BaseClass { private: void BaseClassMustBeDerivedFromService(const Service* /*service*/) {} public: - WithRawMethod_Cmd() { - ::grpc::Service::MarkMethodRaw(10); + WithRawMethod_DropTable() { + ::grpc::Service::MarkMethodRaw(5); } - ~WithRawMethod_Cmd() override { + ~WithRawMethod_DropTable() override { BaseClassMustBeDerivedFromService(this); } // disable synchronous version of this method - ::grpc::Status Cmd(::grpc::ServerContext* /*context*/, const ::milvus::grpc::Command* /*request*/, ::milvus::grpc::StringReply* /*response*/) override { + ::grpc::Status DropTable(::grpc::ServerContext* /*context*/, const ::milvus::grpc::TableName* /*request*/, ::milvus::grpc::Status* /*response*/) override { abort(); return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, ""); } - void RequestCmd(::grpc::ServerContext* context, ::grpc::ByteBuffer* request, ::grpc::ServerAsyncResponseWriter< ::grpc::ByteBuffer>* response, ::grpc::CompletionQueue* new_call_cq, ::grpc::ServerCompletionQueue* notification_cq, void *tag) { - ::grpc::Service::RequestAsyncUnary(10, context, request, response, new_call_cq, notification_cq, tag); + void RequestDropTable(::grpc::ServerContext* context, ::grpc::ByteBuffer* request, ::grpc::ServerAsyncResponseWriter< ::grpc::ByteBuffer>* response, ::grpc::CompletionQueue* new_call_cq, ::grpc::ServerCompletionQueue* notification_cq, void *tag) { + ::grpc::Service::RequestAsyncUnary(5, context, request, response, new_call_cq, notification_cq, tag); } }; template - class WithRawMethod_DeleteByRange : public BaseClass { + class WithRawMethod_CreateIndex : public BaseClass { private: void BaseClassMustBeDerivedFromService(const Service* /*service*/) {} public: - WithRawMethod_DeleteByRange() { - ::grpc::Service::MarkMethodRaw(11); + WithRawMethod_CreateIndex() { + ::grpc::Service::MarkMethodRaw(6); } - ~WithRawMethod_DeleteByRange() override { + ~WithRawMethod_CreateIndex() override { BaseClassMustBeDerivedFromService(this); } // disable synchronous version of this method - ::grpc::Status DeleteByRange(::grpc::ServerContext* /*context*/, const ::milvus::grpc::DeleteByRangeParam* /*request*/, ::milvus::grpc::Status* /*response*/) override { + ::grpc::Status CreateIndex(::grpc::ServerContext* /*context*/, const ::milvus::grpc::IndexParam* /*request*/, ::milvus::grpc::Status* /*response*/) override { abort(); return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, ""); } - void RequestDeleteByRange(::grpc::ServerContext* context, ::grpc::ByteBuffer* request, ::grpc::ServerAsyncResponseWriter< ::grpc::ByteBuffer>* response, ::grpc::CompletionQueue* new_call_cq, ::grpc::ServerCompletionQueue* notification_cq, void *tag) { - ::grpc::Service::RequestAsyncUnary(11, context, request, response, new_call_cq, notification_cq, tag); - } - }; - template - class WithRawMethod_PreloadTable : public BaseClass { - private: - void BaseClassMustBeDerivedFromService(const Service* /*service*/) {} - public: - WithRawMethod_PreloadTable() { - ::grpc::Service::MarkMethodRaw(12); - } - ~WithRawMethod_PreloadTable() override { - BaseClassMustBeDerivedFromService(this); - } - // disable synchronous version of this method - ::grpc::Status PreloadTable(::grpc::ServerContext* /*context*/, const ::milvus::grpc::TableName* /*request*/, ::milvus::grpc::Status* /*response*/) override { - abort(); - return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, ""); - } - void RequestPreloadTable(::grpc::ServerContext* context, ::grpc::ByteBuffer* request, ::grpc::ServerAsyncResponseWriter< ::grpc::ByteBuffer>* response, ::grpc::CompletionQueue* new_call_cq, ::grpc::ServerCompletionQueue* notification_cq, void *tag) { - ::grpc::Service::RequestAsyncUnary(12, context, request, response, new_call_cq, notification_cq, tag); + void RequestCreateIndex(::grpc::ServerContext* context, ::grpc::ByteBuffer* request, ::grpc::ServerAsyncResponseWriter< ::grpc::ByteBuffer>* response, ::grpc::CompletionQueue* new_call_cq, ::grpc::ServerCompletionQueue* notification_cq, void *tag) { + ::grpc::Service::RequestAsyncUnary(6, context, request, response, new_call_cq, notification_cq, tag); } }; template @@ -2123,7 +2279,7 @@ class MilvusService final { void BaseClassMustBeDerivedFromService(const Service* /*service*/) {} public: WithRawMethod_DescribeIndex() { - ::grpc::Service::MarkMethodRaw(13); + ::grpc::Service::MarkMethodRaw(7); } ~WithRawMethod_DescribeIndex() override { BaseClassMustBeDerivedFromService(this); @@ -2134,7 +2290,7 @@ class MilvusService final { return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, ""); } void RequestDescribeIndex(::grpc::ServerContext* context, ::grpc::ByteBuffer* request, ::grpc::ServerAsyncResponseWriter< ::grpc::ByteBuffer>* response, ::grpc::CompletionQueue* new_call_cq, ::grpc::ServerCompletionQueue* notification_cq, void *tag) { - ::grpc::Service::RequestAsyncUnary(13, context, request, response, new_call_cq, notification_cq, tag); + ::grpc::Service::RequestAsyncUnary(7, context, request, response, new_call_cq, notification_cq, tag); } }; template @@ -2143,7 +2299,7 @@ class MilvusService final { void BaseClassMustBeDerivedFromService(const Service* /*service*/) {} public: WithRawMethod_DropIndex() { - ::grpc::Service::MarkMethodRaw(14); + ::grpc::Service::MarkMethodRaw(8); } ~WithRawMethod_DropIndex() override { BaseClassMustBeDerivedFromService(this); @@ -2154,10 +2310,190 @@ class MilvusService final { return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, ""); } void RequestDropIndex(::grpc::ServerContext* context, ::grpc::ByteBuffer* request, ::grpc::ServerAsyncResponseWriter< ::grpc::ByteBuffer>* response, ::grpc::CompletionQueue* new_call_cq, ::grpc::ServerCompletionQueue* notification_cq, void *tag) { + ::grpc::Service::RequestAsyncUnary(8, context, request, response, new_call_cq, notification_cq, tag); + } + }; + template + class WithRawMethod_CreatePartition : public BaseClass { + private: + void BaseClassMustBeDerivedFromService(const Service* /*service*/) {} + public: + WithRawMethod_CreatePartition() { + ::grpc::Service::MarkMethodRaw(9); + } + ~WithRawMethod_CreatePartition() override { + BaseClassMustBeDerivedFromService(this); + } + // disable synchronous version of this method + ::grpc::Status CreatePartition(::grpc::ServerContext* /*context*/, const ::milvus::grpc::PartitionParam* /*request*/, ::milvus::grpc::Status* /*response*/) override { + abort(); + return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, ""); + } + void RequestCreatePartition(::grpc::ServerContext* context, ::grpc::ByteBuffer* request, ::grpc::ServerAsyncResponseWriter< ::grpc::ByteBuffer>* response, ::grpc::CompletionQueue* new_call_cq, ::grpc::ServerCompletionQueue* notification_cq, void *tag) { + ::grpc::Service::RequestAsyncUnary(9, context, request, response, new_call_cq, notification_cq, tag); + } + }; + template + class WithRawMethod_ShowPartitions : public BaseClass { + private: + void BaseClassMustBeDerivedFromService(const Service* /*service*/) {} + public: + WithRawMethod_ShowPartitions() { + ::grpc::Service::MarkMethodRaw(10); + } + ~WithRawMethod_ShowPartitions() override { + BaseClassMustBeDerivedFromService(this); + } + // disable synchronous version of this method + ::grpc::Status ShowPartitions(::grpc::ServerContext* /*context*/, const ::milvus::grpc::TableName* /*request*/, ::milvus::grpc::PartitionList* /*response*/) override { + abort(); + return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, ""); + } + void RequestShowPartitions(::grpc::ServerContext* context, ::grpc::ByteBuffer* request, ::grpc::ServerAsyncResponseWriter< ::grpc::ByteBuffer>* response, ::grpc::CompletionQueue* new_call_cq, ::grpc::ServerCompletionQueue* notification_cq, void *tag) { + ::grpc::Service::RequestAsyncUnary(10, context, request, response, new_call_cq, notification_cq, tag); + } + }; + template + class WithRawMethod_DropPartition : public BaseClass { + private: + void BaseClassMustBeDerivedFromService(const Service* /*service*/) {} + public: + WithRawMethod_DropPartition() { + ::grpc::Service::MarkMethodRaw(11); + } + ~WithRawMethod_DropPartition() override { + BaseClassMustBeDerivedFromService(this); + } + // disable synchronous version of this method + ::grpc::Status DropPartition(::grpc::ServerContext* /*context*/, const ::milvus::grpc::PartitionParam* /*request*/, ::milvus::grpc::Status* /*response*/) override { + abort(); + return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, ""); + } + void RequestDropPartition(::grpc::ServerContext* context, ::grpc::ByteBuffer* request, ::grpc::ServerAsyncResponseWriter< ::grpc::ByteBuffer>* response, ::grpc::CompletionQueue* new_call_cq, ::grpc::ServerCompletionQueue* notification_cq, void *tag) { + ::grpc::Service::RequestAsyncUnary(11, context, request, response, new_call_cq, notification_cq, tag); + } + }; + template + class WithRawMethod_Insert : public BaseClass { + private: + void BaseClassMustBeDerivedFromService(const Service* /*service*/) {} + public: + WithRawMethod_Insert() { + ::grpc::Service::MarkMethodRaw(12); + } + ~WithRawMethod_Insert() override { + BaseClassMustBeDerivedFromService(this); + } + // disable synchronous version of this method + ::grpc::Status Insert(::grpc::ServerContext* /*context*/, const ::milvus::grpc::InsertParam* /*request*/, ::milvus::grpc::VectorIds* /*response*/) override { + abort(); + return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, ""); + } + void RequestInsert(::grpc::ServerContext* context, ::grpc::ByteBuffer* request, ::grpc::ServerAsyncResponseWriter< ::grpc::ByteBuffer>* response, ::grpc::CompletionQueue* new_call_cq, ::grpc::ServerCompletionQueue* notification_cq, void *tag) { + ::grpc::Service::RequestAsyncUnary(12, context, request, response, new_call_cq, notification_cq, tag); + } + }; + template + class WithRawMethod_Search : public BaseClass { + private: + void BaseClassMustBeDerivedFromService(const Service* /*service*/) {} + public: + WithRawMethod_Search() { + ::grpc::Service::MarkMethodRaw(13); + } + ~WithRawMethod_Search() override { + BaseClassMustBeDerivedFromService(this); + } + // disable synchronous version of this method + ::grpc::Status Search(::grpc::ServerContext* /*context*/, const ::milvus::grpc::SearchParam* /*request*/, ::milvus::grpc::TopKQueryResult* /*response*/) override { + abort(); + return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, ""); + } + void RequestSearch(::grpc::ServerContext* context, ::grpc::ByteBuffer* request, ::grpc::ServerAsyncResponseWriter< ::grpc::ByteBuffer>* response, ::grpc::CompletionQueue* new_call_cq, ::grpc::ServerCompletionQueue* notification_cq, void *tag) { + ::grpc::Service::RequestAsyncUnary(13, context, request, response, new_call_cq, notification_cq, tag); + } + }; + template + class WithRawMethod_SearchInFiles : public BaseClass { + private: + void BaseClassMustBeDerivedFromService(const Service* /*service*/) {} + public: + WithRawMethod_SearchInFiles() { + ::grpc::Service::MarkMethodRaw(14); + } + ~WithRawMethod_SearchInFiles() override { + BaseClassMustBeDerivedFromService(this); + } + // disable synchronous version of this method + ::grpc::Status SearchInFiles(::grpc::ServerContext* /*context*/, const ::milvus::grpc::SearchInFilesParam* /*request*/, ::milvus::grpc::TopKQueryResult* /*response*/) override { + abort(); + return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, ""); + } + void RequestSearchInFiles(::grpc::ServerContext* context, ::grpc::ByteBuffer* request, ::grpc::ServerAsyncResponseWriter< ::grpc::ByteBuffer>* response, ::grpc::CompletionQueue* new_call_cq, ::grpc::ServerCompletionQueue* notification_cq, void *tag) { ::grpc::Service::RequestAsyncUnary(14, context, request, response, new_call_cq, notification_cq, tag); } }; template + class WithRawMethod_Cmd : public BaseClass { + private: + void BaseClassMustBeDerivedFromService(const Service* /*service*/) {} + public: + WithRawMethod_Cmd() { + ::grpc::Service::MarkMethodRaw(15); + } + ~WithRawMethod_Cmd() override { + BaseClassMustBeDerivedFromService(this); + } + // disable synchronous version of this method + ::grpc::Status Cmd(::grpc::ServerContext* /*context*/, const ::milvus::grpc::Command* /*request*/, ::milvus::grpc::StringReply* /*response*/) override { + abort(); + return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, ""); + } + void RequestCmd(::grpc::ServerContext* context, ::grpc::ByteBuffer* request, ::grpc::ServerAsyncResponseWriter< ::grpc::ByteBuffer>* response, ::grpc::CompletionQueue* new_call_cq, ::grpc::ServerCompletionQueue* notification_cq, void *tag) { + ::grpc::Service::RequestAsyncUnary(15, context, request, response, new_call_cq, notification_cq, tag); + } + }; + template + class WithRawMethod_DeleteByDate : public BaseClass { + private: + void BaseClassMustBeDerivedFromService(const Service* /*service*/) {} + public: + WithRawMethod_DeleteByDate() { + ::grpc::Service::MarkMethodRaw(16); + } + ~WithRawMethod_DeleteByDate() override { + BaseClassMustBeDerivedFromService(this); + } + // disable synchronous version of this method + ::grpc::Status DeleteByDate(::grpc::ServerContext* /*context*/, const ::milvus::grpc::DeleteByDateParam* /*request*/, ::milvus::grpc::Status* /*response*/) override { + abort(); + return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, ""); + } + void RequestDeleteByDate(::grpc::ServerContext* context, ::grpc::ByteBuffer* request, ::grpc::ServerAsyncResponseWriter< ::grpc::ByteBuffer>* response, ::grpc::CompletionQueue* new_call_cq, ::grpc::ServerCompletionQueue* notification_cq, void *tag) { + ::grpc::Service::RequestAsyncUnary(16, context, request, response, new_call_cq, notification_cq, tag); + } + }; + template + class WithRawMethod_PreloadTable : public BaseClass { + private: + void BaseClassMustBeDerivedFromService(const Service* /*service*/) {} + public: + WithRawMethod_PreloadTable() { + ::grpc::Service::MarkMethodRaw(17); + } + ~WithRawMethod_PreloadTable() override { + BaseClassMustBeDerivedFromService(this); + } + // disable synchronous version of this method + ::grpc::Status PreloadTable(::grpc::ServerContext* /*context*/, const ::milvus::grpc::TableName* /*request*/, ::milvus::grpc::Status* /*response*/) override { + abort(); + return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, ""); + } + void RequestPreloadTable(::grpc::ServerContext* context, ::grpc::ByteBuffer* request, ::grpc::ServerAsyncResponseWriter< ::grpc::ByteBuffer>* response, ::grpc::CompletionQueue* new_call_cq, ::grpc::ServerCompletionQueue* notification_cq, void *tag) { + ::grpc::Service::RequestAsyncUnary(17, context, request, response, new_call_cq, notification_cq, tag); + } + }; + template class ExperimentalWithRawCallbackMethod_CreateTable : public BaseClass { private: void BaseClassMustBeDerivedFromService(const Service* /*service*/) {} @@ -2208,137 +2544,12 @@ class MilvusService final { virtual void HasTable(::grpc::ServerContext* /*context*/, const ::grpc::ByteBuffer* /*request*/, ::grpc::ByteBuffer* /*response*/, ::grpc::experimental::ServerCallbackRpcController* controller) { controller->Finish(::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, "")); } }; template - class ExperimentalWithRawCallbackMethod_DropTable : public BaseClass { - private: - void BaseClassMustBeDerivedFromService(const Service* /*service*/) {} - public: - ExperimentalWithRawCallbackMethod_DropTable() { - ::grpc::Service::experimental().MarkMethodRawCallback(2, - new ::grpc_impl::internal::CallbackUnaryHandler< ::grpc::ByteBuffer, ::grpc::ByteBuffer>( - [this](::grpc::ServerContext* context, - const ::grpc::ByteBuffer* request, - ::grpc::ByteBuffer* response, - ::grpc::experimental::ServerCallbackRpcController* controller) { - this->DropTable(context, request, response, controller); - })); - } - ~ExperimentalWithRawCallbackMethod_DropTable() override { - BaseClassMustBeDerivedFromService(this); - } - // disable synchronous version of this method - ::grpc::Status DropTable(::grpc::ServerContext* /*context*/, const ::milvus::grpc::TableName* /*request*/, ::milvus::grpc::Status* /*response*/) override { - abort(); - return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, ""); - } - virtual void DropTable(::grpc::ServerContext* /*context*/, const ::grpc::ByteBuffer* /*request*/, ::grpc::ByteBuffer* /*response*/, ::grpc::experimental::ServerCallbackRpcController* controller) { controller->Finish(::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, "")); } - }; - template - class ExperimentalWithRawCallbackMethod_CreateIndex : public BaseClass { - private: - void BaseClassMustBeDerivedFromService(const Service* /*service*/) {} - public: - ExperimentalWithRawCallbackMethod_CreateIndex() { - ::grpc::Service::experimental().MarkMethodRawCallback(3, - new ::grpc_impl::internal::CallbackUnaryHandler< ::grpc::ByteBuffer, ::grpc::ByteBuffer>( - [this](::grpc::ServerContext* context, - const ::grpc::ByteBuffer* request, - ::grpc::ByteBuffer* response, - ::grpc::experimental::ServerCallbackRpcController* controller) { - this->CreateIndex(context, request, response, controller); - })); - } - ~ExperimentalWithRawCallbackMethod_CreateIndex() override { - BaseClassMustBeDerivedFromService(this); - } - // disable synchronous version of this method - ::grpc::Status CreateIndex(::grpc::ServerContext* /*context*/, const ::milvus::grpc::IndexParam* /*request*/, ::milvus::grpc::Status* /*response*/) override { - abort(); - return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, ""); - } - virtual void CreateIndex(::grpc::ServerContext* /*context*/, const ::grpc::ByteBuffer* /*request*/, ::grpc::ByteBuffer* /*response*/, ::grpc::experimental::ServerCallbackRpcController* controller) { controller->Finish(::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, "")); } - }; - template - class ExperimentalWithRawCallbackMethod_Insert : public BaseClass { - private: - void BaseClassMustBeDerivedFromService(const Service* /*service*/) {} - public: - ExperimentalWithRawCallbackMethod_Insert() { - ::grpc::Service::experimental().MarkMethodRawCallback(4, - new ::grpc_impl::internal::CallbackUnaryHandler< ::grpc::ByteBuffer, ::grpc::ByteBuffer>( - [this](::grpc::ServerContext* context, - const ::grpc::ByteBuffer* request, - ::grpc::ByteBuffer* response, - ::grpc::experimental::ServerCallbackRpcController* controller) { - this->Insert(context, request, response, controller); - })); - } - ~ExperimentalWithRawCallbackMethod_Insert() override { - BaseClassMustBeDerivedFromService(this); - } - // disable synchronous version of this method - ::grpc::Status Insert(::grpc::ServerContext* /*context*/, const ::milvus::grpc::InsertParam* /*request*/, ::milvus::grpc::VectorIds* /*response*/) override { - abort(); - return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, ""); - } - virtual void Insert(::grpc::ServerContext* /*context*/, const ::grpc::ByteBuffer* /*request*/, ::grpc::ByteBuffer* /*response*/, ::grpc::experimental::ServerCallbackRpcController* controller) { controller->Finish(::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, "")); } - }; - template - class ExperimentalWithRawCallbackMethod_Search : public BaseClass { - private: - void BaseClassMustBeDerivedFromService(const Service* /*service*/) {} - public: - ExperimentalWithRawCallbackMethod_Search() { - ::grpc::Service::experimental().MarkMethodRawCallback(5, - new ::grpc_impl::internal::CallbackUnaryHandler< ::grpc::ByteBuffer, ::grpc::ByteBuffer>( - [this](::grpc::ServerContext* context, - const ::grpc::ByteBuffer* request, - ::grpc::ByteBuffer* response, - ::grpc::experimental::ServerCallbackRpcController* controller) { - this->Search(context, request, response, controller); - })); - } - ~ExperimentalWithRawCallbackMethod_Search() override { - BaseClassMustBeDerivedFromService(this); - } - // disable synchronous version of this method - ::grpc::Status Search(::grpc::ServerContext* /*context*/, const ::milvus::grpc::SearchParam* /*request*/, ::milvus::grpc::TopKQueryResult* /*response*/) override { - abort(); - return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, ""); - } - virtual void Search(::grpc::ServerContext* /*context*/, const ::grpc::ByteBuffer* /*request*/, ::grpc::ByteBuffer* /*response*/, ::grpc::experimental::ServerCallbackRpcController* controller) { controller->Finish(::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, "")); } - }; - template - class ExperimentalWithRawCallbackMethod_SearchInFiles : public BaseClass { - private: - void BaseClassMustBeDerivedFromService(const Service* /*service*/) {} - public: - ExperimentalWithRawCallbackMethod_SearchInFiles() { - ::grpc::Service::experimental().MarkMethodRawCallback(6, - new ::grpc_impl::internal::CallbackUnaryHandler< ::grpc::ByteBuffer, ::grpc::ByteBuffer>( - [this](::grpc::ServerContext* context, - const ::grpc::ByteBuffer* request, - ::grpc::ByteBuffer* response, - ::grpc::experimental::ServerCallbackRpcController* controller) { - this->SearchInFiles(context, request, response, controller); - })); - } - ~ExperimentalWithRawCallbackMethod_SearchInFiles() override { - BaseClassMustBeDerivedFromService(this); - } - // disable synchronous version of this method - ::grpc::Status SearchInFiles(::grpc::ServerContext* /*context*/, const ::milvus::grpc::SearchInFilesParam* /*request*/, ::milvus::grpc::TopKQueryResult* /*response*/) override { - abort(); - return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, ""); - } - virtual void SearchInFiles(::grpc::ServerContext* /*context*/, const ::grpc::ByteBuffer* /*request*/, ::grpc::ByteBuffer* /*response*/, ::grpc::experimental::ServerCallbackRpcController* controller) { controller->Finish(::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, "")); } - }; - template class ExperimentalWithRawCallbackMethod_DescribeTable : public BaseClass { private: void BaseClassMustBeDerivedFromService(const Service* /*service*/) {} public: ExperimentalWithRawCallbackMethod_DescribeTable() { - ::grpc::Service::experimental().MarkMethodRawCallback(7, + ::grpc::Service::experimental().MarkMethodRawCallback(2, new ::grpc_impl::internal::CallbackUnaryHandler< ::grpc::ByteBuffer, ::grpc::ByteBuffer>( [this](::grpc::ServerContext* context, const ::grpc::ByteBuffer* request, @@ -2363,7 +2574,7 @@ class MilvusService final { void BaseClassMustBeDerivedFromService(const Service* /*service*/) {} public: ExperimentalWithRawCallbackMethod_CountTable() { - ::grpc::Service::experimental().MarkMethodRawCallback(8, + ::grpc::Service::experimental().MarkMethodRawCallback(3, new ::grpc_impl::internal::CallbackUnaryHandler< ::grpc::ByteBuffer, ::grpc::ByteBuffer>( [this](::grpc::ServerContext* context, const ::grpc::ByteBuffer* request, @@ -2388,7 +2599,7 @@ class MilvusService final { void BaseClassMustBeDerivedFromService(const Service* /*service*/) {} public: ExperimentalWithRawCallbackMethod_ShowTables() { - ::grpc::Service::experimental().MarkMethodRawCallback(9, + ::grpc::Service::experimental().MarkMethodRawCallback(4, new ::grpc_impl::internal::CallbackUnaryHandler< ::grpc::ByteBuffer, ::grpc::ByteBuffer>( [this](::grpc::ServerContext* context, const ::grpc::ByteBuffer* request, @@ -2408,79 +2619,54 @@ class MilvusService final { virtual void ShowTables(::grpc::ServerContext* /*context*/, const ::grpc::ByteBuffer* /*request*/, ::grpc::ByteBuffer* /*response*/, ::grpc::experimental::ServerCallbackRpcController* controller) { controller->Finish(::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, "")); } }; template - class ExperimentalWithRawCallbackMethod_Cmd : public BaseClass { + class ExperimentalWithRawCallbackMethod_DropTable : public BaseClass { private: void BaseClassMustBeDerivedFromService(const Service* /*service*/) {} public: - ExperimentalWithRawCallbackMethod_Cmd() { - ::grpc::Service::experimental().MarkMethodRawCallback(10, + ExperimentalWithRawCallbackMethod_DropTable() { + ::grpc::Service::experimental().MarkMethodRawCallback(5, new ::grpc_impl::internal::CallbackUnaryHandler< ::grpc::ByteBuffer, ::grpc::ByteBuffer>( [this](::grpc::ServerContext* context, const ::grpc::ByteBuffer* request, ::grpc::ByteBuffer* response, ::grpc::experimental::ServerCallbackRpcController* controller) { - this->Cmd(context, request, response, controller); + this->DropTable(context, request, response, controller); })); } - ~ExperimentalWithRawCallbackMethod_Cmd() override { + ~ExperimentalWithRawCallbackMethod_DropTable() override { BaseClassMustBeDerivedFromService(this); } // disable synchronous version of this method - ::grpc::Status Cmd(::grpc::ServerContext* /*context*/, const ::milvus::grpc::Command* /*request*/, ::milvus::grpc::StringReply* /*response*/) override { + ::grpc::Status DropTable(::grpc::ServerContext* /*context*/, const ::milvus::grpc::TableName* /*request*/, ::milvus::grpc::Status* /*response*/) override { abort(); return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, ""); } - virtual void Cmd(::grpc::ServerContext* /*context*/, const ::grpc::ByteBuffer* /*request*/, ::grpc::ByteBuffer* /*response*/, ::grpc::experimental::ServerCallbackRpcController* controller) { controller->Finish(::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, "")); } + virtual void DropTable(::grpc::ServerContext* /*context*/, const ::grpc::ByteBuffer* /*request*/, ::grpc::ByteBuffer* /*response*/, ::grpc::experimental::ServerCallbackRpcController* controller) { controller->Finish(::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, "")); } }; template - class ExperimentalWithRawCallbackMethod_DeleteByRange : public BaseClass { + class ExperimentalWithRawCallbackMethod_CreateIndex : public BaseClass { private: void BaseClassMustBeDerivedFromService(const Service* /*service*/) {} public: - ExperimentalWithRawCallbackMethod_DeleteByRange() { - ::grpc::Service::experimental().MarkMethodRawCallback(11, + ExperimentalWithRawCallbackMethod_CreateIndex() { + ::grpc::Service::experimental().MarkMethodRawCallback(6, new ::grpc_impl::internal::CallbackUnaryHandler< ::grpc::ByteBuffer, ::grpc::ByteBuffer>( [this](::grpc::ServerContext* context, const ::grpc::ByteBuffer* request, ::grpc::ByteBuffer* response, ::grpc::experimental::ServerCallbackRpcController* controller) { - this->DeleteByRange(context, request, response, controller); + this->CreateIndex(context, request, response, controller); })); } - ~ExperimentalWithRawCallbackMethod_DeleteByRange() override { + ~ExperimentalWithRawCallbackMethod_CreateIndex() override { BaseClassMustBeDerivedFromService(this); } // disable synchronous version of this method - ::grpc::Status DeleteByRange(::grpc::ServerContext* /*context*/, const ::milvus::grpc::DeleteByRangeParam* /*request*/, ::milvus::grpc::Status* /*response*/) override { + ::grpc::Status CreateIndex(::grpc::ServerContext* /*context*/, const ::milvus::grpc::IndexParam* /*request*/, ::milvus::grpc::Status* /*response*/) override { abort(); return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, ""); } - virtual void DeleteByRange(::grpc::ServerContext* /*context*/, const ::grpc::ByteBuffer* /*request*/, ::grpc::ByteBuffer* /*response*/, ::grpc::experimental::ServerCallbackRpcController* controller) { controller->Finish(::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, "")); } - }; - template - class ExperimentalWithRawCallbackMethod_PreloadTable : public BaseClass { - private: - void BaseClassMustBeDerivedFromService(const Service* /*service*/) {} - public: - ExperimentalWithRawCallbackMethod_PreloadTable() { - ::grpc::Service::experimental().MarkMethodRawCallback(12, - new ::grpc_impl::internal::CallbackUnaryHandler< ::grpc::ByteBuffer, ::grpc::ByteBuffer>( - [this](::grpc::ServerContext* context, - const ::grpc::ByteBuffer* request, - ::grpc::ByteBuffer* response, - ::grpc::experimental::ServerCallbackRpcController* controller) { - this->PreloadTable(context, request, response, controller); - })); - } - ~ExperimentalWithRawCallbackMethod_PreloadTable() override { - BaseClassMustBeDerivedFromService(this); - } - // disable synchronous version of this method - ::grpc::Status PreloadTable(::grpc::ServerContext* /*context*/, const ::milvus::grpc::TableName* /*request*/, ::milvus::grpc::Status* /*response*/) override { - abort(); - return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, ""); - } - virtual void PreloadTable(::grpc::ServerContext* /*context*/, const ::grpc::ByteBuffer* /*request*/, ::grpc::ByteBuffer* /*response*/, ::grpc::experimental::ServerCallbackRpcController* controller) { controller->Finish(::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, "")); } + virtual void CreateIndex(::grpc::ServerContext* /*context*/, const ::grpc::ByteBuffer* /*request*/, ::grpc::ByteBuffer* /*response*/, ::grpc::experimental::ServerCallbackRpcController* controller) { controller->Finish(::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, "")); } }; template class ExperimentalWithRawCallbackMethod_DescribeIndex : public BaseClass { @@ -2488,7 +2674,7 @@ class MilvusService final { void BaseClassMustBeDerivedFromService(const Service* /*service*/) {} public: ExperimentalWithRawCallbackMethod_DescribeIndex() { - ::grpc::Service::experimental().MarkMethodRawCallback(13, + ::grpc::Service::experimental().MarkMethodRawCallback(7, new ::grpc_impl::internal::CallbackUnaryHandler< ::grpc::ByteBuffer, ::grpc::ByteBuffer>( [this](::grpc::ServerContext* context, const ::grpc::ByteBuffer* request, @@ -2513,7 +2699,7 @@ class MilvusService final { void BaseClassMustBeDerivedFromService(const Service* /*service*/) {} public: ExperimentalWithRawCallbackMethod_DropIndex() { - ::grpc::Service::experimental().MarkMethodRawCallback(14, + ::grpc::Service::experimental().MarkMethodRawCallback(8, new ::grpc_impl::internal::CallbackUnaryHandler< ::grpc::ByteBuffer, ::grpc::ByteBuffer>( [this](::grpc::ServerContext* context, const ::grpc::ByteBuffer* request, @@ -2533,6 +2719,231 @@ class MilvusService final { virtual void DropIndex(::grpc::ServerContext* /*context*/, const ::grpc::ByteBuffer* /*request*/, ::grpc::ByteBuffer* /*response*/, ::grpc::experimental::ServerCallbackRpcController* controller) { controller->Finish(::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, "")); } }; template + class ExperimentalWithRawCallbackMethod_CreatePartition : public BaseClass { + private: + void BaseClassMustBeDerivedFromService(const Service* /*service*/) {} + public: + ExperimentalWithRawCallbackMethod_CreatePartition() { + ::grpc::Service::experimental().MarkMethodRawCallback(9, + new ::grpc_impl::internal::CallbackUnaryHandler< ::grpc::ByteBuffer, ::grpc::ByteBuffer>( + [this](::grpc::ServerContext* context, + const ::grpc::ByteBuffer* request, + ::grpc::ByteBuffer* response, + ::grpc::experimental::ServerCallbackRpcController* controller) { + this->CreatePartition(context, request, response, controller); + })); + } + ~ExperimentalWithRawCallbackMethod_CreatePartition() override { + BaseClassMustBeDerivedFromService(this); + } + // disable synchronous version of this method + ::grpc::Status CreatePartition(::grpc::ServerContext* /*context*/, const ::milvus::grpc::PartitionParam* /*request*/, ::milvus::grpc::Status* /*response*/) override { + abort(); + return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, ""); + } + virtual void CreatePartition(::grpc::ServerContext* /*context*/, const ::grpc::ByteBuffer* /*request*/, ::grpc::ByteBuffer* /*response*/, ::grpc::experimental::ServerCallbackRpcController* controller) { controller->Finish(::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, "")); } + }; + template + class ExperimentalWithRawCallbackMethod_ShowPartitions : public BaseClass { + private: + void BaseClassMustBeDerivedFromService(const Service* /*service*/) {} + public: + ExperimentalWithRawCallbackMethod_ShowPartitions() { + ::grpc::Service::experimental().MarkMethodRawCallback(10, + new ::grpc_impl::internal::CallbackUnaryHandler< ::grpc::ByteBuffer, ::grpc::ByteBuffer>( + [this](::grpc::ServerContext* context, + const ::grpc::ByteBuffer* request, + ::grpc::ByteBuffer* response, + ::grpc::experimental::ServerCallbackRpcController* controller) { + this->ShowPartitions(context, request, response, controller); + })); + } + ~ExperimentalWithRawCallbackMethod_ShowPartitions() override { + BaseClassMustBeDerivedFromService(this); + } + // disable synchronous version of this method + ::grpc::Status ShowPartitions(::grpc::ServerContext* /*context*/, const ::milvus::grpc::TableName* /*request*/, ::milvus::grpc::PartitionList* /*response*/) override { + abort(); + return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, ""); + } + virtual void ShowPartitions(::grpc::ServerContext* /*context*/, const ::grpc::ByteBuffer* /*request*/, ::grpc::ByteBuffer* /*response*/, ::grpc::experimental::ServerCallbackRpcController* controller) { controller->Finish(::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, "")); } + }; + template + class ExperimentalWithRawCallbackMethod_DropPartition : public BaseClass { + private: + void BaseClassMustBeDerivedFromService(const Service* /*service*/) {} + public: + ExperimentalWithRawCallbackMethod_DropPartition() { + ::grpc::Service::experimental().MarkMethodRawCallback(11, + new ::grpc_impl::internal::CallbackUnaryHandler< ::grpc::ByteBuffer, ::grpc::ByteBuffer>( + [this](::grpc::ServerContext* context, + const ::grpc::ByteBuffer* request, + ::grpc::ByteBuffer* response, + ::grpc::experimental::ServerCallbackRpcController* controller) { + this->DropPartition(context, request, response, controller); + })); + } + ~ExperimentalWithRawCallbackMethod_DropPartition() override { + BaseClassMustBeDerivedFromService(this); + } + // disable synchronous version of this method + ::grpc::Status DropPartition(::grpc::ServerContext* /*context*/, const ::milvus::grpc::PartitionParam* /*request*/, ::milvus::grpc::Status* /*response*/) override { + abort(); + return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, ""); + } + virtual void DropPartition(::grpc::ServerContext* /*context*/, const ::grpc::ByteBuffer* /*request*/, ::grpc::ByteBuffer* /*response*/, ::grpc::experimental::ServerCallbackRpcController* controller) { controller->Finish(::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, "")); } + }; + template + class ExperimentalWithRawCallbackMethod_Insert : public BaseClass { + private: + void BaseClassMustBeDerivedFromService(const Service* /*service*/) {} + public: + ExperimentalWithRawCallbackMethod_Insert() { + ::grpc::Service::experimental().MarkMethodRawCallback(12, + new ::grpc_impl::internal::CallbackUnaryHandler< ::grpc::ByteBuffer, ::grpc::ByteBuffer>( + [this](::grpc::ServerContext* context, + const ::grpc::ByteBuffer* request, + ::grpc::ByteBuffer* response, + ::grpc::experimental::ServerCallbackRpcController* controller) { + this->Insert(context, request, response, controller); + })); + } + ~ExperimentalWithRawCallbackMethod_Insert() override { + BaseClassMustBeDerivedFromService(this); + } + // disable synchronous version of this method + ::grpc::Status Insert(::grpc::ServerContext* /*context*/, const ::milvus::grpc::InsertParam* /*request*/, ::milvus::grpc::VectorIds* /*response*/) override { + abort(); + return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, ""); + } + virtual void Insert(::grpc::ServerContext* /*context*/, const ::grpc::ByteBuffer* /*request*/, ::grpc::ByteBuffer* /*response*/, ::grpc::experimental::ServerCallbackRpcController* controller) { controller->Finish(::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, "")); } + }; + template + class ExperimentalWithRawCallbackMethod_Search : public BaseClass { + private: + void BaseClassMustBeDerivedFromService(const Service* /*service*/) {} + public: + ExperimentalWithRawCallbackMethod_Search() { + ::grpc::Service::experimental().MarkMethodRawCallback(13, + new ::grpc_impl::internal::CallbackUnaryHandler< ::grpc::ByteBuffer, ::grpc::ByteBuffer>( + [this](::grpc::ServerContext* context, + const ::grpc::ByteBuffer* request, + ::grpc::ByteBuffer* response, + ::grpc::experimental::ServerCallbackRpcController* controller) { + this->Search(context, request, response, controller); + })); + } + ~ExperimentalWithRawCallbackMethod_Search() override { + BaseClassMustBeDerivedFromService(this); + } + // disable synchronous version of this method + ::grpc::Status Search(::grpc::ServerContext* /*context*/, const ::milvus::grpc::SearchParam* /*request*/, ::milvus::grpc::TopKQueryResult* /*response*/) override { + abort(); + return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, ""); + } + virtual void Search(::grpc::ServerContext* /*context*/, const ::grpc::ByteBuffer* /*request*/, ::grpc::ByteBuffer* /*response*/, ::grpc::experimental::ServerCallbackRpcController* controller) { controller->Finish(::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, "")); } + }; + template + class ExperimentalWithRawCallbackMethod_SearchInFiles : public BaseClass { + private: + void BaseClassMustBeDerivedFromService(const Service* /*service*/) {} + public: + ExperimentalWithRawCallbackMethod_SearchInFiles() { + ::grpc::Service::experimental().MarkMethodRawCallback(14, + new ::grpc_impl::internal::CallbackUnaryHandler< ::grpc::ByteBuffer, ::grpc::ByteBuffer>( + [this](::grpc::ServerContext* context, + const ::grpc::ByteBuffer* request, + ::grpc::ByteBuffer* response, + ::grpc::experimental::ServerCallbackRpcController* controller) { + this->SearchInFiles(context, request, response, controller); + })); + } + ~ExperimentalWithRawCallbackMethod_SearchInFiles() override { + BaseClassMustBeDerivedFromService(this); + } + // disable synchronous version of this method + ::grpc::Status SearchInFiles(::grpc::ServerContext* /*context*/, const ::milvus::grpc::SearchInFilesParam* /*request*/, ::milvus::grpc::TopKQueryResult* /*response*/) override { + abort(); + return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, ""); + } + virtual void SearchInFiles(::grpc::ServerContext* /*context*/, const ::grpc::ByteBuffer* /*request*/, ::grpc::ByteBuffer* /*response*/, ::grpc::experimental::ServerCallbackRpcController* controller) { controller->Finish(::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, "")); } + }; + template + class ExperimentalWithRawCallbackMethod_Cmd : public BaseClass { + private: + void BaseClassMustBeDerivedFromService(const Service* /*service*/) {} + public: + ExperimentalWithRawCallbackMethod_Cmd() { + ::grpc::Service::experimental().MarkMethodRawCallback(15, + new ::grpc_impl::internal::CallbackUnaryHandler< ::grpc::ByteBuffer, ::grpc::ByteBuffer>( + [this](::grpc::ServerContext* context, + const ::grpc::ByteBuffer* request, + ::grpc::ByteBuffer* response, + ::grpc::experimental::ServerCallbackRpcController* controller) { + this->Cmd(context, request, response, controller); + })); + } + ~ExperimentalWithRawCallbackMethod_Cmd() override { + BaseClassMustBeDerivedFromService(this); + } + // disable synchronous version of this method + ::grpc::Status Cmd(::grpc::ServerContext* /*context*/, const ::milvus::grpc::Command* /*request*/, ::milvus::grpc::StringReply* /*response*/) override { + abort(); + return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, ""); + } + virtual void Cmd(::grpc::ServerContext* /*context*/, const ::grpc::ByteBuffer* /*request*/, ::grpc::ByteBuffer* /*response*/, ::grpc::experimental::ServerCallbackRpcController* controller) { controller->Finish(::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, "")); } + }; + template + class ExperimentalWithRawCallbackMethod_DeleteByDate : public BaseClass { + private: + void BaseClassMustBeDerivedFromService(const Service* /*service*/) {} + public: + ExperimentalWithRawCallbackMethod_DeleteByDate() { + ::grpc::Service::experimental().MarkMethodRawCallback(16, + new ::grpc_impl::internal::CallbackUnaryHandler< ::grpc::ByteBuffer, ::grpc::ByteBuffer>( + [this](::grpc::ServerContext* context, + const ::grpc::ByteBuffer* request, + ::grpc::ByteBuffer* response, + ::grpc::experimental::ServerCallbackRpcController* controller) { + this->DeleteByDate(context, request, response, controller); + })); + } + ~ExperimentalWithRawCallbackMethod_DeleteByDate() override { + BaseClassMustBeDerivedFromService(this); + } + // disable synchronous version of this method + ::grpc::Status DeleteByDate(::grpc::ServerContext* /*context*/, const ::milvus::grpc::DeleteByDateParam* /*request*/, ::milvus::grpc::Status* /*response*/) override { + abort(); + return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, ""); + } + virtual void DeleteByDate(::grpc::ServerContext* /*context*/, const ::grpc::ByteBuffer* /*request*/, ::grpc::ByteBuffer* /*response*/, ::grpc::experimental::ServerCallbackRpcController* controller) { controller->Finish(::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, "")); } + }; + template + class ExperimentalWithRawCallbackMethod_PreloadTable : public BaseClass { + private: + void BaseClassMustBeDerivedFromService(const Service* /*service*/) {} + public: + ExperimentalWithRawCallbackMethod_PreloadTable() { + ::grpc::Service::experimental().MarkMethodRawCallback(17, + new ::grpc_impl::internal::CallbackUnaryHandler< ::grpc::ByteBuffer, ::grpc::ByteBuffer>( + [this](::grpc::ServerContext* context, + const ::grpc::ByteBuffer* request, + ::grpc::ByteBuffer* response, + ::grpc::experimental::ServerCallbackRpcController* controller) { + this->PreloadTable(context, request, response, controller); + })); + } + ~ExperimentalWithRawCallbackMethod_PreloadTable() override { + BaseClassMustBeDerivedFromService(this); + } + // disable synchronous version of this method + ::grpc::Status PreloadTable(::grpc::ServerContext* /*context*/, const ::milvus::grpc::TableName* /*request*/, ::milvus::grpc::Status* /*response*/) override { + abort(); + return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, ""); + } + virtual void PreloadTable(::grpc::ServerContext* /*context*/, const ::grpc::ByteBuffer* /*request*/, ::grpc::ByteBuffer* /*response*/, ::grpc::experimental::ServerCallbackRpcController* controller) { controller->Finish(::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, "")); } + }; + template class WithStreamedUnaryMethod_CreateTable : public BaseClass { private: void BaseClassMustBeDerivedFromService(const Service* /*service*/) {} @@ -2573,112 +2984,12 @@ class MilvusService final { virtual ::grpc::Status StreamedHasTable(::grpc::ServerContext* context, ::grpc::ServerUnaryStreamer< ::milvus::grpc::TableName,::milvus::grpc::BoolReply>* server_unary_streamer) = 0; }; template - class WithStreamedUnaryMethod_DropTable : public BaseClass { - private: - void BaseClassMustBeDerivedFromService(const Service* /*service*/) {} - public: - WithStreamedUnaryMethod_DropTable() { - ::grpc::Service::MarkMethodStreamed(2, - new ::grpc::internal::StreamedUnaryHandler< ::milvus::grpc::TableName, ::milvus::grpc::Status>(std::bind(&WithStreamedUnaryMethod_DropTable::StreamedDropTable, this, std::placeholders::_1, std::placeholders::_2))); - } - ~WithStreamedUnaryMethod_DropTable() override { - BaseClassMustBeDerivedFromService(this); - } - // disable regular version of this method - ::grpc::Status DropTable(::grpc::ServerContext* /*context*/, const ::milvus::grpc::TableName* /*request*/, ::milvus::grpc::Status* /*response*/) override { - abort(); - return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, ""); - } - // replace default version of method with streamed unary - virtual ::grpc::Status StreamedDropTable(::grpc::ServerContext* context, ::grpc::ServerUnaryStreamer< ::milvus::grpc::TableName,::milvus::grpc::Status>* server_unary_streamer) = 0; - }; - template - class WithStreamedUnaryMethod_CreateIndex : public BaseClass { - private: - void BaseClassMustBeDerivedFromService(const Service* /*service*/) {} - public: - WithStreamedUnaryMethod_CreateIndex() { - ::grpc::Service::MarkMethodStreamed(3, - new ::grpc::internal::StreamedUnaryHandler< ::milvus::grpc::IndexParam, ::milvus::grpc::Status>(std::bind(&WithStreamedUnaryMethod_CreateIndex::StreamedCreateIndex, this, std::placeholders::_1, std::placeholders::_2))); - } - ~WithStreamedUnaryMethod_CreateIndex() override { - BaseClassMustBeDerivedFromService(this); - } - // disable regular version of this method - ::grpc::Status CreateIndex(::grpc::ServerContext* /*context*/, const ::milvus::grpc::IndexParam* /*request*/, ::milvus::grpc::Status* /*response*/) override { - abort(); - return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, ""); - } - // replace default version of method with streamed unary - virtual ::grpc::Status StreamedCreateIndex(::grpc::ServerContext* context, ::grpc::ServerUnaryStreamer< ::milvus::grpc::IndexParam,::milvus::grpc::Status>* server_unary_streamer) = 0; - }; - template - class WithStreamedUnaryMethod_Insert : public BaseClass { - private: - void BaseClassMustBeDerivedFromService(const Service* /*service*/) {} - public: - WithStreamedUnaryMethod_Insert() { - ::grpc::Service::MarkMethodStreamed(4, - new ::grpc::internal::StreamedUnaryHandler< ::milvus::grpc::InsertParam, ::milvus::grpc::VectorIds>(std::bind(&WithStreamedUnaryMethod_Insert::StreamedInsert, this, std::placeholders::_1, std::placeholders::_2))); - } - ~WithStreamedUnaryMethod_Insert() override { - BaseClassMustBeDerivedFromService(this); - } - // disable regular version of this method - ::grpc::Status Insert(::grpc::ServerContext* /*context*/, const ::milvus::grpc::InsertParam* /*request*/, ::milvus::grpc::VectorIds* /*response*/) override { - abort(); - return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, ""); - } - // replace default version of method with streamed unary - virtual ::grpc::Status StreamedInsert(::grpc::ServerContext* context, ::grpc::ServerUnaryStreamer< ::milvus::grpc::InsertParam,::milvus::grpc::VectorIds>* server_unary_streamer) = 0; - }; - template - class WithStreamedUnaryMethod_Search : public BaseClass { - private: - void BaseClassMustBeDerivedFromService(const Service* /*service*/) {} - public: - WithStreamedUnaryMethod_Search() { - ::grpc::Service::MarkMethodStreamed(5, - new ::grpc::internal::StreamedUnaryHandler< ::milvus::grpc::SearchParam, ::milvus::grpc::TopKQueryResult>(std::bind(&WithStreamedUnaryMethod_Search::StreamedSearch, this, std::placeholders::_1, std::placeholders::_2))); - } - ~WithStreamedUnaryMethod_Search() override { - BaseClassMustBeDerivedFromService(this); - } - // disable regular version of this method - ::grpc::Status Search(::grpc::ServerContext* /*context*/, const ::milvus::grpc::SearchParam* /*request*/, ::milvus::grpc::TopKQueryResult* /*response*/) override { - abort(); - return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, ""); - } - // replace default version of method with streamed unary - virtual ::grpc::Status StreamedSearch(::grpc::ServerContext* context, ::grpc::ServerUnaryStreamer< ::milvus::grpc::SearchParam,::milvus::grpc::TopKQueryResult>* server_unary_streamer) = 0; - }; - template - class WithStreamedUnaryMethod_SearchInFiles : public BaseClass { - private: - void BaseClassMustBeDerivedFromService(const Service* /*service*/) {} - public: - WithStreamedUnaryMethod_SearchInFiles() { - ::grpc::Service::MarkMethodStreamed(6, - new ::grpc::internal::StreamedUnaryHandler< ::milvus::grpc::SearchInFilesParam, ::milvus::grpc::TopKQueryResult>(std::bind(&WithStreamedUnaryMethod_SearchInFiles::StreamedSearchInFiles, this, std::placeholders::_1, std::placeholders::_2))); - } - ~WithStreamedUnaryMethod_SearchInFiles() override { - BaseClassMustBeDerivedFromService(this); - } - // disable regular version of this method - ::grpc::Status SearchInFiles(::grpc::ServerContext* /*context*/, const ::milvus::grpc::SearchInFilesParam* /*request*/, ::milvus::grpc::TopKQueryResult* /*response*/) override { - abort(); - return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, ""); - } - // replace default version of method with streamed unary - virtual ::grpc::Status StreamedSearchInFiles(::grpc::ServerContext* context, ::grpc::ServerUnaryStreamer< ::milvus::grpc::SearchInFilesParam,::milvus::grpc::TopKQueryResult>* server_unary_streamer) = 0; - }; - template class WithStreamedUnaryMethod_DescribeTable : public BaseClass { private: void BaseClassMustBeDerivedFromService(const Service* /*service*/) {} public: WithStreamedUnaryMethod_DescribeTable() { - ::grpc::Service::MarkMethodStreamed(7, + ::grpc::Service::MarkMethodStreamed(2, new ::grpc::internal::StreamedUnaryHandler< ::milvus::grpc::TableName, ::milvus::grpc::TableSchema>(std::bind(&WithStreamedUnaryMethod_DescribeTable::StreamedDescribeTable, this, std::placeholders::_1, std::placeholders::_2))); } ~WithStreamedUnaryMethod_DescribeTable() override { @@ -2698,7 +3009,7 @@ class MilvusService final { void BaseClassMustBeDerivedFromService(const Service* /*service*/) {} public: WithStreamedUnaryMethod_CountTable() { - ::grpc::Service::MarkMethodStreamed(8, + ::grpc::Service::MarkMethodStreamed(3, new ::grpc::internal::StreamedUnaryHandler< ::milvus::grpc::TableName, ::milvus::grpc::TableRowCount>(std::bind(&WithStreamedUnaryMethod_CountTable::StreamedCountTable, this, std::placeholders::_1, std::placeholders::_2))); } ~WithStreamedUnaryMethod_CountTable() override { @@ -2718,7 +3029,7 @@ class MilvusService final { void BaseClassMustBeDerivedFromService(const Service* /*service*/) {} public: WithStreamedUnaryMethod_ShowTables() { - ::grpc::Service::MarkMethodStreamed(9, + ::grpc::Service::MarkMethodStreamed(4, new ::grpc::internal::StreamedUnaryHandler< ::milvus::grpc::Command, ::milvus::grpc::TableNameList>(std::bind(&WithStreamedUnaryMethod_ShowTables::StreamedShowTables, this, std::placeholders::_1, std::placeholders::_2))); } ~WithStreamedUnaryMethod_ShowTables() override { @@ -2733,64 +3044,44 @@ class MilvusService final { virtual ::grpc::Status StreamedShowTables(::grpc::ServerContext* context, ::grpc::ServerUnaryStreamer< ::milvus::grpc::Command,::milvus::grpc::TableNameList>* server_unary_streamer) = 0; }; template - class WithStreamedUnaryMethod_Cmd : public BaseClass { + class WithStreamedUnaryMethod_DropTable : public BaseClass { private: void BaseClassMustBeDerivedFromService(const Service* /*service*/) {} public: - WithStreamedUnaryMethod_Cmd() { - ::grpc::Service::MarkMethodStreamed(10, - new ::grpc::internal::StreamedUnaryHandler< ::milvus::grpc::Command, ::milvus::grpc::StringReply>(std::bind(&WithStreamedUnaryMethod_Cmd::StreamedCmd, this, std::placeholders::_1, std::placeholders::_2))); + WithStreamedUnaryMethod_DropTable() { + ::grpc::Service::MarkMethodStreamed(5, + new ::grpc::internal::StreamedUnaryHandler< ::milvus::grpc::TableName, ::milvus::grpc::Status>(std::bind(&WithStreamedUnaryMethod_DropTable::StreamedDropTable, this, std::placeholders::_1, std::placeholders::_2))); } - ~WithStreamedUnaryMethod_Cmd() override { + ~WithStreamedUnaryMethod_DropTable() override { BaseClassMustBeDerivedFromService(this); } // disable regular version of this method - ::grpc::Status Cmd(::grpc::ServerContext* /*context*/, const ::milvus::grpc::Command* /*request*/, ::milvus::grpc::StringReply* /*response*/) override { + ::grpc::Status DropTable(::grpc::ServerContext* /*context*/, const ::milvus::grpc::TableName* /*request*/, ::milvus::grpc::Status* /*response*/) override { abort(); return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, ""); } // replace default version of method with streamed unary - virtual ::grpc::Status StreamedCmd(::grpc::ServerContext* context, ::grpc::ServerUnaryStreamer< ::milvus::grpc::Command,::milvus::grpc::StringReply>* server_unary_streamer) = 0; + virtual ::grpc::Status StreamedDropTable(::grpc::ServerContext* context, ::grpc::ServerUnaryStreamer< ::milvus::grpc::TableName,::milvus::grpc::Status>* server_unary_streamer) = 0; }; template - class WithStreamedUnaryMethod_DeleteByRange : public BaseClass { + class WithStreamedUnaryMethod_CreateIndex : public BaseClass { private: void BaseClassMustBeDerivedFromService(const Service* /*service*/) {} public: - WithStreamedUnaryMethod_DeleteByRange() { - ::grpc::Service::MarkMethodStreamed(11, - new ::grpc::internal::StreamedUnaryHandler< ::milvus::grpc::DeleteByRangeParam, ::milvus::grpc::Status>(std::bind(&WithStreamedUnaryMethod_DeleteByRange::StreamedDeleteByRange, this, std::placeholders::_1, std::placeholders::_2))); + WithStreamedUnaryMethod_CreateIndex() { + ::grpc::Service::MarkMethodStreamed(6, + new ::grpc::internal::StreamedUnaryHandler< ::milvus::grpc::IndexParam, ::milvus::grpc::Status>(std::bind(&WithStreamedUnaryMethod_CreateIndex::StreamedCreateIndex, this, std::placeholders::_1, std::placeholders::_2))); } - ~WithStreamedUnaryMethod_DeleteByRange() override { + ~WithStreamedUnaryMethod_CreateIndex() override { BaseClassMustBeDerivedFromService(this); } // disable regular version of this method - ::grpc::Status DeleteByRange(::grpc::ServerContext* /*context*/, const ::milvus::grpc::DeleteByRangeParam* /*request*/, ::milvus::grpc::Status* /*response*/) override { + ::grpc::Status CreateIndex(::grpc::ServerContext* /*context*/, const ::milvus::grpc::IndexParam* /*request*/, ::milvus::grpc::Status* /*response*/) override { abort(); return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, ""); } // replace default version of method with streamed unary - virtual ::grpc::Status StreamedDeleteByRange(::grpc::ServerContext* context, ::grpc::ServerUnaryStreamer< ::milvus::grpc::DeleteByRangeParam,::milvus::grpc::Status>* server_unary_streamer) = 0; - }; - template - class WithStreamedUnaryMethod_PreloadTable : public BaseClass { - private: - void BaseClassMustBeDerivedFromService(const Service* /*service*/) {} - public: - WithStreamedUnaryMethod_PreloadTable() { - ::grpc::Service::MarkMethodStreamed(12, - new ::grpc::internal::StreamedUnaryHandler< ::milvus::grpc::TableName, ::milvus::grpc::Status>(std::bind(&WithStreamedUnaryMethod_PreloadTable::StreamedPreloadTable, this, std::placeholders::_1, std::placeholders::_2))); - } - ~WithStreamedUnaryMethod_PreloadTable() override { - BaseClassMustBeDerivedFromService(this); - } - // disable regular version of this method - ::grpc::Status PreloadTable(::grpc::ServerContext* /*context*/, const ::milvus::grpc::TableName* /*request*/, ::milvus::grpc::Status* /*response*/) override { - abort(); - return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, ""); - } - // replace default version of method with streamed unary - virtual ::grpc::Status StreamedPreloadTable(::grpc::ServerContext* context, ::grpc::ServerUnaryStreamer< ::milvus::grpc::TableName,::milvus::grpc::Status>* server_unary_streamer) = 0; + virtual ::grpc::Status StreamedCreateIndex(::grpc::ServerContext* context, ::grpc::ServerUnaryStreamer< ::milvus::grpc::IndexParam,::milvus::grpc::Status>* server_unary_streamer) = 0; }; template class WithStreamedUnaryMethod_DescribeIndex : public BaseClass { @@ -2798,7 +3089,7 @@ class MilvusService final { void BaseClassMustBeDerivedFromService(const Service* /*service*/) {} public: WithStreamedUnaryMethod_DescribeIndex() { - ::grpc::Service::MarkMethodStreamed(13, + ::grpc::Service::MarkMethodStreamed(7, new ::grpc::internal::StreamedUnaryHandler< ::milvus::grpc::TableName, ::milvus::grpc::IndexParam>(std::bind(&WithStreamedUnaryMethod_DescribeIndex::StreamedDescribeIndex, this, std::placeholders::_1, std::placeholders::_2))); } ~WithStreamedUnaryMethod_DescribeIndex() override { @@ -2818,7 +3109,7 @@ class MilvusService final { void BaseClassMustBeDerivedFromService(const Service* /*service*/) {} public: WithStreamedUnaryMethod_DropIndex() { - ::grpc::Service::MarkMethodStreamed(14, + ::grpc::Service::MarkMethodStreamed(8, new ::grpc::internal::StreamedUnaryHandler< ::milvus::grpc::TableName, ::milvus::grpc::Status>(std::bind(&WithStreamedUnaryMethod_DropIndex::StreamedDropIndex, this, std::placeholders::_1, std::placeholders::_2))); } ~WithStreamedUnaryMethod_DropIndex() override { @@ -2832,9 +3123,189 @@ class MilvusService final { // replace default version of method with streamed unary virtual ::grpc::Status StreamedDropIndex(::grpc::ServerContext* context, ::grpc::ServerUnaryStreamer< ::milvus::grpc::TableName,::milvus::grpc::Status>* server_unary_streamer) = 0; }; - typedef WithStreamedUnaryMethod_CreateTable > > > > > > > > > > > > > > StreamedUnaryService; + template + class WithStreamedUnaryMethod_CreatePartition : public BaseClass { + private: + void BaseClassMustBeDerivedFromService(const Service* /*service*/) {} + public: + WithStreamedUnaryMethod_CreatePartition() { + ::grpc::Service::MarkMethodStreamed(9, + new ::grpc::internal::StreamedUnaryHandler< ::milvus::grpc::PartitionParam, ::milvus::grpc::Status>(std::bind(&WithStreamedUnaryMethod_CreatePartition::StreamedCreatePartition, this, std::placeholders::_1, std::placeholders::_2))); + } + ~WithStreamedUnaryMethod_CreatePartition() override { + BaseClassMustBeDerivedFromService(this); + } + // disable regular version of this method + ::grpc::Status CreatePartition(::grpc::ServerContext* /*context*/, const ::milvus::grpc::PartitionParam* /*request*/, ::milvus::grpc::Status* /*response*/) override { + abort(); + return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, ""); + } + // replace default version of method with streamed unary + virtual ::grpc::Status StreamedCreatePartition(::grpc::ServerContext* context, ::grpc::ServerUnaryStreamer< ::milvus::grpc::PartitionParam,::milvus::grpc::Status>* server_unary_streamer) = 0; + }; + template + class WithStreamedUnaryMethod_ShowPartitions : public BaseClass { + private: + void BaseClassMustBeDerivedFromService(const Service* /*service*/) {} + public: + WithStreamedUnaryMethod_ShowPartitions() { + ::grpc::Service::MarkMethodStreamed(10, + new ::grpc::internal::StreamedUnaryHandler< ::milvus::grpc::TableName, ::milvus::grpc::PartitionList>(std::bind(&WithStreamedUnaryMethod_ShowPartitions::StreamedShowPartitions, this, std::placeholders::_1, std::placeholders::_2))); + } + ~WithStreamedUnaryMethod_ShowPartitions() override { + BaseClassMustBeDerivedFromService(this); + } + // disable regular version of this method + ::grpc::Status ShowPartitions(::grpc::ServerContext* /*context*/, const ::milvus::grpc::TableName* /*request*/, ::milvus::grpc::PartitionList* /*response*/) override { + abort(); + return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, ""); + } + // replace default version of method with streamed unary + virtual ::grpc::Status StreamedShowPartitions(::grpc::ServerContext* context, ::grpc::ServerUnaryStreamer< ::milvus::grpc::TableName,::milvus::grpc::PartitionList>* server_unary_streamer) = 0; + }; + template + class WithStreamedUnaryMethod_DropPartition : public BaseClass { + private: + void BaseClassMustBeDerivedFromService(const Service* /*service*/) {} + public: + WithStreamedUnaryMethod_DropPartition() { + ::grpc::Service::MarkMethodStreamed(11, + new ::grpc::internal::StreamedUnaryHandler< ::milvus::grpc::PartitionParam, ::milvus::grpc::Status>(std::bind(&WithStreamedUnaryMethod_DropPartition::StreamedDropPartition, this, std::placeholders::_1, std::placeholders::_2))); + } + ~WithStreamedUnaryMethod_DropPartition() override { + BaseClassMustBeDerivedFromService(this); + } + // disable regular version of this method + ::grpc::Status DropPartition(::grpc::ServerContext* /*context*/, const ::milvus::grpc::PartitionParam* /*request*/, ::milvus::grpc::Status* /*response*/) override { + abort(); + return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, ""); + } + // replace default version of method with streamed unary + virtual ::grpc::Status StreamedDropPartition(::grpc::ServerContext* context, ::grpc::ServerUnaryStreamer< ::milvus::grpc::PartitionParam,::milvus::grpc::Status>* server_unary_streamer) = 0; + }; + template + class WithStreamedUnaryMethod_Insert : public BaseClass { + private: + void BaseClassMustBeDerivedFromService(const Service* /*service*/) {} + public: + WithStreamedUnaryMethod_Insert() { + ::grpc::Service::MarkMethodStreamed(12, + new ::grpc::internal::StreamedUnaryHandler< ::milvus::grpc::InsertParam, ::milvus::grpc::VectorIds>(std::bind(&WithStreamedUnaryMethod_Insert::StreamedInsert, this, std::placeholders::_1, std::placeholders::_2))); + } + ~WithStreamedUnaryMethod_Insert() override { + BaseClassMustBeDerivedFromService(this); + } + // disable regular version of this method + ::grpc::Status Insert(::grpc::ServerContext* /*context*/, const ::milvus::grpc::InsertParam* /*request*/, ::milvus::grpc::VectorIds* /*response*/) override { + abort(); + return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, ""); + } + // replace default version of method with streamed unary + virtual ::grpc::Status StreamedInsert(::grpc::ServerContext* context, ::grpc::ServerUnaryStreamer< ::milvus::grpc::InsertParam,::milvus::grpc::VectorIds>* server_unary_streamer) = 0; + }; + template + class WithStreamedUnaryMethod_Search : public BaseClass { + private: + void BaseClassMustBeDerivedFromService(const Service* /*service*/) {} + public: + WithStreamedUnaryMethod_Search() { + ::grpc::Service::MarkMethodStreamed(13, + new ::grpc::internal::StreamedUnaryHandler< ::milvus::grpc::SearchParam, ::milvus::grpc::TopKQueryResult>(std::bind(&WithStreamedUnaryMethod_Search::StreamedSearch, this, std::placeholders::_1, std::placeholders::_2))); + } + ~WithStreamedUnaryMethod_Search() override { + BaseClassMustBeDerivedFromService(this); + } + // disable regular version of this method + ::grpc::Status Search(::grpc::ServerContext* /*context*/, const ::milvus::grpc::SearchParam* /*request*/, ::milvus::grpc::TopKQueryResult* /*response*/) override { + abort(); + return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, ""); + } + // replace default version of method with streamed unary + virtual ::grpc::Status StreamedSearch(::grpc::ServerContext* context, ::grpc::ServerUnaryStreamer< ::milvus::grpc::SearchParam,::milvus::grpc::TopKQueryResult>* server_unary_streamer) = 0; + }; + template + class WithStreamedUnaryMethod_SearchInFiles : public BaseClass { + private: + void BaseClassMustBeDerivedFromService(const Service* /*service*/) {} + public: + WithStreamedUnaryMethod_SearchInFiles() { + ::grpc::Service::MarkMethodStreamed(14, + new ::grpc::internal::StreamedUnaryHandler< ::milvus::grpc::SearchInFilesParam, ::milvus::grpc::TopKQueryResult>(std::bind(&WithStreamedUnaryMethod_SearchInFiles::StreamedSearchInFiles, this, std::placeholders::_1, std::placeholders::_2))); + } + ~WithStreamedUnaryMethod_SearchInFiles() override { + BaseClassMustBeDerivedFromService(this); + } + // disable regular version of this method + ::grpc::Status SearchInFiles(::grpc::ServerContext* /*context*/, const ::milvus::grpc::SearchInFilesParam* /*request*/, ::milvus::grpc::TopKQueryResult* /*response*/) override { + abort(); + return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, ""); + } + // replace default version of method with streamed unary + virtual ::grpc::Status StreamedSearchInFiles(::grpc::ServerContext* context, ::grpc::ServerUnaryStreamer< ::milvus::grpc::SearchInFilesParam,::milvus::grpc::TopKQueryResult>* server_unary_streamer) = 0; + }; + template + class WithStreamedUnaryMethod_Cmd : public BaseClass { + private: + void BaseClassMustBeDerivedFromService(const Service* /*service*/) {} + public: + WithStreamedUnaryMethod_Cmd() { + ::grpc::Service::MarkMethodStreamed(15, + new ::grpc::internal::StreamedUnaryHandler< ::milvus::grpc::Command, ::milvus::grpc::StringReply>(std::bind(&WithStreamedUnaryMethod_Cmd::StreamedCmd, this, std::placeholders::_1, std::placeholders::_2))); + } + ~WithStreamedUnaryMethod_Cmd() override { + BaseClassMustBeDerivedFromService(this); + } + // disable regular version of this method + ::grpc::Status Cmd(::grpc::ServerContext* /*context*/, const ::milvus::grpc::Command* /*request*/, ::milvus::grpc::StringReply* /*response*/) override { + abort(); + return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, ""); + } + // replace default version of method with streamed unary + virtual ::grpc::Status StreamedCmd(::grpc::ServerContext* context, ::grpc::ServerUnaryStreamer< ::milvus::grpc::Command,::milvus::grpc::StringReply>* server_unary_streamer) = 0; + }; + template + class WithStreamedUnaryMethod_DeleteByDate : public BaseClass { + private: + void BaseClassMustBeDerivedFromService(const Service* /*service*/) {} + public: + WithStreamedUnaryMethod_DeleteByDate() { + ::grpc::Service::MarkMethodStreamed(16, + new ::grpc::internal::StreamedUnaryHandler< ::milvus::grpc::DeleteByDateParam, ::milvus::grpc::Status>(std::bind(&WithStreamedUnaryMethod_DeleteByDate::StreamedDeleteByDate, this, std::placeholders::_1, std::placeholders::_2))); + } + ~WithStreamedUnaryMethod_DeleteByDate() override { + BaseClassMustBeDerivedFromService(this); + } + // disable regular version of this method + ::grpc::Status DeleteByDate(::grpc::ServerContext* /*context*/, const ::milvus::grpc::DeleteByDateParam* /*request*/, ::milvus::grpc::Status* /*response*/) override { + abort(); + return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, ""); + } + // replace default version of method with streamed unary + virtual ::grpc::Status StreamedDeleteByDate(::grpc::ServerContext* context, ::grpc::ServerUnaryStreamer< ::milvus::grpc::DeleteByDateParam,::milvus::grpc::Status>* server_unary_streamer) = 0; + }; + template + class WithStreamedUnaryMethod_PreloadTable : public BaseClass { + private: + void BaseClassMustBeDerivedFromService(const Service* /*service*/) {} + public: + WithStreamedUnaryMethod_PreloadTable() { + ::grpc::Service::MarkMethodStreamed(17, + new ::grpc::internal::StreamedUnaryHandler< ::milvus::grpc::TableName, ::milvus::grpc::Status>(std::bind(&WithStreamedUnaryMethod_PreloadTable::StreamedPreloadTable, this, std::placeholders::_1, std::placeholders::_2))); + } + ~WithStreamedUnaryMethod_PreloadTable() override { + BaseClassMustBeDerivedFromService(this); + } + // disable regular version of this method + ::grpc::Status PreloadTable(::grpc::ServerContext* /*context*/, const ::milvus::grpc::TableName* /*request*/, ::milvus::grpc::Status* /*response*/) override { + abort(); + return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, ""); + } + // replace default version of method with streamed unary + virtual ::grpc::Status StreamedPreloadTable(::grpc::ServerContext* context, ::grpc::ServerUnaryStreamer< ::milvus::grpc::TableName,::milvus::grpc::Status>* server_unary_streamer) = 0; + }; + typedef WithStreamedUnaryMethod_CreateTable > > > > > > > > > > > > > > > > > StreamedUnaryService; typedef Service SplitStreamedService; - typedef WithStreamedUnaryMethod_CreateTable > > > > > > > > > > > > > > StreamedService; + typedef WithStreamedUnaryMethod_CreateTable > > > > > > > > > > > > > > > > > StreamedService; }; } // namespace grpc diff --git a/core/src/grpc/gen-milvus/milvus.pb.cc b/core/src/grpc/gen-milvus/milvus.pb.cc index 753e491663..cbc1b1118f 100644 --- a/core/src/grpc/gen-milvus/milvus.pb.cc +++ b/core/src/grpc/gen-milvus/milvus.pb.cc @@ -16,6 +16,7 @@ // @@protoc_insertion_point(includes) #include extern PROTOBUF_INTERNAL_EXPORT_milvus_2eproto ::PROTOBUF_NAMESPACE_ID::internal::SCCInfo<0> scc_info_Index_milvus_2eproto; +extern PROTOBUF_INTERNAL_EXPORT_milvus_2eproto ::PROTOBUF_NAMESPACE_ID::internal::SCCInfo<0> scc_info_PartitionParam_milvus_2eproto; extern PROTOBUF_INTERNAL_EXPORT_milvus_2eproto ::PROTOBUF_NAMESPACE_ID::internal::SCCInfo<0> scc_info_Range_milvus_2eproto; extern PROTOBUF_INTERNAL_EXPORT_milvus_2eproto ::PROTOBUF_NAMESPACE_ID::internal::SCCInfo<0> scc_info_RowRecord_milvus_2eproto; extern PROTOBUF_INTERNAL_EXPORT_milvus_2eproto ::PROTOBUF_NAMESPACE_ID::internal::SCCInfo<2> scc_info_SearchParam_milvus_2eproto; @@ -26,6 +27,10 @@ class TableNameDefaultTypeInternal { public: ::PROTOBUF_NAMESPACE_ID::internal::ExplicitlyConstructed _instance; } _TableName_default_instance_; +class PartitionNameDefaultTypeInternal { + public: + ::PROTOBUF_NAMESPACE_ID::internal::ExplicitlyConstructed _instance; +} _PartitionName_default_instance_; class TableNameListDefaultTypeInternal { public: ::PROTOBUF_NAMESPACE_ID::internal::ExplicitlyConstructed _instance; @@ -34,6 +39,14 @@ class TableSchemaDefaultTypeInternal { public: ::PROTOBUF_NAMESPACE_ID::internal::ExplicitlyConstructed _instance; } _TableSchema_default_instance_; +class PartitionParamDefaultTypeInternal { + public: + ::PROTOBUF_NAMESPACE_ID::internal::ExplicitlyConstructed _instance; +} _PartitionParam_default_instance_; +class PartitionListDefaultTypeInternal { + public: + ::PROTOBUF_NAMESPACE_ID::internal::ExplicitlyConstructed _instance; +} _PartitionList_default_instance_; class RangeDefaultTypeInternal { public: ::PROTOBUF_NAMESPACE_ID::internal::ExplicitlyConstructed _instance; @@ -86,10 +99,10 @@ class IndexParamDefaultTypeInternal { public: ::PROTOBUF_NAMESPACE_ID::internal::ExplicitlyConstructed _instance; } _IndexParam_default_instance_; -class DeleteByRangeParamDefaultTypeInternal { +class DeleteByDateParamDefaultTypeInternal { public: - ::PROTOBUF_NAMESPACE_ID::internal::ExplicitlyConstructed _instance; -} _DeleteByRangeParam_default_instance_; + ::PROTOBUF_NAMESPACE_ID::internal::ExplicitlyConstructed _instance; +} _DeleteByDateParam_default_instance_; } // namespace grpc } // namespace milvus static void InitDefaultsscc_info_BoolReply_milvus_2eproto() { @@ -121,19 +134,19 @@ static void InitDefaultsscc_info_Command_milvus_2eproto() { ::PROTOBUF_NAMESPACE_ID::internal::SCCInfo<0> scc_info_Command_milvus_2eproto = {{ATOMIC_VAR_INIT(::PROTOBUF_NAMESPACE_ID::internal::SCCInfoBase::kUninitialized), 0, InitDefaultsscc_info_Command_milvus_2eproto}, {}}; -static void InitDefaultsscc_info_DeleteByRangeParam_milvus_2eproto() { +static void InitDefaultsscc_info_DeleteByDateParam_milvus_2eproto() { GOOGLE_PROTOBUF_VERIFY_VERSION; { - void* ptr = &::milvus::grpc::_DeleteByRangeParam_default_instance_; - new (ptr) ::milvus::grpc::DeleteByRangeParam(); + void* ptr = &::milvus::grpc::_DeleteByDateParam_default_instance_; + new (ptr) ::milvus::grpc::DeleteByDateParam(); ::PROTOBUF_NAMESPACE_ID::internal::OnShutdownDestroyMessage(ptr); } - ::milvus::grpc::DeleteByRangeParam::InitAsDefaultInstance(); + ::milvus::grpc::DeleteByDateParam::InitAsDefaultInstance(); } -::PROTOBUF_NAMESPACE_ID::internal::SCCInfo<1> scc_info_DeleteByRangeParam_milvus_2eproto = - {{ATOMIC_VAR_INIT(::PROTOBUF_NAMESPACE_ID::internal::SCCInfoBase::kUninitialized), 1, InitDefaultsscc_info_DeleteByRangeParam_milvus_2eproto}, { +::PROTOBUF_NAMESPACE_ID::internal::SCCInfo<1> scc_info_DeleteByDateParam_milvus_2eproto = + {{ATOMIC_VAR_INIT(::PROTOBUF_NAMESPACE_ID::internal::SCCInfoBase::kUninitialized), 1, InitDefaultsscc_info_DeleteByDateParam_milvus_2eproto}, { &scc_info_Range_milvus_2eproto.base,}}; static void InitDefaultsscc_info_Index_milvus_2eproto() { @@ -181,6 +194,50 @@ static void InitDefaultsscc_info_InsertParam_milvus_2eproto() { {{ATOMIC_VAR_INIT(::PROTOBUF_NAMESPACE_ID::internal::SCCInfoBase::kUninitialized), 1, InitDefaultsscc_info_InsertParam_milvus_2eproto}, { &scc_info_RowRecord_milvus_2eproto.base,}}; +static void InitDefaultsscc_info_PartitionList_milvus_2eproto() { + GOOGLE_PROTOBUF_VERIFY_VERSION; + + { + void* ptr = &::milvus::grpc::_PartitionList_default_instance_; + new (ptr) ::milvus::grpc::PartitionList(); + ::PROTOBUF_NAMESPACE_ID::internal::OnShutdownDestroyMessage(ptr); + } + ::milvus::grpc::PartitionList::InitAsDefaultInstance(); +} + +::PROTOBUF_NAMESPACE_ID::internal::SCCInfo<2> scc_info_PartitionList_milvus_2eproto = + {{ATOMIC_VAR_INIT(::PROTOBUF_NAMESPACE_ID::internal::SCCInfoBase::kUninitialized), 2, InitDefaultsscc_info_PartitionList_milvus_2eproto}, { + &scc_info_Status_status_2eproto.base, + &scc_info_PartitionParam_milvus_2eproto.base,}}; + +static void InitDefaultsscc_info_PartitionName_milvus_2eproto() { + GOOGLE_PROTOBUF_VERIFY_VERSION; + + { + void* ptr = &::milvus::grpc::_PartitionName_default_instance_; + new (ptr) ::milvus::grpc::PartitionName(); + ::PROTOBUF_NAMESPACE_ID::internal::OnShutdownDestroyMessage(ptr); + } + ::milvus::grpc::PartitionName::InitAsDefaultInstance(); +} + +::PROTOBUF_NAMESPACE_ID::internal::SCCInfo<0> scc_info_PartitionName_milvus_2eproto = + {{ATOMIC_VAR_INIT(::PROTOBUF_NAMESPACE_ID::internal::SCCInfoBase::kUninitialized), 0, InitDefaultsscc_info_PartitionName_milvus_2eproto}, {}}; + +static void InitDefaultsscc_info_PartitionParam_milvus_2eproto() { + GOOGLE_PROTOBUF_VERIFY_VERSION; + + { + void* ptr = &::milvus::grpc::_PartitionParam_default_instance_; + new (ptr) ::milvus::grpc::PartitionParam(); + ::PROTOBUF_NAMESPACE_ID::internal::OnShutdownDestroyMessage(ptr); + } + ::milvus::grpc::PartitionParam::InitAsDefaultInstance(); +} + +::PROTOBUF_NAMESPACE_ID::internal::SCCInfo<0> scc_info_PartitionParam_milvus_2eproto = + {{ATOMIC_VAR_INIT(::PROTOBUF_NAMESPACE_ID::internal::SCCInfoBase::kUninitialized), 0, InitDefaultsscc_info_PartitionParam_milvus_2eproto}, {}}; + static void InitDefaultsscc_info_Range_milvus_2eproto() { GOOGLE_PROTOBUF_VERIFY_VERSION; @@ -344,7 +401,7 @@ static void InitDefaultsscc_info_VectorIds_milvus_2eproto() { {{ATOMIC_VAR_INIT(::PROTOBUF_NAMESPACE_ID::internal::SCCInfoBase::kUninitialized), 1, InitDefaultsscc_info_VectorIds_milvus_2eproto}, { &scc_info_Status_status_2eproto.base,}}; -static ::PROTOBUF_NAMESPACE_ID::Metadata file_level_metadata_milvus_2eproto[17]; +static ::PROTOBUF_NAMESPACE_ID::Metadata file_level_metadata_milvus_2eproto[20]; static constexpr ::PROTOBUF_NAMESPACE_ID::EnumDescriptor const** file_level_enum_descriptors_milvus_2eproto = nullptr; static constexpr ::PROTOBUF_NAMESPACE_ID::ServiceDescriptor const** file_level_service_descriptors_milvus_2eproto = nullptr; @@ -356,6 +413,12 @@ const ::PROTOBUF_NAMESPACE_ID::uint32 TableStruct_milvus_2eproto::offsets[] PROT ~0u, // no _weak_field_map_ PROTOBUF_FIELD_OFFSET(::milvus::grpc::TableName, table_name_), ~0u, // no _has_bits_ + PROTOBUF_FIELD_OFFSET(::milvus::grpc::PartitionName, _internal_metadata_), + ~0u, // no _extensions_ + ~0u, // no _oneof_case_ + ~0u, // no _weak_field_map_ + PROTOBUF_FIELD_OFFSET(::milvus::grpc::PartitionName, partition_name_), + ~0u, // no _has_bits_ PROTOBUF_FIELD_OFFSET(::milvus::grpc::TableNameList, _internal_metadata_), ~0u, // no _extensions_ ~0u, // no _oneof_case_ @@ -373,6 +436,21 @@ const ::PROTOBUF_NAMESPACE_ID::uint32 TableStruct_milvus_2eproto::offsets[] PROT PROTOBUF_FIELD_OFFSET(::milvus::grpc::TableSchema, index_file_size_), PROTOBUF_FIELD_OFFSET(::milvus::grpc::TableSchema, metric_type_), ~0u, // no _has_bits_ + PROTOBUF_FIELD_OFFSET(::milvus::grpc::PartitionParam, _internal_metadata_), + ~0u, // no _extensions_ + ~0u, // no _oneof_case_ + ~0u, // no _weak_field_map_ + PROTOBUF_FIELD_OFFSET(::milvus::grpc::PartitionParam, table_name_), + PROTOBUF_FIELD_OFFSET(::milvus::grpc::PartitionParam, partition_name_), + PROTOBUF_FIELD_OFFSET(::milvus::grpc::PartitionParam, tag_), + ~0u, // no _has_bits_ + PROTOBUF_FIELD_OFFSET(::milvus::grpc::PartitionList, _internal_metadata_), + ~0u, // no _extensions_ + ~0u, // no _oneof_case_ + ~0u, // no _weak_field_map_ + PROTOBUF_FIELD_OFFSET(::milvus::grpc::PartitionList, status_), + PROTOBUF_FIELD_OFFSET(::milvus::grpc::PartitionList, partition_array_), + ~0u, // no _has_bits_ PROTOBUF_FIELD_OFFSET(::milvus::grpc::Range, _internal_metadata_), ~0u, // no _extensions_ ~0u, // no _oneof_case_ @@ -393,6 +471,7 @@ const ::PROTOBUF_NAMESPACE_ID::uint32 TableStruct_milvus_2eproto::offsets[] PROT PROTOBUF_FIELD_OFFSET(::milvus::grpc::InsertParam, table_name_), PROTOBUF_FIELD_OFFSET(::milvus::grpc::InsertParam, row_record_array_), PROTOBUF_FIELD_OFFSET(::milvus::grpc::InsertParam, row_id_array_), + PROTOBUF_FIELD_OFFSET(::milvus::grpc::InsertParam, partition_tag_), ~0u, // no _has_bits_ PROTOBUF_FIELD_OFFSET(::milvus::grpc::VectorIds, _internal_metadata_), ~0u, // no _extensions_ @@ -410,6 +489,7 @@ const ::PROTOBUF_NAMESPACE_ID::uint32 TableStruct_milvus_2eproto::offsets[] PROT PROTOBUF_FIELD_OFFSET(::milvus::grpc::SearchParam, query_range_array_), PROTOBUF_FIELD_OFFSET(::milvus::grpc::SearchParam, topk_), PROTOBUF_FIELD_OFFSET(::milvus::grpc::SearchParam, nprobe_), + PROTOBUF_FIELD_OFFSET(::milvus::grpc::SearchParam, partition_tag_array_), ~0u, // no _has_bits_ PROTOBUF_FIELD_OFFSET(::milvus::grpc::SearchInFilesParam, _internal_metadata_), ~0u, // no _extensions_ @@ -469,37 +549,43 @@ const ::PROTOBUF_NAMESPACE_ID::uint32 TableStruct_milvus_2eproto::offsets[] PROT PROTOBUF_FIELD_OFFSET(::milvus::grpc::IndexParam, table_name_), PROTOBUF_FIELD_OFFSET(::milvus::grpc::IndexParam, index_), ~0u, // no _has_bits_ - PROTOBUF_FIELD_OFFSET(::milvus::grpc::DeleteByRangeParam, _internal_metadata_), + PROTOBUF_FIELD_OFFSET(::milvus::grpc::DeleteByDateParam, _internal_metadata_), ~0u, // no _extensions_ ~0u, // no _oneof_case_ ~0u, // no _weak_field_map_ - PROTOBUF_FIELD_OFFSET(::milvus::grpc::DeleteByRangeParam, range_), - PROTOBUF_FIELD_OFFSET(::milvus::grpc::DeleteByRangeParam, table_name_), + PROTOBUF_FIELD_OFFSET(::milvus::grpc::DeleteByDateParam, range_), + PROTOBUF_FIELD_OFFSET(::milvus::grpc::DeleteByDateParam, table_name_), }; static const ::PROTOBUF_NAMESPACE_ID::internal::MigrationSchema schemas[] PROTOBUF_SECTION_VARIABLE(protodesc_cold) = { { 0, -1, sizeof(::milvus::grpc::TableName)}, - { 6, -1, sizeof(::milvus::grpc::TableNameList)}, - { 13, -1, sizeof(::milvus::grpc::TableSchema)}, - { 23, -1, sizeof(::milvus::grpc::Range)}, - { 30, -1, sizeof(::milvus::grpc::RowRecord)}, - { 36, -1, sizeof(::milvus::grpc::InsertParam)}, - { 44, -1, sizeof(::milvus::grpc::VectorIds)}, - { 51, -1, sizeof(::milvus::grpc::SearchParam)}, - { 61, -1, sizeof(::milvus::grpc::SearchInFilesParam)}, - { 68, -1, sizeof(::milvus::grpc::TopKQueryResult)}, - { 77, -1, sizeof(::milvus::grpc::StringReply)}, - { 84, -1, sizeof(::milvus::grpc::BoolReply)}, - { 91, -1, sizeof(::milvus::grpc::TableRowCount)}, - { 98, -1, sizeof(::milvus::grpc::Command)}, - { 104, -1, sizeof(::milvus::grpc::Index)}, - { 111, -1, sizeof(::milvus::grpc::IndexParam)}, - { 119, -1, sizeof(::milvus::grpc::DeleteByRangeParam)}, + { 6, -1, sizeof(::milvus::grpc::PartitionName)}, + { 12, -1, sizeof(::milvus::grpc::TableNameList)}, + { 19, -1, sizeof(::milvus::grpc::TableSchema)}, + { 29, -1, sizeof(::milvus::grpc::PartitionParam)}, + { 37, -1, sizeof(::milvus::grpc::PartitionList)}, + { 44, -1, sizeof(::milvus::grpc::Range)}, + { 51, -1, sizeof(::milvus::grpc::RowRecord)}, + { 57, -1, sizeof(::milvus::grpc::InsertParam)}, + { 66, -1, sizeof(::milvus::grpc::VectorIds)}, + { 73, -1, sizeof(::milvus::grpc::SearchParam)}, + { 84, -1, sizeof(::milvus::grpc::SearchInFilesParam)}, + { 91, -1, sizeof(::milvus::grpc::TopKQueryResult)}, + { 100, -1, sizeof(::milvus::grpc::StringReply)}, + { 107, -1, sizeof(::milvus::grpc::BoolReply)}, + { 114, -1, sizeof(::milvus::grpc::TableRowCount)}, + { 121, -1, sizeof(::milvus::grpc::Command)}, + { 127, -1, sizeof(::milvus::grpc::Index)}, + { 134, -1, sizeof(::milvus::grpc::IndexParam)}, + { 142, -1, sizeof(::milvus::grpc::DeleteByDateParam)}, }; static ::PROTOBUF_NAMESPACE_ID::Message const * const file_default_instances[] = { reinterpret_cast(&::milvus::grpc::_TableName_default_instance_), + reinterpret_cast(&::milvus::grpc::_PartitionName_default_instance_), reinterpret_cast(&::milvus::grpc::_TableNameList_default_instance_), reinterpret_cast(&::milvus::grpc::_TableSchema_default_instance_), + reinterpret_cast(&::milvus::grpc::_PartitionParam_default_instance_), + reinterpret_cast(&::milvus::grpc::_PartitionList_default_instance_), reinterpret_cast(&::milvus::grpc::_Range_default_instance_), reinterpret_cast(&::milvus::grpc::_RowRecord_default_instance_), reinterpret_cast(&::milvus::grpc::_InsertParam_default_instance_), @@ -513,81 +599,97 @@ static ::PROTOBUF_NAMESPACE_ID::Message const * const file_default_instances[] = reinterpret_cast(&::milvus::grpc::_Command_default_instance_), reinterpret_cast(&::milvus::grpc::_Index_default_instance_), reinterpret_cast(&::milvus::grpc::_IndexParam_default_instance_), - reinterpret_cast(&::milvus::grpc::_DeleteByRangeParam_default_instance_), + reinterpret_cast(&::milvus::grpc::_DeleteByDateParam_default_instance_), }; const char descriptor_table_protodef_milvus_2eproto[] PROTOBUF_SECTION_VARIABLE(protodesc_cold) = "\n\014milvus.proto\022\013milvus.grpc\032\014status.prot" - "o\"\037\n\tTableName\022\022\n\ntable_name\030\001 \001(\t\"I\n\rTa" - "bleNameList\022#\n\006status\030\001 \001(\0132\023.milvus.grp" - "c.Status\022\023\n\013table_names\030\002 \003(\t\"\207\001\n\013TableS" - "chema\022#\n\006status\030\001 \001(\0132\023.milvus.grpc.Stat" - "us\022\022\n\ntable_name\030\002 \001(\t\022\021\n\tdimension\030\003 \001(" - "\003\022\027\n\017index_file_size\030\004 \001(\003\022\023\n\013metric_typ" - "e\030\005 \001(\005\"/\n\005Range\022\023\n\013start_value\030\001 \001(\t\022\021\n" - "\tend_value\030\002 \001(\t\" \n\tRowRecord\022\023\n\013vector_" - "data\030\001 \003(\002\"i\n\013InsertParam\022\022\n\ntable_name\030" - "\001 \001(\t\0220\n\020row_record_array\030\002 \003(\0132\026.milvus" - ".grpc.RowRecord\022\024\n\014row_id_array\030\003 \003(\003\"I\n" - "\tVectorIds\022#\n\006status\030\001 \001(\0132\023.milvus.grpc" - ".Status\022\027\n\017vector_id_array\030\002 \003(\003\"\242\001\n\013Sea" - "rchParam\022\022\n\ntable_name\030\001 \001(\t\0222\n\022query_re" + "o\"\037\n\tTableName\022\022\n\ntable_name\030\001 \001(\t\"\'\n\rPa" + "rtitionName\022\026\n\016partition_name\030\001 \001(\t\"I\n\rT" + "ableNameList\022#\n\006status\030\001 \001(\0132\023.milvus.gr" + "pc.Status\022\023\n\013table_names\030\002 \003(\t\"\207\001\n\013Table" + "Schema\022#\n\006status\030\001 \001(\0132\023.milvus.grpc.Sta" + "tus\022\022\n\ntable_name\030\002 \001(\t\022\021\n\tdimension\030\003 \001" + "(\003\022\027\n\017index_file_size\030\004 \001(\003\022\023\n\013metric_ty" + "pe\030\005 \001(\005\"I\n\016PartitionParam\022\022\n\ntable_name" + "\030\001 \001(\t\022\026\n\016partition_name\030\002 \001(\t\022\013\n\003tag\030\003 " + "\001(\t\"j\n\rPartitionList\022#\n\006status\030\001 \001(\0132\023.m" + "ilvus.grpc.Status\0224\n\017partition_array\030\002 \003" + "(\0132\033.milvus.grpc.PartitionParam\"/\n\005Range" + "\022\023\n\013start_value\030\001 \001(\t\022\021\n\tend_value\030\002 \001(\t" + "\" \n\tRowRecord\022\023\n\013vector_data\030\001 \003(\002\"\200\001\n\013I" + "nsertParam\022\022\n\ntable_name\030\001 \001(\t\0220\n\020row_re" "cord_array\030\002 \003(\0132\026.milvus.grpc.RowRecord" - "\022-\n\021query_range_array\030\003 \003(\0132\022.milvus.grp" - "c.Range\022\014\n\004topk\030\004 \001(\003\022\016\n\006nprobe\030\005 \001(\003\"[\n" - "\022SearchInFilesParam\022\025\n\rfile_id_array\030\001 \003" - "(\t\022.\n\014search_param\030\002 \001(\0132\030.milvus.grpc.S" - "earchParam\"g\n\017TopKQueryResult\022#\n\006status\030" - "\001 \001(\0132\023.milvus.grpc.Status\022\017\n\007row_num\030\002 " - "\001(\003\022\013\n\003ids\030\003 \003(\003\022\021\n\tdistances\030\004 \003(\002\"H\n\013S" - "tringReply\022#\n\006status\030\001 \001(\0132\023.milvus.grpc" - ".Status\022\024\n\014string_reply\030\002 \001(\t\"D\n\tBoolRep" - "ly\022#\n\006status\030\001 \001(\0132\023.milvus.grpc.Status\022" - "\022\n\nbool_reply\030\002 \001(\010\"M\n\rTableRowCount\022#\n\006" - "status\030\001 \001(\0132\023.milvus.grpc.Status\022\027\n\017tab" - "le_row_count\030\002 \001(\003\"\026\n\007Command\022\013\n\003cmd\030\001 \001" - "(\t\"*\n\005Index\022\022\n\nindex_type\030\001 \001(\005\022\r\n\005nlist" - "\030\002 \001(\005\"h\n\nIndexParam\022#\n\006status\030\001 \001(\0132\023.m" - "ilvus.grpc.Status\022\022\n\ntable_name\030\002 \001(\t\022!\n" - "\005index\030\003 \001(\0132\022.milvus.grpc.Index\"K\n\022Dele" - "teByRangeParam\022!\n\005range\030\001 \001(\0132\022.milvus.g" - "rpc.Range\022\022\n\ntable_name\030\002 \001(\t2\350\007\n\rMilvus" - "Service\022>\n\013CreateTable\022\030.milvus.grpc.Tab" - "leSchema\032\023.milvus.grpc.Status\"\000\022<\n\010HasTa" - "ble\022\026.milvus.grpc.TableName\032\026.milvus.grp" - "c.BoolReply\"\000\022:\n\tDropTable\022\026.milvus.grpc" - ".TableName\032\023.milvus.grpc.Status\"\000\022=\n\013Cre" - "ateIndex\022\027.milvus.grpc.IndexParam\032\023.milv" - "us.grpc.Status\"\000\022<\n\006Insert\022\030.milvus.grpc" - ".InsertParam\032\026.milvus.grpc.VectorIds\"\000\022B" - "\n\006Search\022\030.milvus.grpc.SearchParam\032\034.mil" - "vus.grpc.TopKQueryResult\"\000\022P\n\rSearchInFi" - "les\022\037.milvus.grpc.SearchInFilesParam\032\034.m" - "ilvus.grpc.TopKQueryResult\"\000\022C\n\rDescribe" - "Table\022\026.milvus.grpc.TableName\032\030.milvus.g" - "rpc.TableSchema\"\000\022B\n\nCountTable\022\026.milvus" - ".grpc.TableName\032\032.milvus.grpc.TableRowCo" - "unt\"\000\022@\n\nShowTables\022\024.milvus.grpc.Comman" - "d\032\032.milvus.grpc.TableNameList\"\000\0227\n\003Cmd\022\024" - ".milvus.grpc.Command\032\030.milvus.grpc.Strin" - "gReply\"\000\022G\n\rDeleteByRange\022\037.milvus.grpc." - "DeleteByRangeParam\032\023.milvus.grpc.Status\"" - "\000\022=\n\014PreloadTable\022\026.milvus.grpc.TableNam" - "e\032\023.milvus.grpc.Status\"\000\022B\n\rDescribeInde" - "x\022\026.milvus.grpc.TableName\032\027.milvus.grpc." - "IndexParam\"\000\022:\n\tDropIndex\022\026.milvus.grpc." - "TableName\032\023.milvus.grpc.Status\"\000b\006proto3" + "\022\024\n\014row_id_array\030\003 \003(\003\022\025\n\rpartition_tag\030" + "\004 \001(\t\"I\n\tVectorIds\022#\n\006status\030\001 \001(\0132\023.mil" + "vus.grpc.Status\022\027\n\017vector_id_array\030\002 \003(\003" + "\"\277\001\n\013SearchParam\022\022\n\ntable_name\030\001 \001(\t\0222\n\022" + "query_record_array\030\002 \003(\0132\026.milvus.grpc.R" + "owRecord\022-\n\021query_range_array\030\003 \003(\0132\022.mi" + "lvus.grpc.Range\022\014\n\004topk\030\004 \001(\003\022\016\n\006nprobe\030" + "\005 \001(\003\022\033\n\023partition_tag_array\030\006 \003(\t\"[\n\022Se" + "archInFilesParam\022\025\n\rfile_id_array\030\001 \003(\t\022" + ".\n\014search_param\030\002 \001(\0132\030.milvus.grpc.Sear" + "chParam\"g\n\017TopKQueryResult\022#\n\006status\030\001 \001" + "(\0132\023.milvus.grpc.Status\022\017\n\007row_num\030\002 \001(\003" + "\022\013\n\003ids\030\003 \003(\003\022\021\n\tdistances\030\004 \003(\002\"H\n\013Stri" + "ngReply\022#\n\006status\030\001 \001(\0132\023.milvus.grpc.St" + "atus\022\024\n\014string_reply\030\002 \001(\t\"D\n\tBoolReply\022" + "#\n\006status\030\001 \001(\0132\023.milvus.grpc.Status\022\022\n\n" + "bool_reply\030\002 \001(\010\"M\n\rTableRowCount\022#\n\006sta" + "tus\030\001 \001(\0132\023.milvus.grpc.Status\022\027\n\017table_" + "row_count\030\002 \001(\003\"\026\n\007Command\022\013\n\003cmd\030\001 \001(\t\"" + "*\n\005Index\022\022\n\nindex_type\030\001 \001(\005\022\r\n\005nlist\030\002 " + "\001(\005\"h\n\nIndexParam\022#\n\006status\030\001 \001(\0132\023.milv" + "us.grpc.Status\022\022\n\ntable_name\030\002 \001(\t\022!\n\005in" + "dex\030\003 \001(\0132\022.milvus.grpc.Index\"J\n\021DeleteB" + "yDateParam\022!\n\005range\030\001 \001(\0132\022.milvus.grpc." + "Range\022\022\n\ntable_name\030\002 \001(\t2\272\t\n\rMilvusServ" + "ice\022>\n\013CreateTable\022\030.milvus.grpc.TableSc" + "hema\032\023.milvus.grpc.Status\"\000\022<\n\010HasTable\022" + "\026.milvus.grpc.TableName\032\026.milvus.grpc.Bo" + "olReply\"\000\022C\n\rDescribeTable\022\026.milvus.grpc" + ".TableName\032\030.milvus.grpc.TableSchema\"\000\022B" + "\n\nCountTable\022\026.milvus.grpc.TableName\032\032.m" + "ilvus.grpc.TableRowCount\"\000\022@\n\nShowTables" + "\022\024.milvus.grpc.Command\032\032.milvus.grpc.Tab" + "leNameList\"\000\022:\n\tDropTable\022\026.milvus.grpc." + "TableName\032\023.milvus.grpc.Status\"\000\022=\n\013Crea" + "teIndex\022\027.milvus.grpc.IndexParam\032\023.milvu" + "s.grpc.Status\"\000\022B\n\rDescribeIndex\022\026.milvu" + "s.grpc.TableName\032\027.milvus.grpc.IndexPara" + "m\"\000\022:\n\tDropIndex\022\026.milvus.grpc.TableName" + "\032\023.milvus.grpc.Status\"\000\022E\n\017CreatePartiti" + "on\022\033.milvus.grpc.PartitionParam\032\023.milvus" + ".grpc.Status\"\000\022F\n\016ShowPartitions\022\026.milvu" + "s.grpc.TableName\032\032.milvus.grpc.Partition" + "List\"\000\022C\n\rDropPartition\022\033.milvus.grpc.Pa" + "rtitionParam\032\023.milvus.grpc.Status\"\000\022<\n\006I" + "nsert\022\030.milvus.grpc.InsertParam\032\026.milvus" + ".grpc.VectorIds\"\000\022B\n\006Search\022\030.milvus.grp" + "c.SearchParam\032\034.milvus.grpc.TopKQueryRes" + "ult\"\000\022P\n\rSearchInFiles\022\037.milvus.grpc.Sea" + "rchInFilesParam\032\034.milvus.grpc.TopKQueryR" + "esult\"\000\0227\n\003Cmd\022\024.milvus.grpc.Command\032\030.m" + "ilvus.grpc.StringReply\"\000\022E\n\014DeleteByDate" + "\022\036.milvus.grpc.DeleteByDateParam\032\023.milvu" + "s.grpc.Status\"\000\022=\n\014PreloadTable\022\026.milvus" + ".grpc.TableName\032\023.milvus.grpc.Status\"\000b\006" + "proto3" ; static const ::PROTOBUF_NAMESPACE_ID::internal::DescriptorTable*const descriptor_table_milvus_2eproto_deps[1] = { &::descriptor_table_status_2eproto, }; -static ::PROTOBUF_NAMESPACE_ID::internal::SCCInfoBase*const descriptor_table_milvus_2eproto_sccs[17] = { +static ::PROTOBUF_NAMESPACE_ID::internal::SCCInfoBase*const descriptor_table_milvus_2eproto_sccs[20] = { &scc_info_BoolReply_milvus_2eproto.base, &scc_info_Command_milvus_2eproto.base, - &scc_info_DeleteByRangeParam_milvus_2eproto.base, + &scc_info_DeleteByDateParam_milvus_2eproto.base, &scc_info_Index_milvus_2eproto.base, &scc_info_IndexParam_milvus_2eproto.base, &scc_info_InsertParam_milvus_2eproto.base, + &scc_info_PartitionList_milvus_2eproto.base, + &scc_info_PartitionName_milvus_2eproto.base, + &scc_info_PartitionParam_milvus_2eproto.base, &scc_info_Range_milvus_2eproto.base, &scc_info_RowRecord_milvus_2eproto.base, &scc_info_SearchInFilesParam_milvus_2eproto.base, @@ -603,10 +705,10 @@ static ::PROTOBUF_NAMESPACE_ID::internal::SCCInfoBase*const descriptor_table_mil static ::PROTOBUF_NAMESPACE_ID::internal::once_flag descriptor_table_milvus_2eproto_once; static bool descriptor_table_milvus_2eproto_initialized = false; const ::PROTOBUF_NAMESPACE_ID::internal::DescriptorTable descriptor_table_milvus_2eproto = { - &descriptor_table_milvus_2eproto_initialized, descriptor_table_protodef_milvus_2eproto, "milvus.proto", 2400, - &descriptor_table_milvus_2eproto_once, descriptor_table_milvus_2eproto_sccs, descriptor_table_milvus_2eproto_deps, 17, 1, + &descriptor_table_milvus_2eproto_initialized, descriptor_table_protodef_milvus_2eproto, "milvus.proto", 2886, + &descriptor_table_milvus_2eproto_once, descriptor_table_milvus_2eproto_sccs, descriptor_table_milvus_2eproto_deps, 20, 1, schemas, file_default_instances, TableStruct_milvus_2eproto::offsets, - file_level_metadata_milvus_2eproto, 17, file_level_enum_descriptors_milvus_2eproto, file_level_service_descriptors_milvus_2eproto, + file_level_metadata_milvus_2eproto, 20, file_level_enum_descriptors_milvus_2eproto, file_level_service_descriptors_milvus_2eproto, }; // Force running AddDescriptors() at dynamic initialization time. @@ -883,6 +985,275 @@ void TableName::InternalSwap(TableName* other) { } +// =================================================================== + +void PartitionName::InitAsDefaultInstance() { +} +class PartitionName::_Internal { + public: +}; + +PartitionName::PartitionName() + : ::PROTOBUF_NAMESPACE_ID::Message(), _internal_metadata_(nullptr) { + SharedCtor(); + // @@protoc_insertion_point(constructor:milvus.grpc.PartitionName) +} +PartitionName::PartitionName(const PartitionName& from) + : ::PROTOBUF_NAMESPACE_ID::Message(), + _internal_metadata_(nullptr) { + _internal_metadata_.MergeFrom(from._internal_metadata_); + partition_name_.UnsafeSetDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); + if (!from.partition_name().empty()) { + partition_name_.AssignWithDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), from.partition_name_); + } + // @@protoc_insertion_point(copy_constructor:milvus.grpc.PartitionName) +} + +void PartitionName::SharedCtor() { + ::PROTOBUF_NAMESPACE_ID::internal::InitSCC(&scc_info_PartitionName_milvus_2eproto.base); + partition_name_.UnsafeSetDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); +} + +PartitionName::~PartitionName() { + // @@protoc_insertion_point(destructor:milvus.grpc.PartitionName) + SharedDtor(); +} + +void PartitionName::SharedDtor() { + partition_name_.DestroyNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); +} + +void PartitionName::SetCachedSize(int size) const { + _cached_size_.Set(size); +} +const PartitionName& PartitionName::default_instance() { + ::PROTOBUF_NAMESPACE_ID::internal::InitSCC(&::scc_info_PartitionName_milvus_2eproto.base); + return *internal_default_instance(); +} + + +void PartitionName::Clear() { +// @@protoc_insertion_point(message_clear_start:milvus.grpc.PartitionName) + ::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0; + // Prevent compiler warnings about cached_has_bits being unused + (void) cached_has_bits; + + partition_name_.ClearToEmptyNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); + _internal_metadata_.Clear(); +} + +#if GOOGLE_PROTOBUF_ENABLE_EXPERIMENTAL_PARSER +const char* PartitionName::_InternalParse(const char* ptr, ::PROTOBUF_NAMESPACE_ID::internal::ParseContext* ctx) { +#define CHK_(x) if (PROTOBUF_PREDICT_FALSE(!(x))) goto failure + while (!ctx->Done(&ptr)) { + ::PROTOBUF_NAMESPACE_ID::uint32 tag; + ptr = ::PROTOBUF_NAMESPACE_ID::internal::ReadTag(ptr, &tag); + CHK_(ptr); + switch (tag >> 3) { + // string partition_name = 1; + case 1: + if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 10)) { + ptr = ::PROTOBUF_NAMESPACE_ID::internal::InlineGreedyStringParserUTF8(mutable_partition_name(), ptr, ctx, "milvus.grpc.PartitionName.partition_name"); + CHK_(ptr); + } else goto handle_unusual; + continue; + default: { + handle_unusual: + if ((tag & 7) == 4 || tag == 0) { + ctx->SetLastTag(tag); + goto success; + } + ptr = UnknownFieldParse(tag, &_internal_metadata_, ptr, ctx); + CHK_(ptr != nullptr); + continue; + } + } // switch + } // while +success: + return ptr; +failure: + ptr = nullptr; + goto success; +#undef CHK_ +} +#else // GOOGLE_PROTOBUF_ENABLE_EXPERIMENTAL_PARSER +bool PartitionName::MergePartialFromCodedStream( + ::PROTOBUF_NAMESPACE_ID::io::CodedInputStream* input) { +#define DO_(EXPRESSION) if (!PROTOBUF_PREDICT_TRUE(EXPRESSION)) goto failure + ::PROTOBUF_NAMESPACE_ID::uint32 tag; + // @@protoc_insertion_point(parse_start:milvus.grpc.PartitionName) + for (;;) { + ::std::pair<::PROTOBUF_NAMESPACE_ID::uint32, bool> p = input->ReadTagWithCutoffNoLastTag(127u); + tag = p.first; + if (!p.second) goto handle_unusual; + switch (::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::GetTagFieldNumber(tag)) { + // string partition_name = 1; + case 1: { + if (static_cast< ::PROTOBUF_NAMESPACE_ID::uint8>(tag) == (10 & 0xFF)) { + DO_(::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::ReadString( + input, this->mutable_partition_name())); + DO_(::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::VerifyUtf8String( + this->partition_name().data(), static_cast(this->partition_name().length()), + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::PARSE, + "milvus.grpc.PartitionName.partition_name")); + } else { + goto handle_unusual; + } + break; + } + + default: { + handle_unusual: + if (tag == 0) { + goto success; + } + DO_(::PROTOBUF_NAMESPACE_ID::internal::WireFormat::SkipField( + input, tag, _internal_metadata_.mutable_unknown_fields())); + break; + } + } + } +success: + // @@protoc_insertion_point(parse_success:milvus.grpc.PartitionName) + return true; +failure: + // @@protoc_insertion_point(parse_failure:milvus.grpc.PartitionName) + return false; +#undef DO_ +} +#endif // GOOGLE_PROTOBUF_ENABLE_EXPERIMENTAL_PARSER + +void PartitionName::SerializeWithCachedSizes( + ::PROTOBUF_NAMESPACE_ID::io::CodedOutputStream* output) const { + // @@protoc_insertion_point(serialize_start:milvus.grpc.PartitionName) + ::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0; + (void) cached_has_bits; + + // string partition_name = 1; + if (this->partition_name().size() > 0) { + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::VerifyUtf8String( + this->partition_name().data(), static_cast(this->partition_name().length()), + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::SERIALIZE, + "milvus.grpc.PartitionName.partition_name"); + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteStringMaybeAliased( + 1, this->partition_name(), output); + } + + if (_internal_metadata_.have_unknown_fields()) { + ::PROTOBUF_NAMESPACE_ID::internal::WireFormat::SerializeUnknownFields( + _internal_metadata_.unknown_fields(), output); + } + // @@protoc_insertion_point(serialize_end:milvus.grpc.PartitionName) +} + +::PROTOBUF_NAMESPACE_ID::uint8* PartitionName::InternalSerializeWithCachedSizesToArray( + ::PROTOBUF_NAMESPACE_ID::uint8* target) const { + // @@protoc_insertion_point(serialize_to_array_start:milvus.grpc.PartitionName) + ::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0; + (void) cached_has_bits; + + // string partition_name = 1; + if (this->partition_name().size() > 0) { + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::VerifyUtf8String( + this->partition_name().data(), static_cast(this->partition_name().length()), + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::SERIALIZE, + "milvus.grpc.PartitionName.partition_name"); + target = + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteStringToArray( + 1, this->partition_name(), target); + } + + if (_internal_metadata_.have_unknown_fields()) { + target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormat::SerializeUnknownFieldsToArray( + _internal_metadata_.unknown_fields(), target); + } + // @@protoc_insertion_point(serialize_to_array_end:milvus.grpc.PartitionName) + return target; +} + +size_t PartitionName::ByteSizeLong() const { +// @@protoc_insertion_point(message_byte_size_start:milvus.grpc.PartitionName) + size_t total_size = 0; + + if (_internal_metadata_.have_unknown_fields()) { + total_size += + ::PROTOBUF_NAMESPACE_ID::internal::WireFormat::ComputeUnknownFieldsSize( + _internal_metadata_.unknown_fields()); + } + ::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0; + // Prevent compiler warnings about cached_has_bits being unused + (void) cached_has_bits; + + // string partition_name = 1; + if (this->partition_name().size() > 0) { + total_size += 1 + + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::StringSize( + this->partition_name()); + } + + int cached_size = ::PROTOBUF_NAMESPACE_ID::internal::ToCachedSize(total_size); + SetCachedSize(cached_size); + return total_size; +} + +void PartitionName::MergeFrom(const ::PROTOBUF_NAMESPACE_ID::Message& from) { +// @@protoc_insertion_point(generalized_merge_from_start:milvus.grpc.PartitionName) + GOOGLE_DCHECK_NE(&from, this); + const PartitionName* source = + ::PROTOBUF_NAMESPACE_ID::DynamicCastToGenerated( + &from); + if (source == nullptr) { + // @@protoc_insertion_point(generalized_merge_from_cast_fail:milvus.grpc.PartitionName) + ::PROTOBUF_NAMESPACE_ID::internal::ReflectionOps::Merge(from, this); + } else { + // @@protoc_insertion_point(generalized_merge_from_cast_success:milvus.grpc.PartitionName) + MergeFrom(*source); + } +} + +void PartitionName::MergeFrom(const PartitionName& from) { +// @@protoc_insertion_point(class_specific_merge_from_start:milvus.grpc.PartitionName) + GOOGLE_DCHECK_NE(&from, this); + _internal_metadata_.MergeFrom(from._internal_metadata_); + ::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0; + (void) cached_has_bits; + + if (from.partition_name().size() > 0) { + + partition_name_.AssignWithDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), from.partition_name_); + } +} + +void PartitionName::CopyFrom(const ::PROTOBUF_NAMESPACE_ID::Message& from) { +// @@protoc_insertion_point(generalized_copy_from_start:milvus.grpc.PartitionName) + if (&from == this) return; + Clear(); + MergeFrom(from); +} + +void PartitionName::CopyFrom(const PartitionName& from) { +// @@protoc_insertion_point(class_specific_copy_from_start:milvus.grpc.PartitionName) + if (&from == this) return; + Clear(); + MergeFrom(from); +} + +bool PartitionName::IsInitialized() const { + return true; +} + +void PartitionName::InternalSwap(PartitionName* other) { + using std::swap; + _internal_metadata_.Swap(&other->_internal_metadata_); + partition_name_.Swap(&other->partition_name_, &::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), + GetArenaNoVirtual()); +} + +::PROTOBUF_NAMESPACE_ID::Metadata PartitionName::GetMetadata() const { + return GetMetadataStatic(); +} + + // =================================================================== void TableNameList::InitAsDefaultInstance() { @@ -1681,6 +2052,728 @@ void TableSchema::InternalSwap(TableSchema* other) { } +// =================================================================== + +void PartitionParam::InitAsDefaultInstance() { +} +class PartitionParam::_Internal { + public: +}; + +PartitionParam::PartitionParam() + : ::PROTOBUF_NAMESPACE_ID::Message(), _internal_metadata_(nullptr) { + SharedCtor(); + // @@protoc_insertion_point(constructor:milvus.grpc.PartitionParam) +} +PartitionParam::PartitionParam(const PartitionParam& from) + : ::PROTOBUF_NAMESPACE_ID::Message(), + _internal_metadata_(nullptr) { + _internal_metadata_.MergeFrom(from._internal_metadata_); + table_name_.UnsafeSetDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); + if (!from.table_name().empty()) { + table_name_.AssignWithDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), from.table_name_); + } + partition_name_.UnsafeSetDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); + if (!from.partition_name().empty()) { + partition_name_.AssignWithDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), from.partition_name_); + } + tag_.UnsafeSetDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); + if (!from.tag().empty()) { + tag_.AssignWithDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), from.tag_); + } + // @@protoc_insertion_point(copy_constructor:milvus.grpc.PartitionParam) +} + +void PartitionParam::SharedCtor() { + ::PROTOBUF_NAMESPACE_ID::internal::InitSCC(&scc_info_PartitionParam_milvus_2eproto.base); + table_name_.UnsafeSetDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); + partition_name_.UnsafeSetDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); + tag_.UnsafeSetDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); +} + +PartitionParam::~PartitionParam() { + // @@protoc_insertion_point(destructor:milvus.grpc.PartitionParam) + SharedDtor(); +} + +void PartitionParam::SharedDtor() { + table_name_.DestroyNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); + partition_name_.DestroyNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); + tag_.DestroyNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); +} + +void PartitionParam::SetCachedSize(int size) const { + _cached_size_.Set(size); +} +const PartitionParam& PartitionParam::default_instance() { + ::PROTOBUF_NAMESPACE_ID::internal::InitSCC(&::scc_info_PartitionParam_milvus_2eproto.base); + return *internal_default_instance(); +} + + +void PartitionParam::Clear() { +// @@protoc_insertion_point(message_clear_start:milvus.grpc.PartitionParam) + ::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0; + // Prevent compiler warnings about cached_has_bits being unused + (void) cached_has_bits; + + table_name_.ClearToEmptyNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); + partition_name_.ClearToEmptyNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); + tag_.ClearToEmptyNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); + _internal_metadata_.Clear(); +} + +#if GOOGLE_PROTOBUF_ENABLE_EXPERIMENTAL_PARSER +const char* PartitionParam::_InternalParse(const char* ptr, ::PROTOBUF_NAMESPACE_ID::internal::ParseContext* ctx) { +#define CHK_(x) if (PROTOBUF_PREDICT_FALSE(!(x))) goto failure + while (!ctx->Done(&ptr)) { + ::PROTOBUF_NAMESPACE_ID::uint32 tag; + ptr = ::PROTOBUF_NAMESPACE_ID::internal::ReadTag(ptr, &tag); + CHK_(ptr); + switch (tag >> 3) { + // string table_name = 1; + case 1: + if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 10)) { + ptr = ::PROTOBUF_NAMESPACE_ID::internal::InlineGreedyStringParserUTF8(mutable_table_name(), ptr, ctx, "milvus.grpc.PartitionParam.table_name"); + CHK_(ptr); + } else goto handle_unusual; + continue; + // string partition_name = 2; + case 2: + if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 18)) { + ptr = ::PROTOBUF_NAMESPACE_ID::internal::InlineGreedyStringParserUTF8(mutable_partition_name(), ptr, ctx, "milvus.grpc.PartitionParam.partition_name"); + CHK_(ptr); + } else goto handle_unusual; + continue; + // string tag = 3; + case 3: + if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 26)) { + ptr = ::PROTOBUF_NAMESPACE_ID::internal::InlineGreedyStringParserUTF8(mutable_tag(), ptr, ctx, "milvus.grpc.PartitionParam.tag"); + CHK_(ptr); + } else goto handle_unusual; + continue; + default: { + handle_unusual: + if ((tag & 7) == 4 || tag == 0) { + ctx->SetLastTag(tag); + goto success; + } + ptr = UnknownFieldParse(tag, &_internal_metadata_, ptr, ctx); + CHK_(ptr != nullptr); + continue; + } + } // switch + } // while +success: + return ptr; +failure: + ptr = nullptr; + goto success; +#undef CHK_ +} +#else // GOOGLE_PROTOBUF_ENABLE_EXPERIMENTAL_PARSER +bool PartitionParam::MergePartialFromCodedStream( + ::PROTOBUF_NAMESPACE_ID::io::CodedInputStream* input) { +#define DO_(EXPRESSION) if (!PROTOBUF_PREDICT_TRUE(EXPRESSION)) goto failure + ::PROTOBUF_NAMESPACE_ID::uint32 tag; + // @@protoc_insertion_point(parse_start:milvus.grpc.PartitionParam) + for (;;) { + ::std::pair<::PROTOBUF_NAMESPACE_ID::uint32, bool> p = input->ReadTagWithCutoffNoLastTag(127u); + tag = p.first; + if (!p.second) goto handle_unusual; + switch (::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::GetTagFieldNumber(tag)) { + // string table_name = 1; + case 1: { + if (static_cast< ::PROTOBUF_NAMESPACE_ID::uint8>(tag) == (10 & 0xFF)) { + DO_(::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::ReadString( + input, this->mutable_table_name())); + DO_(::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::VerifyUtf8String( + this->table_name().data(), static_cast(this->table_name().length()), + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::PARSE, + "milvus.grpc.PartitionParam.table_name")); + } else { + goto handle_unusual; + } + break; + } + + // string partition_name = 2; + case 2: { + if (static_cast< ::PROTOBUF_NAMESPACE_ID::uint8>(tag) == (18 & 0xFF)) { + DO_(::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::ReadString( + input, this->mutable_partition_name())); + DO_(::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::VerifyUtf8String( + this->partition_name().data(), static_cast(this->partition_name().length()), + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::PARSE, + "milvus.grpc.PartitionParam.partition_name")); + } else { + goto handle_unusual; + } + break; + } + + // string tag = 3; + case 3: { + if (static_cast< ::PROTOBUF_NAMESPACE_ID::uint8>(tag) == (26 & 0xFF)) { + DO_(::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::ReadString( + input, this->mutable_tag())); + DO_(::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::VerifyUtf8String( + this->tag().data(), static_cast(this->tag().length()), + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::PARSE, + "milvus.grpc.PartitionParam.tag")); + } else { + goto handle_unusual; + } + break; + } + + default: { + handle_unusual: + if (tag == 0) { + goto success; + } + DO_(::PROTOBUF_NAMESPACE_ID::internal::WireFormat::SkipField( + input, tag, _internal_metadata_.mutable_unknown_fields())); + break; + } + } + } +success: + // @@protoc_insertion_point(parse_success:milvus.grpc.PartitionParam) + return true; +failure: + // @@protoc_insertion_point(parse_failure:milvus.grpc.PartitionParam) + return false; +#undef DO_ +} +#endif // GOOGLE_PROTOBUF_ENABLE_EXPERIMENTAL_PARSER + +void PartitionParam::SerializeWithCachedSizes( + ::PROTOBUF_NAMESPACE_ID::io::CodedOutputStream* output) const { + // @@protoc_insertion_point(serialize_start:milvus.grpc.PartitionParam) + ::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0; + (void) cached_has_bits; + + // string table_name = 1; + if (this->table_name().size() > 0) { + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::VerifyUtf8String( + this->table_name().data(), static_cast(this->table_name().length()), + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::SERIALIZE, + "milvus.grpc.PartitionParam.table_name"); + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteStringMaybeAliased( + 1, this->table_name(), output); + } + + // string partition_name = 2; + if (this->partition_name().size() > 0) { + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::VerifyUtf8String( + this->partition_name().data(), static_cast(this->partition_name().length()), + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::SERIALIZE, + "milvus.grpc.PartitionParam.partition_name"); + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteStringMaybeAliased( + 2, this->partition_name(), output); + } + + // string tag = 3; + if (this->tag().size() > 0) { + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::VerifyUtf8String( + this->tag().data(), static_cast(this->tag().length()), + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::SERIALIZE, + "milvus.grpc.PartitionParam.tag"); + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteStringMaybeAliased( + 3, this->tag(), output); + } + + if (_internal_metadata_.have_unknown_fields()) { + ::PROTOBUF_NAMESPACE_ID::internal::WireFormat::SerializeUnknownFields( + _internal_metadata_.unknown_fields(), output); + } + // @@protoc_insertion_point(serialize_end:milvus.grpc.PartitionParam) +} + +::PROTOBUF_NAMESPACE_ID::uint8* PartitionParam::InternalSerializeWithCachedSizesToArray( + ::PROTOBUF_NAMESPACE_ID::uint8* target) const { + // @@protoc_insertion_point(serialize_to_array_start:milvus.grpc.PartitionParam) + ::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0; + (void) cached_has_bits; + + // string table_name = 1; + if (this->table_name().size() > 0) { + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::VerifyUtf8String( + this->table_name().data(), static_cast(this->table_name().length()), + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::SERIALIZE, + "milvus.grpc.PartitionParam.table_name"); + target = + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteStringToArray( + 1, this->table_name(), target); + } + + // string partition_name = 2; + if (this->partition_name().size() > 0) { + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::VerifyUtf8String( + this->partition_name().data(), static_cast(this->partition_name().length()), + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::SERIALIZE, + "milvus.grpc.PartitionParam.partition_name"); + target = + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteStringToArray( + 2, this->partition_name(), target); + } + + // string tag = 3; + if (this->tag().size() > 0) { + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::VerifyUtf8String( + this->tag().data(), static_cast(this->tag().length()), + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::SERIALIZE, + "milvus.grpc.PartitionParam.tag"); + target = + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteStringToArray( + 3, this->tag(), target); + } + + if (_internal_metadata_.have_unknown_fields()) { + target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormat::SerializeUnknownFieldsToArray( + _internal_metadata_.unknown_fields(), target); + } + // @@protoc_insertion_point(serialize_to_array_end:milvus.grpc.PartitionParam) + return target; +} + +size_t PartitionParam::ByteSizeLong() const { +// @@protoc_insertion_point(message_byte_size_start:milvus.grpc.PartitionParam) + size_t total_size = 0; + + if (_internal_metadata_.have_unknown_fields()) { + total_size += + ::PROTOBUF_NAMESPACE_ID::internal::WireFormat::ComputeUnknownFieldsSize( + _internal_metadata_.unknown_fields()); + } + ::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0; + // Prevent compiler warnings about cached_has_bits being unused + (void) cached_has_bits; + + // string table_name = 1; + if (this->table_name().size() > 0) { + total_size += 1 + + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::StringSize( + this->table_name()); + } + + // string partition_name = 2; + if (this->partition_name().size() > 0) { + total_size += 1 + + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::StringSize( + this->partition_name()); + } + + // string tag = 3; + if (this->tag().size() > 0) { + total_size += 1 + + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::StringSize( + this->tag()); + } + + int cached_size = ::PROTOBUF_NAMESPACE_ID::internal::ToCachedSize(total_size); + SetCachedSize(cached_size); + return total_size; +} + +void PartitionParam::MergeFrom(const ::PROTOBUF_NAMESPACE_ID::Message& from) { +// @@protoc_insertion_point(generalized_merge_from_start:milvus.grpc.PartitionParam) + GOOGLE_DCHECK_NE(&from, this); + const PartitionParam* source = + ::PROTOBUF_NAMESPACE_ID::DynamicCastToGenerated( + &from); + if (source == nullptr) { + // @@protoc_insertion_point(generalized_merge_from_cast_fail:milvus.grpc.PartitionParam) + ::PROTOBUF_NAMESPACE_ID::internal::ReflectionOps::Merge(from, this); + } else { + // @@protoc_insertion_point(generalized_merge_from_cast_success:milvus.grpc.PartitionParam) + MergeFrom(*source); + } +} + +void PartitionParam::MergeFrom(const PartitionParam& from) { +// @@protoc_insertion_point(class_specific_merge_from_start:milvus.grpc.PartitionParam) + GOOGLE_DCHECK_NE(&from, this); + _internal_metadata_.MergeFrom(from._internal_metadata_); + ::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0; + (void) cached_has_bits; + + if (from.table_name().size() > 0) { + + table_name_.AssignWithDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), from.table_name_); + } + if (from.partition_name().size() > 0) { + + partition_name_.AssignWithDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), from.partition_name_); + } + if (from.tag().size() > 0) { + + tag_.AssignWithDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), from.tag_); + } +} + +void PartitionParam::CopyFrom(const ::PROTOBUF_NAMESPACE_ID::Message& from) { +// @@protoc_insertion_point(generalized_copy_from_start:milvus.grpc.PartitionParam) + if (&from == this) return; + Clear(); + MergeFrom(from); +} + +void PartitionParam::CopyFrom(const PartitionParam& from) { +// @@protoc_insertion_point(class_specific_copy_from_start:milvus.grpc.PartitionParam) + if (&from == this) return; + Clear(); + MergeFrom(from); +} + +bool PartitionParam::IsInitialized() const { + return true; +} + +void PartitionParam::InternalSwap(PartitionParam* other) { + using std::swap; + _internal_metadata_.Swap(&other->_internal_metadata_); + table_name_.Swap(&other->table_name_, &::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), + GetArenaNoVirtual()); + partition_name_.Swap(&other->partition_name_, &::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), + GetArenaNoVirtual()); + tag_.Swap(&other->tag_, &::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), + GetArenaNoVirtual()); +} + +::PROTOBUF_NAMESPACE_ID::Metadata PartitionParam::GetMetadata() const { + return GetMetadataStatic(); +} + + +// =================================================================== + +void PartitionList::InitAsDefaultInstance() { + ::milvus::grpc::_PartitionList_default_instance_._instance.get_mutable()->status_ = const_cast< ::milvus::grpc::Status*>( + ::milvus::grpc::Status::internal_default_instance()); +} +class PartitionList::_Internal { + public: + static const ::milvus::grpc::Status& status(const PartitionList* msg); +}; + +const ::milvus::grpc::Status& +PartitionList::_Internal::status(const PartitionList* msg) { + return *msg->status_; +} +void PartitionList::clear_status() { + if (GetArenaNoVirtual() == nullptr && status_ != nullptr) { + delete status_; + } + status_ = nullptr; +} +PartitionList::PartitionList() + : ::PROTOBUF_NAMESPACE_ID::Message(), _internal_metadata_(nullptr) { + SharedCtor(); + // @@protoc_insertion_point(constructor:milvus.grpc.PartitionList) +} +PartitionList::PartitionList(const PartitionList& from) + : ::PROTOBUF_NAMESPACE_ID::Message(), + _internal_metadata_(nullptr), + partition_array_(from.partition_array_) { + _internal_metadata_.MergeFrom(from._internal_metadata_); + if (from.has_status()) { + status_ = new ::milvus::grpc::Status(*from.status_); + } else { + status_ = nullptr; + } + // @@protoc_insertion_point(copy_constructor:milvus.grpc.PartitionList) +} + +void PartitionList::SharedCtor() { + ::PROTOBUF_NAMESPACE_ID::internal::InitSCC(&scc_info_PartitionList_milvus_2eproto.base); + status_ = nullptr; +} + +PartitionList::~PartitionList() { + // @@protoc_insertion_point(destructor:milvus.grpc.PartitionList) + SharedDtor(); +} + +void PartitionList::SharedDtor() { + if (this != internal_default_instance()) delete status_; +} + +void PartitionList::SetCachedSize(int size) const { + _cached_size_.Set(size); +} +const PartitionList& PartitionList::default_instance() { + ::PROTOBUF_NAMESPACE_ID::internal::InitSCC(&::scc_info_PartitionList_milvus_2eproto.base); + return *internal_default_instance(); +} + + +void PartitionList::Clear() { +// @@protoc_insertion_point(message_clear_start:milvus.grpc.PartitionList) + ::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0; + // Prevent compiler warnings about cached_has_bits being unused + (void) cached_has_bits; + + partition_array_.Clear(); + if (GetArenaNoVirtual() == nullptr && status_ != nullptr) { + delete status_; + } + status_ = nullptr; + _internal_metadata_.Clear(); +} + +#if GOOGLE_PROTOBUF_ENABLE_EXPERIMENTAL_PARSER +const char* PartitionList::_InternalParse(const char* ptr, ::PROTOBUF_NAMESPACE_ID::internal::ParseContext* ctx) { +#define CHK_(x) if (PROTOBUF_PREDICT_FALSE(!(x))) goto failure + while (!ctx->Done(&ptr)) { + ::PROTOBUF_NAMESPACE_ID::uint32 tag; + ptr = ::PROTOBUF_NAMESPACE_ID::internal::ReadTag(ptr, &tag); + CHK_(ptr); + switch (tag >> 3) { + // .milvus.grpc.Status status = 1; + case 1: + if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 10)) { + ptr = ctx->ParseMessage(mutable_status(), ptr); + CHK_(ptr); + } else goto handle_unusual; + continue; + // repeated .milvus.grpc.PartitionParam partition_array = 2; + case 2: + if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 18)) { + ptr -= 1; + do { + ptr += 1; + ptr = ctx->ParseMessage(add_partition_array(), ptr); + CHK_(ptr); + if (!ctx->DataAvailable(ptr)) break; + } while (::PROTOBUF_NAMESPACE_ID::internal::UnalignedLoad<::PROTOBUF_NAMESPACE_ID::uint8>(ptr) == 18); + } else goto handle_unusual; + continue; + default: { + handle_unusual: + if ((tag & 7) == 4 || tag == 0) { + ctx->SetLastTag(tag); + goto success; + } + ptr = UnknownFieldParse(tag, &_internal_metadata_, ptr, ctx); + CHK_(ptr != nullptr); + continue; + } + } // switch + } // while +success: + return ptr; +failure: + ptr = nullptr; + goto success; +#undef CHK_ +} +#else // GOOGLE_PROTOBUF_ENABLE_EXPERIMENTAL_PARSER +bool PartitionList::MergePartialFromCodedStream( + ::PROTOBUF_NAMESPACE_ID::io::CodedInputStream* input) { +#define DO_(EXPRESSION) if (!PROTOBUF_PREDICT_TRUE(EXPRESSION)) goto failure + ::PROTOBUF_NAMESPACE_ID::uint32 tag; + // @@protoc_insertion_point(parse_start:milvus.grpc.PartitionList) + for (;;) { + ::std::pair<::PROTOBUF_NAMESPACE_ID::uint32, bool> p = input->ReadTagWithCutoffNoLastTag(127u); + tag = p.first; + if (!p.second) goto handle_unusual; + switch (::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::GetTagFieldNumber(tag)) { + // .milvus.grpc.Status status = 1; + case 1: { + if (static_cast< ::PROTOBUF_NAMESPACE_ID::uint8>(tag) == (10 & 0xFF)) { + DO_(::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::ReadMessage( + input, mutable_status())); + } else { + goto handle_unusual; + } + break; + } + + // repeated .milvus.grpc.PartitionParam partition_array = 2; + case 2: { + if (static_cast< ::PROTOBUF_NAMESPACE_ID::uint8>(tag) == (18 & 0xFF)) { + DO_(::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::ReadMessage( + input, add_partition_array())); + } else { + goto handle_unusual; + } + break; + } + + default: { + handle_unusual: + if (tag == 0) { + goto success; + } + DO_(::PROTOBUF_NAMESPACE_ID::internal::WireFormat::SkipField( + input, tag, _internal_metadata_.mutable_unknown_fields())); + break; + } + } + } +success: + // @@protoc_insertion_point(parse_success:milvus.grpc.PartitionList) + return true; +failure: + // @@protoc_insertion_point(parse_failure:milvus.grpc.PartitionList) + return false; +#undef DO_ +} +#endif // GOOGLE_PROTOBUF_ENABLE_EXPERIMENTAL_PARSER + +void PartitionList::SerializeWithCachedSizes( + ::PROTOBUF_NAMESPACE_ID::io::CodedOutputStream* output) const { + // @@protoc_insertion_point(serialize_start:milvus.grpc.PartitionList) + ::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0; + (void) cached_has_bits; + + // .milvus.grpc.Status status = 1; + if (this->has_status()) { + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteMessageMaybeToArray( + 1, _Internal::status(this), output); + } + + // repeated .milvus.grpc.PartitionParam partition_array = 2; + for (unsigned int i = 0, + n = static_cast(this->partition_array_size()); i < n; i++) { + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteMessageMaybeToArray( + 2, + this->partition_array(static_cast(i)), + output); + } + + if (_internal_metadata_.have_unknown_fields()) { + ::PROTOBUF_NAMESPACE_ID::internal::WireFormat::SerializeUnknownFields( + _internal_metadata_.unknown_fields(), output); + } + // @@protoc_insertion_point(serialize_end:milvus.grpc.PartitionList) +} + +::PROTOBUF_NAMESPACE_ID::uint8* PartitionList::InternalSerializeWithCachedSizesToArray( + ::PROTOBUF_NAMESPACE_ID::uint8* target) const { + // @@protoc_insertion_point(serialize_to_array_start:milvus.grpc.PartitionList) + ::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0; + (void) cached_has_bits; + + // .milvus.grpc.Status status = 1; + if (this->has_status()) { + target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite:: + InternalWriteMessageToArray( + 1, _Internal::status(this), target); + } + + // repeated .milvus.grpc.PartitionParam partition_array = 2; + for (unsigned int i = 0, + n = static_cast(this->partition_array_size()); i < n; i++) { + target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite:: + InternalWriteMessageToArray( + 2, this->partition_array(static_cast(i)), target); + } + + if (_internal_metadata_.have_unknown_fields()) { + target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormat::SerializeUnknownFieldsToArray( + _internal_metadata_.unknown_fields(), target); + } + // @@protoc_insertion_point(serialize_to_array_end:milvus.grpc.PartitionList) + return target; +} + +size_t PartitionList::ByteSizeLong() const { +// @@protoc_insertion_point(message_byte_size_start:milvus.grpc.PartitionList) + size_t total_size = 0; + + if (_internal_metadata_.have_unknown_fields()) { + total_size += + ::PROTOBUF_NAMESPACE_ID::internal::WireFormat::ComputeUnknownFieldsSize( + _internal_metadata_.unknown_fields()); + } + ::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0; + // Prevent compiler warnings about cached_has_bits being unused + (void) cached_has_bits; + + // repeated .milvus.grpc.PartitionParam partition_array = 2; + { + unsigned int count = static_cast(this->partition_array_size()); + total_size += 1UL * count; + for (unsigned int i = 0; i < count; i++) { + total_size += + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::MessageSize( + this->partition_array(static_cast(i))); + } + } + + // .milvus.grpc.Status status = 1; + if (this->has_status()) { + total_size += 1 + + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::MessageSize( + *status_); + } + + int cached_size = ::PROTOBUF_NAMESPACE_ID::internal::ToCachedSize(total_size); + SetCachedSize(cached_size); + return total_size; +} + +void PartitionList::MergeFrom(const ::PROTOBUF_NAMESPACE_ID::Message& from) { +// @@protoc_insertion_point(generalized_merge_from_start:milvus.grpc.PartitionList) + GOOGLE_DCHECK_NE(&from, this); + const PartitionList* source = + ::PROTOBUF_NAMESPACE_ID::DynamicCastToGenerated( + &from); + if (source == nullptr) { + // @@protoc_insertion_point(generalized_merge_from_cast_fail:milvus.grpc.PartitionList) + ::PROTOBUF_NAMESPACE_ID::internal::ReflectionOps::Merge(from, this); + } else { + // @@protoc_insertion_point(generalized_merge_from_cast_success:milvus.grpc.PartitionList) + MergeFrom(*source); + } +} + +void PartitionList::MergeFrom(const PartitionList& from) { +// @@protoc_insertion_point(class_specific_merge_from_start:milvus.grpc.PartitionList) + GOOGLE_DCHECK_NE(&from, this); + _internal_metadata_.MergeFrom(from._internal_metadata_); + ::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0; + (void) cached_has_bits; + + partition_array_.MergeFrom(from.partition_array_); + if (from.has_status()) { + mutable_status()->::milvus::grpc::Status::MergeFrom(from.status()); + } +} + +void PartitionList::CopyFrom(const ::PROTOBUF_NAMESPACE_ID::Message& from) { +// @@protoc_insertion_point(generalized_copy_from_start:milvus.grpc.PartitionList) + if (&from == this) return; + Clear(); + MergeFrom(from); +} + +void PartitionList::CopyFrom(const PartitionList& from) { +// @@protoc_insertion_point(class_specific_copy_from_start:milvus.grpc.PartitionList) + if (&from == this) return; + Clear(); + MergeFrom(from); +} + +bool PartitionList::IsInitialized() const { + return true; +} + +void PartitionList::InternalSwap(PartitionList* other) { + using std::swap; + _internal_metadata_.Swap(&other->_internal_metadata_); + CastToBase(&partition_array_)->InternalSwap(CastToBase(&other->partition_array_)); + swap(status_, other->status_); +} + +::PROTOBUF_NAMESPACE_ID::Metadata PartitionList::GetMetadata() const { + return GetMetadataStatic(); +} + + // =================================================================== void Range::InitAsDefaultInstance() { @@ -2308,12 +3401,17 @@ InsertParam::InsertParam(const InsertParam& from) if (!from.table_name().empty()) { table_name_.AssignWithDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), from.table_name_); } + partition_tag_.UnsafeSetDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); + if (!from.partition_tag().empty()) { + partition_tag_.AssignWithDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), from.partition_tag_); + } // @@protoc_insertion_point(copy_constructor:milvus.grpc.InsertParam) } void InsertParam::SharedCtor() { ::PROTOBUF_NAMESPACE_ID::internal::InitSCC(&scc_info_InsertParam_milvus_2eproto.base); table_name_.UnsafeSetDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); + partition_tag_.UnsafeSetDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); } InsertParam::~InsertParam() { @@ -2323,6 +3421,7 @@ InsertParam::~InsertParam() { void InsertParam::SharedDtor() { table_name_.DestroyNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); + partition_tag_.DestroyNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); } void InsertParam::SetCachedSize(int size) const { @@ -2343,6 +3442,7 @@ void InsertParam::Clear() { row_record_array_.Clear(); row_id_array_.Clear(); table_name_.ClearToEmptyNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); + partition_tag_.ClearToEmptyNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); _internal_metadata_.Clear(); } @@ -2383,6 +3483,13 @@ const char* InsertParam::_InternalParse(const char* ptr, ::PROTOBUF_NAMESPACE_ID CHK_(ptr); } else goto handle_unusual; continue; + // string partition_tag = 4; + case 4: + if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 34)) { + ptr = ::PROTOBUF_NAMESPACE_ID::internal::InlineGreedyStringParserUTF8(mutable_partition_tag(), ptr, ctx, "milvus.grpc.InsertParam.partition_tag"); + CHK_(ptr); + } else goto handle_unusual; + continue; default: { handle_unusual: if ((tag & 7) == 4 || tag == 0) { @@ -2455,6 +3562,21 @@ bool InsertParam::MergePartialFromCodedStream( break; } + // string partition_tag = 4; + case 4: { + if (static_cast< ::PROTOBUF_NAMESPACE_ID::uint8>(tag) == (34 & 0xFF)) { + DO_(::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::ReadString( + input, this->mutable_partition_tag())); + DO_(::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::VerifyUtf8String( + this->partition_tag().data(), static_cast(this->partition_tag().length()), + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::PARSE, + "milvus.grpc.InsertParam.partition_tag")); + } else { + goto handle_unusual; + } + break; + } + default: { handle_unusual: if (tag == 0) { @@ -2512,6 +3634,16 @@ void InsertParam::SerializeWithCachedSizes( this->row_id_array(i), output); } + // string partition_tag = 4; + if (this->partition_tag().size() > 0) { + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::VerifyUtf8String( + this->partition_tag().data(), static_cast(this->partition_tag().length()), + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::SERIALIZE, + "milvus.grpc.InsertParam.partition_tag"); + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteStringMaybeAliased( + 4, this->partition_tag(), output); + } + if (_internal_metadata_.have_unknown_fields()) { ::PROTOBUF_NAMESPACE_ID::internal::WireFormat::SerializeUnknownFields( _internal_metadata_.unknown_fields(), output); @@ -2557,6 +3689,17 @@ void InsertParam::SerializeWithCachedSizes( WriteInt64NoTagToArray(this->row_id_array_, target); } + // string partition_tag = 4; + if (this->partition_tag().size() > 0) { + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::VerifyUtf8String( + this->partition_tag().data(), static_cast(this->partition_tag().length()), + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::SERIALIZE, + "milvus.grpc.InsertParam.partition_tag"); + target = + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteStringToArray( + 4, this->partition_tag(), target); + } + if (_internal_metadata_.have_unknown_fields()) { target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormat::SerializeUnknownFieldsToArray( _internal_metadata_.unknown_fields(), target); @@ -2611,6 +3754,13 @@ size_t InsertParam::ByteSizeLong() const { this->table_name()); } + // string partition_tag = 4; + if (this->partition_tag().size() > 0) { + total_size += 1 + + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::StringSize( + this->partition_tag()); + } + int cached_size = ::PROTOBUF_NAMESPACE_ID::internal::ToCachedSize(total_size); SetCachedSize(cached_size); return total_size; @@ -2644,6 +3794,10 @@ void InsertParam::MergeFrom(const InsertParam& from) { table_name_.AssignWithDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), from.table_name_); } + if (from.partition_tag().size() > 0) { + + partition_tag_.AssignWithDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), from.partition_tag_); + } } void InsertParam::CopyFrom(const ::PROTOBUF_NAMESPACE_ID::Message& from) { @@ -2671,6 +3825,8 @@ void InsertParam::InternalSwap(InsertParam* other) { row_id_array_.InternalSwap(&other->row_id_array_); table_name_.Swap(&other->table_name_, &::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), GetArenaNoVirtual()); + partition_tag_.Swap(&other->partition_tag_, &::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), + GetArenaNoVirtual()); } ::PROTOBUF_NAMESPACE_ID::Metadata InsertParam::GetMetadata() const { @@ -3036,7 +4192,8 @@ SearchParam::SearchParam(const SearchParam& from) : ::PROTOBUF_NAMESPACE_ID::Message(), _internal_metadata_(nullptr), query_record_array_(from.query_record_array_), - query_range_array_(from.query_range_array_) { + query_range_array_(from.query_range_array_), + partition_tag_array_(from.partition_tag_array_) { _internal_metadata_.MergeFrom(from._internal_metadata_); table_name_.UnsafeSetDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); if (!from.table_name().empty()) { @@ -3082,6 +4239,7 @@ void SearchParam::Clear() { query_record_array_.Clear(); query_range_array_.Clear(); + partition_tag_array_.Clear(); table_name_.ClearToEmptyNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); ::memset(&topk_, 0, static_cast( reinterpret_cast(&nprobe_) - @@ -3142,6 +4300,18 @@ const char* SearchParam::_InternalParse(const char* ptr, ::PROTOBUF_NAMESPACE_ID CHK_(ptr); } else goto handle_unusual; continue; + // repeated string partition_tag_array = 6; + case 6: + if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 50)) { + ptr -= 1; + do { + ptr += 1; + ptr = ::PROTOBUF_NAMESPACE_ID::internal::InlineGreedyStringParserUTF8(add_partition_tag_array(), ptr, ctx, "milvus.grpc.SearchParam.partition_tag_array"); + CHK_(ptr); + if (!ctx->DataAvailable(ptr)) break; + } while (::PROTOBUF_NAMESPACE_ID::internal::UnalignedLoad<::PROTOBUF_NAMESPACE_ID::uint8>(ptr) == 50); + } else goto handle_unusual; + continue; default: { handle_unusual: if ((tag & 7) == 4 || tag == 0) { @@ -3235,6 +4405,22 @@ bool SearchParam::MergePartialFromCodedStream( break; } + // repeated string partition_tag_array = 6; + case 6: { + if (static_cast< ::PROTOBUF_NAMESPACE_ID::uint8>(tag) == (50 & 0xFF)) { + DO_(::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::ReadString( + input, this->add_partition_tag_array())); + DO_(::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::VerifyUtf8String( + this->partition_tag_array(this->partition_tag_array_size() - 1).data(), + static_cast(this->partition_tag_array(this->partition_tag_array_size() - 1).length()), + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::PARSE, + "milvus.grpc.SearchParam.partition_tag_array")); + } else { + goto handle_unusual; + } + break; + } + default: { handle_unusual: if (tag == 0) { @@ -3300,6 +4486,16 @@ void SearchParam::SerializeWithCachedSizes( ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt64(5, this->nprobe(), output); } + // repeated string partition_tag_array = 6; + for (int i = 0, n = this->partition_tag_array_size(); i < n; i++) { + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::VerifyUtf8String( + this->partition_tag_array(i).data(), static_cast(this->partition_tag_array(i).length()), + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::SERIALIZE, + "milvus.grpc.SearchParam.partition_tag_array"); + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteString( + 6, this->partition_tag_array(i), output); + } + if (_internal_metadata_.have_unknown_fields()) { ::PROTOBUF_NAMESPACE_ID::internal::WireFormat::SerializeUnknownFields( _internal_metadata_.unknown_fields(), output); @@ -3350,6 +4546,16 @@ void SearchParam::SerializeWithCachedSizes( target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt64ToArray(5, this->nprobe(), target); } + // repeated string partition_tag_array = 6; + for (int i = 0, n = this->partition_tag_array_size(); i < n; i++) { + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::VerifyUtf8String( + this->partition_tag_array(i).data(), static_cast(this->partition_tag_array(i).length()), + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::SERIALIZE, + "milvus.grpc.SearchParam.partition_tag_array"); + target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite:: + WriteStringToArray(6, this->partition_tag_array(i), target); + } + if (_internal_metadata_.have_unknown_fields()) { target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormat::SerializeUnknownFieldsToArray( _internal_metadata_.unknown_fields(), target); @@ -3393,6 +4599,14 @@ size_t SearchParam::ByteSizeLong() const { } } + // repeated string partition_tag_array = 6; + total_size += 1 * + ::PROTOBUF_NAMESPACE_ID::internal::FromIntSize(this->partition_tag_array_size()); + for (int i = 0, n = this->partition_tag_array_size(); i < n; i++) { + total_size += ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::StringSize( + this->partition_tag_array(i)); + } + // string table_name = 1; if (this->table_name().size() > 0) { total_size += 1 + @@ -3443,6 +4657,7 @@ void SearchParam::MergeFrom(const SearchParam& from) { query_record_array_.MergeFrom(from.query_record_array_); query_range_array_.MergeFrom(from.query_range_array_); + partition_tag_array_.MergeFrom(from.partition_tag_array_); if (from.table_name().size() > 0) { table_name_.AssignWithDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), from.table_name_); @@ -3478,6 +4693,7 @@ void SearchParam::InternalSwap(SearchParam* other) { _internal_metadata_.Swap(&other->_internal_metadata_); CastToBase(&query_record_array_)->InternalSwap(CastToBase(&other->query_record_array_)); CastToBase(&query_range_array_)->InternalSwap(CastToBase(&other->query_range_array_)); + partition_tag_array_.InternalSwap(CastToBase(&other->partition_tag_array_)); table_name_.Swap(&other->table_name_, &::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), GetArenaNoVirtual()); swap(topk_, other->topk_); @@ -6198,25 +7414,25 @@ void IndexParam::InternalSwap(IndexParam* other) { // =================================================================== -void DeleteByRangeParam::InitAsDefaultInstance() { - ::milvus::grpc::_DeleteByRangeParam_default_instance_._instance.get_mutable()->range_ = const_cast< ::milvus::grpc::Range*>( +void DeleteByDateParam::InitAsDefaultInstance() { + ::milvus::grpc::_DeleteByDateParam_default_instance_._instance.get_mutable()->range_ = const_cast< ::milvus::grpc::Range*>( ::milvus::grpc::Range::internal_default_instance()); } -class DeleteByRangeParam::_Internal { +class DeleteByDateParam::_Internal { public: - static const ::milvus::grpc::Range& range(const DeleteByRangeParam* msg); + static const ::milvus::grpc::Range& range(const DeleteByDateParam* msg); }; const ::milvus::grpc::Range& -DeleteByRangeParam::_Internal::range(const DeleteByRangeParam* msg) { +DeleteByDateParam::_Internal::range(const DeleteByDateParam* msg) { return *msg->range_; } -DeleteByRangeParam::DeleteByRangeParam() +DeleteByDateParam::DeleteByDateParam() : ::PROTOBUF_NAMESPACE_ID::Message(), _internal_metadata_(nullptr) { SharedCtor(); - // @@protoc_insertion_point(constructor:milvus.grpc.DeleteByRangeParam) + // @@protoc_insertion_point(constructor:milvus.grpc.DeleteByDateParam) } -DeleteByRangeParam::DeleteByRangeParam(const DeleteByRangeParam& from) +DeleteByDateParam::DeleteByDateParam(const DeleteByDateParam& from) : ::PROTOBUF_NAMESPACE_ID::Message(), _internal_metadata_(nullptr) { _internal_metadata_.MergeFrom(from._internal_metadata_); @@ -6229,36 +7445,36 @@ DeleteByRangeParam::DeleteByRangeParam(const DeleteByRangeParam& from) } else { range_ = nullptr; } - // @@protoc_insertion_point(copy_constructor:milvus.grpc.DeleteByRangeParam) + // @@protoc_insertion_point(copy_constructor:milvus.grpc.DeleteByDateParam) } -void DeleteByRangeParam::SharedCtor() { - ::PROTOBUF_NAMESPACE_ID::internal::InitSCC(&scc_info_DeleteByRangeParam_milvus_2eproto.base); +void DeleteByDateParam::SharedCtor() { + ::PROTOBUF_NAMESPACE_ID::internal::InitSCC(&scc_info_DeleteByDateParam_milvus_2eproto.base); table_name_.UnsafeSetDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); range_ = nullptr; } -DeleteByRangeParam::~DeleteByRangeParam() { - // @@protoc_insertion_point(destructor:milvus.grpc.DeleteByRangeParam) +DeleteByDateParam::~DeleteByDateParam() { + // @@protoc_insertion_point(destructor:milvus.grpc.DeleteByDateParam) SharedDtor(); } -void DeleteByRangeParam::SharedDtor() { +void DeleteByDateParam::SharedDtor() { table_name_.DestroyNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); if (this != internal_default_instance()) delete range_; } -void DeleteByRangeParam::SetCachedSize(int size) const { +void DeleteByDateParam::SetCachedSize(int size) const { _cached_size_.Set(size); } -const DeleteByRangeParam& DeleteByRangeParam::default_instance() { - ::PROTOBUF_NAMESPACE_ID::internal::InitSCC(&::scc_info_DeleteByRangeParam_milvus_2eproto.base); +const DeleteByDateParam& DeleteByDateParam::default_instance() { + ::PROTOBUF_NAMESPACE_ID::internal::InitSCC(&::scc_info_DeleteByDateParam_milvus_2eproto.base); return *internal_default_instance(); } -void DeleteByRangeParam::Clear() { -// @@protoc_insertion_point(message_clear_start:milvus.grpc.DeleteByRangeParam) +void DeleteByDateParam::Clear() { +// @@protoc_insertion_point(message_clear_start:milvus.grpc.DeleteByDateParam) ::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0; // Prevent compiler warnings about cached_has_bits being unused (void) cached_has_bits; @@ -6272,7 +7488,7 @@ void DeleteByRangeParam::Clear() { } #if GOOGLE_PROTOBUF_ENABLE_EXPERIMENTAL_PARSER -const char* DeleteByRangeParam::_InternalParse(const char* ptr, ::PROTOBUF_NAMESPACE_ID::internal::ParseContext* ctx) { +const char* DeleteByDateParam::_InternalParse(const char* ptr, ::PROTOBUF_NAMESPACE_ID::internal::ParseContext* ctx) { #define CHK_(x) if (PROTOBUF_PREDICT_FALSE(!(x))) goto failure while (!ctx->Done(&ptr)) { ::PROTOBUF_NAMESPACE_ID::uint32 tag; @@ -6289,7 +7505,7 @@ const char* DeleteByRangeParam::_InternalParse(const char* ptr, ::PROTOBUF_NAMES // string table_name = 2; case 2: if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 18)) { - ptr = ::PROTOBUF_NAMESPACE_ID::internal::InlineGreedyStringParserUTF8(mutable_table_name(), ptr, ctx, "milvus.grpc.DeleteByRangeParam.table_name"); + ptr = ::PROTOBUF_NAMESPACE_ID::internal::InlineGreedyStringParserUTF8(mutable_table_name(), ptr, ctx, "milvus.grpc.DeleteByDateParam.table_name"); CHK_(ptr); } else goto handle_unusual; continue; @@ -6313,11 +7529,11 @@ failure: #undef CHK_ } #else // GOOGLE_PROTOBUF_ENABLE_EXPERIMENTAL_PARSER -bool DeleteByRangeParam::MergePartialFromCodedStream( +bool DeleteByDateParam::MergePartialFromCodedStream( ::PROTOBUF_NAMESPACE_ID::io::CodedInputStream* input) { #define DO_(EXPRESSION) if (!PROTOBUF_PREDICT_TRUE(EXPRESSION)) goto failure ::PROTOBUF_NAMESPACE_ID::uint32 tag; - // @@protoc_insertion_point(parse_start:milvus.grpc.DeleteByRangeParam) + // @@protoc_insertion_point(parse_start:milvus.grpc.DeleteByDateParam) for (;;) { ::std::pair<::PROTOBUF_NAMESPACE_ID::uint32, bool> p = input->ReadTagWithCutoffNoLastTag(127u); tag = p.first; @@ -6342,7 +7558,7 @@ bool DeleteByRangeParam::MergePartialFromCodedStream( DO_(::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::VerifyUtf8String( this->table_name().data(), static_cast(this->table_name().length()), ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::PARSE, - "milvus.grpc.DeleteByRangeParam.table_name")); + "milvus.grpc.DeleteByDateParam.table_name")); } else { goto handle_unusual; } @@ -6361,18 +7577,18 @@ bool DeleteByRangeParam::MergePartialFromCodedStream( } } success: - // @@protoc_insertion_point(parse_success:milvus.grpc.DeleteByRangeParam) + // @@protoc_insertion_point(parse_success:milvus.grpc.DeleteByDateParam) return true; failure: - // @@protoc_insertion_point(parse_failure:milvus.grpc.DeleteByRangeParam) + // @@protoc_insertion_point(parse_failure:milvus.grpc.DeleteByDateParam) return false; #undef DO_ } #endif // GOOGLE_PROTOBUF_ENABLE_EXPERIMENTAL_PARSER -void DeleteByRangeParam::SerializeWithCachedSizes( +void DeleteByDateParam::SerializeWithCachedSizes( ::PROTOBUF_NAMESPACE_ID::io::CodedOutputStream* output) const { - // @@protoc_insertion_point(serialize_start:milvus.grpc.DeleteByRangeParam) + // @@protoc_insertion_point(serialize_start:milvus.grpc.DeleteByDateParam) ::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0; (void) cached_has_bits; @@ -6387,7 +7603,7 @@ void DeleteByRangeParam::SerializeWithCachedSizes( ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::VerifyUtf8String( this->table_name().data(), static_cast(this->table_name().length()), ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::SERIALIZE, - "milvus.grpc.DeleteByRangeParam.table_name"); + "milvus.grpc.DeleteByDateParam.table_name"); ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteStringMaybeAliased( 2, this->table_name(), output); } @@ -6396,12 +7612,12 @@ void DeleteByRangeParam::SerializeWithCachedSizes( ::PROTOBUF_NAMESPACE_ID::internal::WireFormat::SerializeUnknownFields( _internal_metadata_.unknown_fields(), output); } - // @@protoc_insertion_point(serialize_end:milvus.grpc.DeleteByRangeParam) + // @@protoc_insertion_point(serialize_end:milvus.grpc.DeleteByDateParam) } -::PROTOBUF_NAMESPACE_ID::uint8* DeleteByRangeParam::InternalSerializeWithCachedSizesToArray( +::PROTOBUF_NAMESPACE_ID::uint8* DeleteByDateParam::InternalSerializeWithCachedSizesToArray( ::PROTOBUF_NAMESPACE_ID::uint8* target) const { - // @@protoc_insertion_point(serialize_to_array_start:milvus.grpc.DeleteByRangeParam) + // @@protoc_insertion_point(serialize_to_array_start:milvus.grpc.DeleteByDateParam) ::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0; (void) cached_has_bits; @@ -6417,7 +7633,7 @@ void DeleteByRangeParam::SerializeWithCachedSizes( ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::VerifyUtf8String( this->table_name().data(), static_cast(this->table_name().length()), ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::SERIALIZE, - "milvus.grpc.DeleteByRangeParam.table_name"); + "milvus.grpc.DeleteByDateParam.table_name"); target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteStringToArray( 2, this->table_name(), target); @@ -6427,12 +7643,12 @@ void DeleteByRangeParam::SerializeWithCachedSizes( target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormat::SerializeUnknownFieldsToArray( _internal_metadata_.unknown_fields(), target); } - // @@protoc_insertion_point(serialize_to_array_end:milvus.grpc.DeleteByRangeParam) + // @@protoc_insertion_point(serialize_to_array_end:milvus.grpc.DeleteByDateParam) return target; } -size_t DeleteByRangeParam::ByteSizeLong() const { -// @@protoc_insertion_point(message_byte_size_start:milvus.grpc.DeleteByRangeParam) +size_t DeleteByDateParam::ByteSizeLong() const { +// @@protoc_insertion_point(message_byte_size_start:milvus.grpc.DeleteByDateParam) size_t total_size = 0; if (_internal_metadata_.have_unknown_fields()) { @@ -6463,23 +7679,23 @@ size_t DeleteByRangeParam::ByteSizeLong() const { return total_size; } -void DeleteByRangeParam::MergeFrom(const ::PROTOBUF_NAMESPACE_ID::Message& from) { -// @@protoc_insertion_point(generalized_merge_from_start:milvus.grpc.DeleteByRangeParam) +void DeleteByDateParam::MergeFrom(const ::PROTOBUF_NAMESPACE_ID::Message& from) { +// @@protoc_insertion_point(generalized_merge_from_start:milvus.grpc.DeleteByDateParam) GOOGLE_DCHECK_NE(&from, this); - const DeleteByRangeParam* source = - ::PROTOBUF_NAMESPACE_ID::DynamicCastToGenerated( + const DeleteByDateParam* source = + ::PROTOBUF_NAMESPACE_ID::DynamicCastToGenerated( &from); if (source == nullptr) { - // @@protoc_insertion_point(generalized_merge_from_cast_fail:milvus.grpc.DeleteByRangeParam) + // @@protoc_insertion_point(generalized_merge_from_cast_fail:milvus.grpc.DeleteByDateParam) ::PROTOBUF_NAMESPACE_ID::internal::ReflectionOps::Merge(from, this); } else { - // @@protoc_insertion_point(generalized_merge_from_cast_success:milvus.grpc.DeleteByRangeParam) + // @@protoc_insertion_point(generalized_merge_from_cast_success:milvus.grpc.DeleteByDateParam) MergeFrom(*source); } } -void DeleteByRangeParam::MergeFrom(const DeleteByRangeParam& from) { -// @@protoc_insertion_point(class_specific_merge_from_start:milvus.grpc.DeleteByRangeParam) +void DeleteByDateParam::MergeFrom(const DeleteByDateParam& from) { +// @@protoc_insertion_point(class_specific_merge_from_start:milvus.grpc.DeleteByDateParam) GOOGLE_DCHECK_NE(&from, this); _internal_metadata_.MergeFrom(from._internal_metadata_); ::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0; @@ -6494,25 +7710,25 @@ void DeleteByRangeParam::MergeFrom(const DeleteByRangeParam& from) { } } -void DeleteByRangeParam::CopyFrom(const ::PROTOBUF_NAMESPACE_ID::Message& from) { -// @@protoc_insertion_point(generalized_copy_from_start:milvus.grpc.DeleteByRangeParam) +void DeleteByDateParam::CopyFrom(const ::PROTOBUF_NAMESPACE_ID::Message& from) { +// @@protoc_insertion_point(generalized_copy_from_start:milvus.grpc.DeleteByDateParam) if (&from == this) return; Clear(); MergeFrom(from); } -void DeleteByRangeParam::CopyFrom(const DeleteByRangeParam& from) { -// @@protoc_insertion_point(class_specific_copy_from_start:milvus.grpc.DeleteByRangeParam) +void DeleteByDateParam::CopyFrom(const DeleteByDateParam& from) { +// @@protoc_insertion_point(class_specific_copy_from_start:milvus.grpc.DeleteByDateParam) if (&from == this) return; Clear(); MergeFrom(from); } -bool DeleteByRangeParam::IsInitialized() const { +bool DeleteByDateParam::IsInitialized() const { return true; } -void DeleteByRangeParam::InternalSwap(DeleteByRangeParam* other) { +void DeleteByDateParam::InternalSwap(DeleteByDateParam* other) { using std::swap; _internal_metadata_.Swap(&other->_internal_metadata_); table_name_.Swap(&other->table_name_, &::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), @@ -6520,7 +7736,7 @@ void DeleteByRangeParam::InternalSwap(DeleteByRangeParam* other) { swap(range_, other->range_); } -::PROTOBUF_NAMESPACE_ID::Metadata DeleteByRangeParam::GetMetadata() const { +::PROTOBUF_NAMESPACE_ID::Metadata DeleteByDateParam::GetMetadata() const { return GetMetadataStatic(); } @@ -6532,12 +7748,21 @@ PROTOBUF_NAMESPACE_OPEN template<> PROTOBUF_NOINLINE ::milvus::grpc::TableName* Arena::CreateMaybeMessage< ::milvus::grpc::TableName >(Arena* arena) { return Arena::CreateInternal< ::milvus::grpc::TableName >(arena); } +template<> PROTOBUF_NOINLINE ::milvus::grpc::PartitionName* Arena::CreateMaybeMessage< ::milvus::grpc::PartitionName >(Arena* arena) { + return Arena::CreateInternal< ::milvus::grpc::PartitionName >(arena); +} template<> PROTOBUF_NOINLINE ::milvus::grpc::TableNameList* Arena::CreateMaybeMessage< ::milvus::grpc::TableNameList >(Arena* arena) { return Arena::CreateInternal< ::milvus::grpc::TableNameList >(arena); } template<> PROTOBUF_NOINLINE ::milvus::grpc::TableSchema* Arena::CreateMaybeMessage< ::milvus::grpc::TableSchema >(Arena* arena) { return Arena::CreateInternal< ::milvus::grpc::TableSchema >(arena); } +template<> PROTOBUF_NOINLINE ::milvus::grpc::PartitionParam* Arena::CreateMaybeMessage< ::milvus::grpc::PartitionParam >(Arena* arena) { + return Arena::CreateInternal< ::milvus::grpc::PartitionParam >(arena); +} +template<> PROTOBUF_NOINLINE ::milvus::grpc::PartitionList* Arena::CreateMaybeMessage< ::milvus::grpc::PartitionList >(Arena* arena) { + return Arena::CreateInternal< ::milvus::grpc::PartitionList >(arena); +} template<> PROTOBUF_NOINLINE ::milvus::grpc::Range* Arena::CreateMaybeMessage< ::milvus::grpc::Range >(Arena* arena) { return Arena::CreateInternal< ::milvus::grpc::Range >(arena); } @@ -6577,8 +7802,8 @@ template<> PROTOBUF_NOINLINE ::milvus::grpc::Index* Arena::CreateMaybeMessage< : template<> PROTOBUF_NOINLINE ::milvus::grpc::IndexParam* Arena::CreateMaybeMessage< ::milvus::grpc::IndexParam >(Arena* arena) { return Arena::CreateInternal< ::milvus::grpc::IndexParam >(arena); } -template<> PROTOBUF_NOINLINE ::milvus::grpc::DeleteByRangeParam* Arena::CreateMaybeMessage< ::milvus::grpc::DeleteByRangeParam >(Arena* arena) { - return Arena::CreateInternal< ::milvus::grpc::DeleteByRangeParam >(arena); +template<> PROTOBUF_NOINLINE ::milvus::grpc::DeleteByDateParam* Arena::CreateMaybeMessage< ::milvus::grpc::DeleteByDateParam >(Arena* arena) { + return Arena::CreateInternal< ::milvus::grpc::DeleteByDateParam >(arena); } PROTOBUF_NAMESPACE_CLOSE diff --git a/core/src/grpc/gen-milvus/milvus.pb.h b/core/src/grpc/gen-milvus/milvus.pb.h index 5094f22341..53ed2db22e 100644 --- a/core/src/grpc/gen-milvus/milvus.pb.h +++ b/core/src/grpc/gen-milvus/milvus.pb.h @@ -48,7 +48,7 @@ struct TableStruct_milvus_2eproto { PROTOBUF_SECTION_VARIABLE(protodesc_cold); static const ::PROTOBUF_NAMESPACE_ID::internal::AuxillaryParseTableField aux[] PROTOBUF_SECTION_VARIABLE(protodesc_cold); - static const ::PROTOBUF_NAMESPACE_ID::internal::ParseTable schema[17] + static const ::PROTOBUF_NAMESPACE_ID::internal::ParseTable schema[20] PROTOBUF_SECTION_VARIABLE(protodesc_cold); static const ::PROTOBUF_NAMESPACE_ID::internal::FieldMetadata field_metadata[]; static const ::PROTOBUF_NAMESPACE_ID::internal::SerializationTable serialization_table[]; @@ -63,9 +63,9 @@ extern BoolReplyDefaultTypeInternal _BoolReply_default_instance_; class Command; class CommandDefaultTypeInternal; extern CommandDefaultTypeInternal _Command_default_instance_; -class DeleteByRangeParam; -class DeleteByRangeParamDefaultTypeInternal; -extern DeleteByRangeParamDefaultTypeInternal _DeleteByRangeParam_default_instance_; +class DeleteByDateParam; +class DeleteByDateParamDefaultTypeInternal; +extern DeleteByDateParamDefaultTypeInternal _DeleteByDateParam_default_instance_; class Index; class IndexDefaultTypeInternal; extern IndexDefaultTypeInternal _Index_default_instance_; @@ -75,6 +75,15 @@ extern IndexParamDefaultTypeInternal _IndexParam_default_instance_; class InsertParam; class InsertParamDefaultTypeInternal; extern InsertParamDefaultTypeInternal _InsertParam_default_instance_; +class PartitionList; +class PartitionListDefaultTypeInternal; +extern PartitionListDefaultTypeInternal _PartitionList_default_instance_; +class PartitionName; +class PartitionNameDefaultTypeInternal; +extern PartitionNameDefaultTypeInternal _PartitionName_default_instance_; +class PartitionParam; +class PartitionParamDefaultTypeInternal; +extern PartitionParamDefaultTypeInternal _PartitionParam_default_instance_; class Range; class RangeDefaultTypeInternal; extern RangeDefaultTypeInternal _Range_default_instance_; @@ -113,10 +122,13 @@ extern VectorIdsDefaultTypeInternal _VectorIds_default_instance_; PROTOBUF_NAMESPACE_OPEN template<> ::milvus::grpc::BoolReply* Arena::CreateMaybeMessage<::milvus::grpc::BoolReply>(Arena*); template<> ::milvus::grpc::Command* Arena::CreateMaybeMessage<::milvus::grpc::Command>(Arena*); -template<> ::milvus::grpc::DeleteByRangeParam* Arena::CreateMaybeMessage<::milvus::grpc::DeleteByRangeParam>(Arena*); +template<> ::milvus::grpc::DeleteByDateParam* Arena::CreateMaybeMessage<::milvus::grpc::DeleteByDateParam>(Arena*); template<> ::milvus::grpc::Index* Arena::CreateMaybeMessage<::milvus::grpc::Index>(Arena*); template<> ::milvus::grpc::IndexParam* Arena::CreateMaybeMessage<::milvus::grpc::IndexParam>(Arena*); template<> ::milvus::grpc::InsertParam* Arena::CreateMaybeMessage<::milvus::grpc::InsertParam>(Arena*); +template<> ::milvus::grpc::PartitionList* Arena::CreateMaybeMessage<::milvus::grpc::PartitionList>(Arena*); +template<> ::milvus::grpc::PartitionName* Arena::CreateMaybeMessage<::milvus::grpc::PartitionName>(Arena*); +template<> ::milvus::grpc::PartitionParam* Arena::CreateMaybeMessage<::milvus::grpc::PartitionParam>(Arena*); template<> ::milvus::grpc::Range* Arena::CreateMaybeMessage<::milvus::grpc::Range>(Arena*); template<> ::milvus::grpc::RowRecord* Arena::CreateMaybeMessage<::milvus::grpc::RowRecord>(Arena*); template<> ::milvus::grpc::SearchInFilesParam* Arena::CreateMaybeMessage<::milvus::grpc::SearchInFilesParam>(Arena*); @@ -271,6 +283,143 @@ class TableName : }; // ------------------------------------------------------------------- +class PartitionName : + public ::PROTOBUF_NAMESPACE_ID::Message /* @@protoc_insertion_point(class_definition:milvus.grpc.PartitionName) */ { + public: + PartitionName(); + virtual ~PartitionName(); + + PartitionName(const PartitionName& from); + PartitionName(PartitionName&& from) noexcept + : PartitionName() { + *this = ::std::move(from); + } + + inline PartitionName& operator=(const PartitionName& from) { + CopyFrom(from); + return *this; + } + inline PartitionName& operator=(PartitionName&& from) noexcept { + if (GetArenaNoVirtual() == from.GetArenaNoVirtual()) { + if (this != &from) InternalSwap(&from); + } else { + CopyFrom(from); + } + return *this; + } + + static const ::PROTOBUF_NAMESPACE_ID::Descriptor* descriptor() { + return GetDescriptor(); + } + static const ::PROTOBUF_NAMESPACE_ID::Descriptor* GetDescriptor() { + return GetMetadataStatic().descriptor; + } + static const ::PROTOBUF_NAMESPACE_ID::Reflection* GetReflection() { + return GetMetadataStatic().reflection; + } + static const PartitionName& default_instance(); + + static void InitAsDefaultInstance(); // FOR INTERNAL USE ONLY + static inline const PartitionName* internal_default_instance() { + return reinterpret_cast( + &_PartitionName_default_instance_); + } + static constexpr int kIndexInFileMessages = + 1; + + friend void swap(PartitionName& a, PartitionName& b) { + a.Swap(&b); + } + inline void Swap(PartitionName* other) { + if (other == this) return; + InternalSwap(other); + } + + // implements Message ---------------------------------------------- + + inline PartitionName* New() const final { + return CreateMaybeMessage(nullptr); + } + + PartitionName* New(::PROTOBUF_NAMESPACE_ID::Arena* arena) const final { + return CreateMaybeMessage(arena); + } + void CopyFrom(const ::PROTOBUF_NAMESPACE_ID::Message& from) final; + void MergeFrom(const ::PROTOBUF_NAMESPACE_ID::Message& from) final; + void CopyFrom(const PartitionName& from); + void MergeFrom(const PartitionName& from); + PROTOBUF_ATTRIBUTE_REINITIALIZES void Clear() final; + bool IsInitialized() const final; + + size_t ByteSizeLong() const final; + #if GOOGLE_PROTOBUF_ENABLE_EXPERIMENTAL_PARSER + const char* _InternalParse(const char* ptr, ::PROTOBUF_NAMESPACE_ID::internal::ParseContext* ctx) final; + #else + bool MergePartialFromCodedStream( + ::PROTOBUF_NAMESPACE_ID::io::CodedInputStream* input) final; + #endif // GOOGLE_PROTOBUF_ENABLE_EXPERIMENTAL_PARSER + void SerializeWithCachedSizes( + ::PROTOBUF_NAMESPACE_ID::io::CodedOutputStream* output) const final; + ::PROTOBUF_NAMESPACE_ID::uint8* InternalSerializeWithCachedSizesToArray( + ::PROTOBUF_NAMESPACE_ID::uint8* target) const final; + int GetCachedSize() const final { return _cached_size_.Get(); } + + private: + inline void SharedCtor(); + inline void SharedDtor(); + void SetCachedSize(int size) const final; + void InternalSwap(PartitionName* other); + friend class ::PROTOBUF_NAMESPACE_ID::internal::AnyMetadata; + static ::PROTOBUF_NAMESPACE_ID::StringPiece FullMessageName() { + return "milvus.grpc.PartitionName"; + } + private: + inline ::PROTOBUF_NAMESPACE_ID::Arena* GetArenaNoVirtual() const { + return nullptr; + } + inline void* MaybeArenaPtr() const { + return nullptr; + } + public: + + ::PROTOBUF_NAMESPACE_ID::Metadata GetMetadata() const final; + private: + static ::PROTOBUF_NAMESPACE_ID::Metadata GetMetadataStatic() { + ::PROTOBUF_NAMESPACE_ID::internal::AssignDescriptors(&::descriptor_table_milvus_2eproto); + return ::descriptor_table_milvus_2eproto.file_level_metadata[kIndexInFileMessages]; + } + + public: + + // nested types ---------------------------------------------------- + + // accessors ------------------------------------------------------- + + enum : int { + kPartitionNameFieldNumber = 1, + }; + // string partition_name = 1; + void clear_partition_name(); + const std::string& partition_name() const; + void set_partition_name(const std::string& value); + void set_partition_name(std::string&& value); + void set_partition_name(const char* value); + void set_partition_name(const char* value, size_t size); + std::string* mutable_partition_name(); + std::string* release_partition_name(); + void set_allocated_partition_name(std::string* partition_name); + + // @@protoc_insertion_point(class_scope:milvus.grpc.PartitionName) + private: + class _Internal; + + ::PROTOBUF_NAMESPACE_ID::internal::InternalMetadataWithArena _internal_metadata_; + ::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr partition_name_; + mutable ::PROTOBUF_NAMESPACE_ID::internal::CachedSize _cached_size_; + friend struct ::TableStruct_milvus_2eproto; +}; +// ------------------------------------------------------------------- + class TableNameList : public ::PROTOBUF_NAMESPACE_ID::Message /* @@protoc_insertion_point(class_definition:milvus.grpc.TableNameList) */ { public: @@ -313,7 +462,7 @@ class TableNameList : &_TableNameList_default_instance_); } static constexpr int kIndexInFileMessages = - 1; + 2; friend void swap(TableNameList& a, TableNameList& b) { a.Swap(&b); @@ -466,7 +615,7 @@ class TableSchema : &_TableSchema_default_instance_); } static constexpr int kIndexInFileMessages = - 2; + 3; friend void swap(TableSchema& a, TableSchema& b) { a.Swap(&b); @@ -592,6 +741,316 @@ class TableSchema : }; // ------------------------------------------------------------------- +class PartitionParam : + public ::PROTOBUF_NAMESPACE_ID::Message /* @@protoc_insertion_point(class_definition:milvus.grpc.PartitionParam) */ { + public: + PartitionParam(); + virtual ~PartitionParam(); + + PartitionParam(const PartitionParam& from); + PartitionParam(PartitionParam&& from) noexcept + : PartitionParam() { + *this = ::std::move(from); + } + + inline PartitionParam& operator=(const PartitionParam& from) { + CopyFrom(from); + return *this; + } + inline PartitionParam& operator=(PartitionParam&& from) noexcept { + if (GetArenaNoVirtual() == from.GetArenaNoVirtual()) { + if (this != &from) InternalSwap(&from); + } else { + CopyFrom(from); + } + return *this; + } + + static const ::PROTOBUF_NAMESPACE_ID::Descriptor* descriptor() { + return GetDescriptor(); + } + static const ::PROTOBUF_NAMESPACE_ID::Descriptor* GetDescriptor() { + return GetMetadataStatic().descriptor; + } + static const ::PROTOBUF_NAMESPACE_ID::Reflection* GetReflection() { + return GetMetadataStatic().reflection; + } + static const PartitionParam& default_instance(); + + static void InitAsDefaultInstance(); // FOR INTERNAL USE ONLY + static inline const PartitionParam* internal_default_instance() { + return reinterpret_cast( + &_PartitionParam_default_instance_); + } + static constexpr int kIndexInFileMessages = + 4; + + friend void swap(PartitionParam& a, PartitionParam& b) { + a.Swap(&b); + } + inline void Swap(PartitionParam* other) { + if (other == this) return; + InternalSwap(other); + } + + // implements Message ---------------------------------------------- + + inline PartitionParam* New() const final { + return CreateMaybeMessage(nullptr); + } + + PartitionParam* New(::PROTOBUF_NAMESPACE_ID::Arena* arena) const final { + return CreateMaybeMessage(arena); + } + void CopyFrom(const ::PROTOBUF_NAMESPACE_ID::Message& from) final; + void MergeFrom(const ::PROTOBUF_NAMESPACE_ID::Message& from) final; + void CopyFrom(const PartitionParam& from); + void MergeFrom(const PartitionParam& from); + PROTOBUF_ATTRIBUTE_REINITIALIZES void Clear() final; + bool IsInitialized() const final; + + size_t ByteSizeLong() const final; + #if GOOGLE_PROTOBUF_ENABLE_EXPERIMENTAL_PARSER + const char* _InternalParse(const char* ptr, ::PROTOBUF_NAMESPACE_ID::internal::ParseContext* ctx) final; + #else + bool MergePartialFromCodedStream( + ::PROTOBUF_NAMESPACE_ID::io::CodedInputStream* input) final; + #endif // GOOGLE_PROTOBUF_ENABLE_EXPERIMENTAL_PARSER + void SerializeWithCachedSizes( + ::PROTOBUF_NAMESPACE_ID::io::CodedOutputStream* output) const final; + ::PROTOBUF_NAMESPACE_ID::uint8* InternalSerializeWithCachedSizesToArray( + ::PROTOBUF_NAMESPACE_ID::uint8* target) const final; + int GetCachedSize() const final { return _cached_size_.Get(); } + + private: + inline void SharedCtor(); + inline void SharedDtor(); + void SetCachedSize(int size) const final; + void InternalSwap(PartitionParam* other); + friend class ::PROTOBUF_NAMESPACE_ID::internal::AnyMetadata; + static ::PROTOBUF_NAMESPACE_ID::StringPiece FullMessageName() { + return "milvus.grpc.PartitionParam"; + } + private: + inline ::PROTOBUF_NAMESPACE_ID::Arena* GetArenaNoVirtual() const { + return nullptr; + } + inline void* MaybeArenaPtr() const { + return nullptr; + } + public: + + ::PROTOBUF_NAMESPACE_ID::Metadata GetMetadata() const final; + private: + static ::PROTOBUF_NAMESPACE_ID::Metadata GetMetadataStatic() { + ::PROTOBUF_NAMESPACE_ID::internal::AssignDescriptors(&::descriptor_table_milvus_2eproto); + return ::descriptor_table_milvus_2eproto.file_level_metadata[kIndexInFileMessages]; + } + + public: + + // nested types ---------------------------------------------------- + + // accessors ------------------------------------------------------- + + enum : int { + kTableNameFieldNumber = 1, + kPartitionNameFieldNumber = 2, + kTagFieldNumber = 3, + }; + // string table_name = 1; + void clear_table_name(); + const std::string& table_name() const; + void set_table_name(const std::string& value); + void set_table_name(std::string&& value); + void set_table_name(const char* value); + void set_table_name(const char* value, size_t size); + std::string* mutable_table_name(); + std::string* release_table_name(); + void set_allocated_table_name(std::string* table_name); + + // string partition_name = 2; + void clear_partition_name(); + const std::string& partition_name() const; + void set_partition_name(const std::string& value); + void set_partition_name(std::string&& value); + void set_partition_name(const char* value); + void set_partition_name(const char* value, size_t size); + std::string* mutable_partition_name(); + std::string* release_partition_name(); + void set_allocated_partition_name(std::string* partition_name); + + // string tag = 3; + void clear_tag(); + const std::string& tag() const; + void set_tag(const std::string& value); + void set_tag(std::string&& value); + void set_tag(const char* value); + void set_tag(const char* value, size_t size); + std::string* mutable_tag(); + std::string* release_tag(); + void set_allocated_tag(std::string* tag); + + // @@protoc_insertion_point(class_scope:milvus.grpc.PartitionParam) + private: + class _Internal; + + ::PROTOBUF_NAMESPACE_ID::internal::InternalMetadataWithArena _internal_metadata_; + ::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr table_name_; + ::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr partition_name_; + ::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr tag_; + mutable ::PROTOBUF_NAMESPACE_ID::internal::CachedSize _cached_size_; + friend struct ::TableStruct_milvus_2eproto; +}; +// ------------------------------------------------------------------- + +class PartitionList : + public ::PROTOBUF_NAMESPACE_ID::Message /* @@protoc_insertion_point(class_definition:milvus.grpc.PartitionList) */ { + public: + PartitionList(); + virtual ~PartitionList(); + + PartitionList(const PartitionList& from); + PartitionList(PartitionList&& from) noexcept + : PartitionList() { + *this = ::std::move(from); + } + + inline PartitionList& operator=(const PartitionList& from) { + CopyFrom(from); + return *this; + } + inline PartitionList& operator=(PartitionList&& from) noexcept { + if (GetArenaNoVirtual() == from.GetArenaNoVirtual()) { + if (this != &from) InternalSwap(&from); + } else { + CopyFrom(from); + } + return *this; + } + + static const ::PROTOBUF_NAMESPACE_ID::Descriptor* descriptor() { + return GetDescriptor(); + } + static const ::PROTOBUF_NAMESPACE_ID::Descriptor* GetDescriptor() { + return GetMetadataStatic().descriptor; + } + static const ::PROTOBUF_NAMESPACE_ID::Reflection* GetReflection() { + return GetMetadataStatic().reflection; + } + static const PartitionList& default_instance(); + + static void InitAsDefaultInstance(); // FOR INTERNAL USE ONLY + static inline const PartitionList* internal_default_instance() { + return reinterpret_cast( + &_PartitionList_default_instance_); + } + static constexpr int kIndexInFileMessages = + 5; + + friend void swap(PartitionList& a, PartitionList& b) { + a.Swap(&b); + } + inline void Swap(PartitionList* other) { + if (other == this) return; + InternalSwap(other); + } + + // implements Message ---------------------------------------------- + + inline PartitionList* New() const final { + return CreateMaybeMessage(nullptr); + } + + PartitionList* New(::PROTOBUF_NAMESPACE_ID::Arena* arena) const final { + return CreateMaybeMessage(arena); + } + void CopyFrom(const ::PROTOBUF_NAMESPACE_ID::Message& from) final; + void MergeFrom(const ::PROTOBUF_NAMESPACE_ID::Message& from) final; + void CopyFrom(const PartitionList& from); + void MergeFrom(const PartitionList& from); + PROTOBUF_ATTRIBUTE_REINITIALIZES void Clear() final; + bool IsInitialized() const final; + + size_t ByteSizeLong() const final; + #if GOOGLE_PROTOBUF_ENABLE_EXPERIMENTAL_PARSER + const char* _InternalParse(const char* ptr, ::PROTOBUF_NAMESPACE_ID::internal::ParseContext* ctx) final; + #else + bool MergePartialFromCodedStream( + ::PROTOBUF_NAMESPACE_ID::io::CodedInputStream* input) final; + #endif // GOOGLE_PROTOBUF_ENABLE_EXPERIMENTAL_PARSER + void SerializeWithCachedSizes( + ::PROTOBUF_NAMESPACE_ID::io::CodedOutputStream* output) const final; + ::PROTOBUF_NAMESPACE_ID::uint8* InternalSerializeWithCachedSizesToArray( + ::PROTOBUF_NAMESPACE_ID::uint8* target) const final; + int GetCachedSize() const final { return _cached_size_.Get(); } + + private: + inline void SharedCtor(); + inline void SharedDtor(); + void SetCachedSize(int size) const final; + void InternalSwap(PartitionList* other); + friend class ::PROTOBUF_NAMESPACE_ID::internal::AnyMetadata; + static ::PROTOBUF_NAMESPACE_ID::StringPiece FullMessageName() { + return "milvus.grpc.PartitionList"; + } + private: + inline ::PROTOBUF_NAMESPACE_ID::Arena* GetArenaNoVirtual() const { + return nullptr; + } + inline void* MaybeArenaPtr() const { + return nullptr; + } + public: + + ::PROTOBUF_NAMESPACE_ID::Metadata GetMetadata() const final; + private: + static ::PROTOBUF_NAMESPACE_ID::Metadata GetMetadataStatic() { + ::PROTOBUF_NAMESPACE_ID::internal::AssignDescriptors(&::descriptor_table_milvus_2eproto); + return ::descriptor_table_milvus_2eproto.file_level_metadata[kIndexInFileMessages]; + } + + public: + + // nested types ---------------------------------------------------- + + // accessors ------------------------------------------------------- + + enum : int { + kPartitionArrayFieldNumber = 2, + kStatusFieldNumber = 1, + }; + // repeated .milvus.grpc.PartitionParam partition_array = 2; + int partition_array_size() const; + void clear_partition_array(); + ::milvus::grpc::PartitionParam* mutable_partition_array(int index); + ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::milvus::grpc::PartitionParam >* + mutable_partition_array(); + const ::milvus::grpc::PartitionParam& partition_array(int index) const; + ::milvus::grpc::PartitionParam* add_partition_array(); + const ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::milvus::grpc::PartitionParam >& + partition_array() const; + + // .milvus.grpc.Status status = 1; + bool has_status() const; + void clear_status(); + const ::milvus::grpc::Status& status() const; + ::milvus::grpc::Status* release_status(); + ::milvus::grpc::Status* mutable_status(); + void set_allocated_status(::milvus::grpc::Status* status); + + // @@protoc_insertion_point(class_scope:milvus.grpc.PartitionList) + private: + class _Internal; + + ::PROTOBUF_NAMESPACE_ID::internal::InternalMetadataWithArena _internal_metadata_; + ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::milvus::grpc::PartitionParam > partition_array_; + ::milvus::grpc::Status* status_; + mutable ::PROTOBUF_NAMESPACE_ID::internal::CachedSize _cached_size_; + friend struct ::TableStruct_milvus_2eproto; +}; +// ------------------------------------------------------------------- + class Range : public ::PROTOBUF_NAMESPACE_ID::Message /* @@protoc_insertion_point(class_definition:milvus.grpc.Range) */ { public: @@ -634,7 +1093,7 @@ class Range : &_Range_default_instance_); } static constexpr int kIndexInFileMessages = - 3; + 6; friend void swap(Range& a, Range& b) { a.Swap(&b); @@ -784,7 +1243,7 @@ class RowRecord : &_RowRecord_default_instance_); } static constexpr int kIndexInFileMessages = - 4; + 7; friend void swap(RowRecord& a, RowRecord& b) { a.Swap(&b); @@ -862,8 +1321,7 @@ class RowRecord : void clear_vector_data(); float vector_data(int index) const; void set_vector_data(int index, float value); - //void add_vector_data(float value); - void add_vector_data(std::vector::const_iterator begin, std::vector::const_iterator end); + void add_vector_data(float value); const ::PROTOBUF_NAMESPACE_ID::RepeatedField< float >& vector_data() const; ::PROTOBUF_NAMESPACE_ID::RepeatedField< float >* @@ -923,7 +1381,7 @@ class InsertParam : &_InsertParam_default_instance_); } static constexpr int kIndexInFileMessages = - 5; + 8; friend void swap(InsertParam& a, InsertParam& b) { a.Swap(&b); @@ -997,6 +1455,7 @@ class InsertParam : kRowRecordArrayFieldNumber = 2, kRowIdArrayFieldNumber = 3, kTableNameFieldNumber = 1, + kPartitionTagFieldNumber = 4, }; // repeated .milvus.grpc.RowRecord row_record_array = 2; int row_record_array_size() const; @@ -1014,9 +1473,7 @@ class InsertParam : void clear_row_id_array(); ::PROTOBUF_NAMESPACE_ID::int64 row_id_array(int index) const; void set_row_id_array(int index, ::PROTOBUF_NAMESPACE_ID::int64 value); - //void add_row_id_array(::PROTOBUF_NAMESPACE_ID::int64 value); - void add_row_id_array(std::vector<::PROTOBUF_NAMESPACE_ID::int64>::const_iterator begin, - std::vector<::PROTOBUF_NAMESPACE_ID::int64>::const_iterator end); + void add_row_id_array(::PROTOBUF_NAMESPACE_ID::int64 value); const ::PROTOBUF_NAMESPACE_ID::RepeatedField< ::PROTOBUF_NAMESPACE_ID::int64 >& row_id_array() const; ::PROTOBUF_NAMESPACE_ID::RepeatedField< ::PROTOBUF_NAMESPACE_ID::int64 >* @@ -1033,6 +1490,17 @@ class InsertParam : std::string* release_table_name(); void set_allocated_table_name(std::string* table_name); + // string partition_tag = 4; + void clear_partition_tag(); + const std::string& partition_tag() const; + void set_partition_tag(const std::string& value); + void set_partition_tag(std::string&& value); + void set_partition_tag(const char* value); + void set_partition_tag(const char* value, size_t size); + std::string* mutable_partition_tag(); + std::string* release_partition_tag(); + void set_allocated_partition_tag(std::string* partition_tag); + // @@protoc_insertion_point(class_scope:milvus.grpc.InsertParam) private: class _Internal; @@ -1042,6 +1510,7 @@ class InsertParam : ::PROTOBUF_NAMESPACE_ID::RepeatedField< ::PROTOBUF_NAMESPACE_ID::int64 > row_id_array_; mutable std::atomic _row_id_array_cached_byte_size_; ::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr table_name_; + ::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr partition_tag_; mutable ::PROTOBUF_NAMESPACE_ID::internal::CachedSize _cached_size_; friend struct ::TableStruct_milvus_2eproto; }; @@ -1089,7 +1558,7 @@ class VectorIds : &_VectorIds_default_instance_); } static constexpr int kIndexInFileMessages = - 6; + 9; friend void swap(VectorIds& a, VectorIds& b) { a.Swap(&b); @@ -1237,7 +1706,7 @@ class SearchParam : &_SearchParam_default_instance_); } static constexpr int kIndexInFileMessages = - 7; + 10; friend void swap(SearchParam& a, SearchParam& b) { a.Swap(&b); @@ -1310,6 +1779,7 @@ class SearchParam : enum : int { kQueryRecordArrayFieldNumber = 2, kQueryRangeArrayFieldNumber = 3, + kPartitionTagArrayFieldNumber = 6, kTableNameFieldNumber = 1, kTopkFieldNumber = 4, kNprobeFieldNumber = 5, @@ -1336,6 +1806,23 @@ class SearchParam : const ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::milvus::grpc::Range >& query_range_array() const; + // repeated string partition_tag_array = 6; + int partition_tag_array_size() const; + void clear_partition_tag_array(); + const std::string& partition_tag_array(int index) const; + std::string* mutable_partition_tag_array(int index); + void set_partition_tag_array(int index, const std::string& value); + void set_partition_tag_array(int index, std::string&& value); + void set_partition_tag_array(int index, const char* value); + void set_partition_tag_array(int index, const char* value, size_t size); + std::string* add_partition_tag_array(); + void add_partition_tag_array(const std::string& value); + void add_partition_tag_array(std::string&& value); + void add_partition_tag_array(const char* value); + void add_partition_tag_array(const char* value, size_t size); + const ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField& partition_tag_array() const; + ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField* mutable_partition_tag_array(); + // string table_name = 1; void clear_table_name(); const std::string& table_name() const; @@ -1364,6 +1851,7 @@ class SearchParam : ::PROTOBUF_NAMESPACE_ID::internal::InternalMetadataWithArena _internal_metadata_; ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::milvus::grpc::RowRecord > query_record_array_; ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::milvus::grpc::Range > query_range_array_; + ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField partition_tag_array_; ::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr table_name_; ::PROTOBUF_NAMESPACE_ID::int64 topk_; ::PROTOBUF_NAMESPACE_ID::int64 nprobe_; @@ -1414,7 +1902,7 @@ class SearchInFilesParam : &_SearchInFilesParam_default_instance_); } static constexpr int kIndexInFileMessages = - 8; + 11; friend void swap(SearchInFilesParam& a, SearchInFilesParam& b) { a.Swap(&b); @@ -1567,7 +2055,7 @@ class TopKQueryResult : &_TopKQueryResult_default_instance_); } static constexpr int kIndexInFileMessages = - 9; + 12; friend void swap(TopKQueryResult& a, TopKQueryResult& b) { a.Swap(&b); @@ -1648,9 +2136,7 @@ class TopKQueryResult : void clear_ids(); ::PROTOBUF_NAMESPACE_ID::int64 ids(int index) const; void set_ids(int index, ::PROTOBUF_NAMESPACE_ID::int64 value); -// void add_ids(::PROTOBUF_NAMESPACE_ID::int64 value); - void add_ids(std::vector<::PROTOBUF_NAMESPACE_ID::int64>::const_iterator begin, - std::vector<::PROTOBUF_NAMESPACE_ID::int64>::const_iterator end); + void add_ids(::PROTOBUF_NAMESPACE_ID::int64 value); const ::PROTOBUF_NAMESPACE_ID::RepeatedField< ::PROTOBUF_NAMESPACE_ID::int64 >& ids() const; ::PROTOBUF_NAMESPACE_ID::RepeatedField< ::PROTOBUF_NAMESPACE_ID::int64 >* @@ -1661,8 +2147,7 @@ class TopKQueryResult : void clear_distances(); float distances(int index) const; void set_distances(int index, float value); -// void add_distances(float value); - void add_distances(std::vector::const_iterator begin, std::vector::const_iterator end); + void add_distances(float value); const ::PROTOBUF_NAMESPACE_ID::RepeatedField< float >& distances() const; ::PROTOBUF_NAMESPACE_ID::RepeatedField< float >* @@ -1739,7 +2224,7 @@ class StringReply : &_StringReply_default_instance_); } static constexpr int kIndexInFileMessages = - 10; + 13; friend void swap(StringReply& a, StringReply& b) { a.Swap(&b); @@ -1886,7 +2371,7 @@ class BoolReply : &_BoolReply_default_instance_); } static constexpr int kIndexInFileMessages = - 11; + 14; friend void swap(BoolReply& a, BoolReply& b) { a.Swap(&b); @@ -2027,7 +2512,7 @@ class TableRowCount : &_TableRowCount_default_instance_); } static constexpr int kIndexInFileMessages = - 12; + 15; friend void swap(TableRowCount& a, TableRowCount& b) { a.Swap(&b); @@ -2168,7 +2653,7 @@ class Command : &_Command_default_instance_); } static constexpr int kIndexInFileMessages = - 13; + 16; friend void swap(Command& a, Command& b) { a.Swap(&b); @@ -2305,7 +2790,7 @@ class Index : &_Index_default_instance_); } static constexpr int kIndexInFileMessages = - 14; + 17; friend void swap(Index& a, Index& b) { a.Swap(&b); @@ -2443,7 +2928,7 @@ class IndexParam : &_IndexParam_default_instance_); } static constexpr int kIndexInFileMessages = - 15; + 18; friend void swap(IndexParam& a, IndexParam& b) { a.Swap(&b); @@ -2558,23 +3043,23 @@ class IndexParam : }; // ------------------------------------------------------------------- -class DeleteByRangeParam : - public ::PROTOBUF_NAMESPACE_ID::Message /* @@protoc_insertion_point(class_definition:milvus.grpc.DeleteByRangeParam) */ { +class DeleteByDateParam : + public ::PROTOBUF_NAMESPACE_ID::Message /* @@protoc_insertion_point(class_definition:milvus.grpc.DeleteByDateParam) */ { public: - DeleteByRangeParam(); - virtual ~DeleteByRangeParam(); + DeleteByDateParam(); + virtual ~DeleteByDateParam(); - DeleteByRangeParam(const DeleteByRangeParam& from); - DeleteByRangeParam(DeleteByRangeParam&& from) noexcept - : DeleteByRangeParam() { + DeleteByDateParam(const DeleteByDateParam& from); + DeleteByDateParam(DeleteByDateParam&& from) noexcept + : DeleteByDateParam() { *this = ::std::move(from); } - inline DeleteByRangeParam& operator=(const DeleteByRangeParam& from) { + inline DeleteByDateParam& operator=(const DeleteByDateParam& from) { CopyFrom(from); return *this; } - inline DeleteByRangeParam& operator=(DeleteByRangeParam&& from) noexcept { + inline DeleteByDateParam& operator=(DeleteByDateParam&& from) noexcept { if (GetArenaNoVirtual() == from.GetArenaNoVirtual()) { if (this != &from) InternalSwap(&from); } else { @@ -2592,37 +3077,37 @@ class DeleteByRangeParam : static const ::PROTOBUF_NAMESPACE_ID::Reflection* GetReflection() { return GetMetadataStatic().reflection; } - static const DeleteByRangeParam& default_instance(); + static const DeleteByDateParam& default_instance(); static void InitAsDefaultInstance(); // FOR INTERNAL USE ONLY - static inline const DeleteByRangeParam* internal_default_instance() { - return reinterpret_cast( - &_DeleteByRangeParam_default_instance_); + static inline const DeleteByDateParam* internal_default_instance() { + return reinterpret_cast( + &_DeleteByDateParam_default_instance_); } static constexpr int kIndexInFileMessages = - 16; + 19; - friend void swap(DeleteByRangeParam& a, DeleteByRangeParam& b) { + friend void swap(DeleteByDateParam& a, DeleteByDateParam& b) { a.Swap(&b); } - inline void Swap(DeleteByRangeParam* other) { + inline void Swap(DeleteByDateParam* other) { if (other == this) return; InternalSwap(other); } // implements Message ---------------------------------------------- - inline DeleteByRangeParam* New() const final { - return CreateMaybeMessage(nullptr); + inline DeleteByDateParam* New() const final { + return CreateMaybeMessage(nullptr); } - DeleteByRangeParam* New(::PROTOBUF_NAMESPACE_ID::Arena* arena) const final { - return CreateMaybeMessage(arena); + DeleteByDateParam* New(::PROTOBUF_NAMESPACE_ID::Arena* arena) const final { + return CreateMaybeMessage(arena); } void CopyFrom(const ::PROTOBUF_NAMESPACE_ID::Message& from) final; void MergeFrom(const ::PROTOBUF_NAMESPACE_ID::Message& from) final; - void CopyFrom(const DeleteByRangeParam& from); - void MergeFrom(const DeleteByRangeParam& from); + void CopyFrom(const DeleteByDateParam& from); + void MergeFrom(const DeleteByDateParam& from); PROTOBUF_ATTRIBUTE_REINITIALIZES void Clear() final; bool IsInitialized() const final; @@ -2643,10 +3128,10 @@ class DeleteByRangeParam : inline void SharedCtor(); inline void SharedDtor(); void SetCachedSize(int size) const final; - void InternalSwap(DeleteByRangeParam* other); + void InternalSwap(DeleteByDateParam* other); friend class ::PROTOBUF_NAMESPACE_ID::internal::AnyMetadata; static ::PROTOBUF_NAMESPACE_ID::StringPiece FullMessageName() { - return "milvus.grpc.DeleteByRangeParam"; + return "milvus.grpc.DeleteByDateParam"; } private: inline ::PROTOBUF_NAMESPACE_ID::Arena* GetArenaNoVirtual() const { @@ -2693,7 +3178,7 @@ class DeleteByRangeParam : ::milvus::grpc::Range* mutable_range(); void set_allocated_range(::milvus::grpc::Range* range); - // @@protoc_insertion_point(class_scope:milvus.grpc.DeleteByRangeParam) + // @@protoc_insertion_point(class_scope:milvus.grpc.DeleteByDateParam) private: class _Internal; @@ -2767,6 +3252,61 @@ inline void TableName::set_allocated_table_name(std::string* table_name) { // ------------------------------------------------------------------- +// PartitionName + +// string partition_name = 1; +inline void PartitionName::clear_partition_name() { + partition_name_.ClearToEmptyNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); +} +inline const std::string& PartitionName::partition_name() const { + // @@protoc_insertion_point(field_get:milvus.grpc.PartitionName.partition_name) + return partition_name_.GetNoArena(); +} +inline void PartitionName::set_partition_name(const std::string& value) { + + partition_name_.SetNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), value); + // @@protoc_insertion_point(field_set:milvus.grpc.PartitionName.partition_name) +} +inline void PartitionName::set_partition_name(std::string&& value) { + + partition_name_.SetNoArena( + &::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), ::std::move(value)); + // @@protoc_insertion_point(field_set_rvalue:milvus.grpc.PartitionName.partition_name) +} +inline void PartitionName::set_partition_name(const char* value) { + GOOGLE_DCHECK(value != nullptr); + + partition_name_.SetNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), ::std::string(value)); + // @@protoc_insertion_point(field_set_char:milvus.grpc.PartitionName.partition_name) +} +inline void PartitionName::set_partition_name(const char* value, size_t size) { + + partition_name_.SetNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), + ::std::string(reinterpret_cast(value), size)); + // @@protoc_insertion_point(field_set_pointer:milvus.grpc.PartitionName.partition_name) +} +inline std::string* PartitionName::mutable_partition_name() { + + // @@protoc_insertion_point(field_mutable:milvus.grpc.PartitionName.partition_name) + return partition_name_.MutableNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); +} +inline std::string* PartitionName::release_partition_name() { + // @@protoc_insertion_point(field_release:milvus.grpc.PartitionName.partition_name) + + return partition_name_.ReleaseNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); +} +inline void PartitionName::set_allocated_partition_name(std::string* partition_name) { + if (partition_name != nullptr) { + + } else { + + } + partition_name_.SetAllocatedNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), partition_name); + // @@protoc_insertion_point(field_set_allocated:milvus.grpc.PartitionName.partition_name) +} + +// ------------------------------------------------------------------- + // TableNameList // .milvus.grpc.Status status = 1; @@ -3023,6 +3563,242 @@ inline void TableSchema::set_metric_type(::PROTOBUF_NAMESPACE_ID::int32 value) { // ------------------------------------------------------------------- +// PartitionParam + +// string table_name = 1; +inline void PartitionParam::clear_table_name() { + table_name_.ClearToEmptyNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); +} +inline const std::string& PartitionParam::table_name() const { + // @@protoc_insertion_point(field_get:milvus.grpc.PartitionParam.table_name) + return table_name_.GetNoArena(); +} +inline void PartitionParam::set_table_name(const std::string& value) { + + table_name_.SetNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), value); + // @@protoc_insertion_point(field_set:milvus.grpc.PartitionParam.table_name) +} +inline void PartitionParam::set_table_name(std::string&& value) { + + table_name_.SetNoArena( + &::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), ::std::move(value)); + // @@protoc_insertion_point(field_set_rvalue:milvus.grpc.PartitionParam.table_name) +} +inline void PartitionParam::set_table_name(const char* value) { + GOOGLE_DCHECK(value != nullptr); + + table_name_.SetNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), ::std::string(value)); + // @@protoc_insertion_point(field_set_char:milvus.grpc.PartitionParam.table_name) +} +inline void PartitionParam::set_table_name(const char* value, size_t size) { + + table_name_.SetNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), + ::std::string(reinterpret_cast(value), size)); + // @@protoc_insertion_point(field_set_pointer:milvus.grpc.PartitionParam.table_name) +} +inline std::string* PartitionParam::mutable_table_name() { + + // @@protoc_insertion_point(field_mutable:milvus.grpc.PartitionParam.table_name) + return table_name_.MutableNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); +} +inline std::string* PartitionParam::release_table_name() { + // @@protoc_insertion_point(field_release:milvus.grpc.PartitionParam.table_name) + + return table_name_.ReleaseNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); +} +inline void PartitionParam::set_allocated_table_name(std::string* table_name) { + if (table_name != nullptr) { + + } else { + + } + table_name_.SetAllocatedNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), table_name); + // @@protoc_insertion_point(field_set_allocated:milvus.grpc.PartitionParam.table_name) +} + +// string partition_name = 2; +inline void PartitionParam::clear_partition_name() { + partition_name_.ClearToEmptyNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); +} +inline const std::string& PartitionParam::partition_name() const { + // @@protoc_insertion_point(field_get:milvus.grpc.PartitionParam.partition_name) + return partition_name_.GetNoArena(); +} +inline void PartitionParam::set_partition_name(const std::string& value) { + + partition_name_.SetNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), value); + // @@protoc_insertion_point(field_set:milvus.grpc.PartitionParam.partition_name) +} +inline void PartitionParam::set_partition_name(std::string&& value) { + + partition_name_.SetNoArena( + &::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), ::std::move(value)); + // @@protoc_insertion_point(field_set_rvalue:milvus.grpc.PartitionParam.partition_name) +} +inline void PartitionParam::set_partition_name(const char* value) { + GOOGLE_DCHECK(value != nullptr); + + partition_name_.SetNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), ::std::string(value)); + // @@protoc_insertion_point(field_set_char:milvus.grpc.PartitionParam.partition_name) +} +inline void PartitionParam::set_partition_name(const char* value, size_t size) { + + partition_name_.SetNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), + ::std::string(reinterpret_cast(value), size)); + // @@protoc_insertion_point(field_set_pointer:milvus.grpc.PartitionParam.partition_name) +} +inline std::string* PartitionParam::mutable_partition_name() { + + // @@protoc_insertion_point(field_mutable:milvus.grpc.PartitionParam.partition_name) + return partition_name_.MutableNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); +} +inline std::string* PartitionParam::release_partition_name() { + // @@protoc_insertion_point(field_release:milvus.grpc.PartitionParam.partition_name) + + return partition_name_.ReleaseNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); +} +inline void PartitionParam::set_allocated_partition_name(std::string* partition_name) { + if (partition_name != nullptr) { + + } else { + + } + partition_name_.SetAllocatedNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), partition_name); + // @@protoc_insertion_point(field_set_allocated:milvus.grpc.PartitionParam.partition_name) +} + +// string tag = 3; +inline void PartitionParam::clear_tag() { + tag_.ClearToEmptyNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); +} +inline const std::string& PartitionParam::tag() const { + // @@protoc_insertion_point(field_get:milvus.grpc.PartitionParam.tag) + return tag_.GetNoArena(); +} +inline void PartitionParam::set_tag(const std::string& value) { + + tag_.SetNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), value); + // @@protoc_insertion_point(field_set:milvus.grpc.PartitionParam.tag) +} +inline void PartitionParam::set_tag(std::string&& value) { + + tag_.SetNoArena( + &::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), ::std::move(value)); + // @@protoc_insertion_point(field_set_rvalue:milvus.grpc.PartitionParam.tag) +} +inline void PartitionParam::set_tag(const char* value) { + GOOGLE_DCHECK(value != nullptr); + + tag_.SetNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), ::std::string(value)); + // @@protoc_insertion_point(field_set_char:milvus.grpc.PartitionParam.tag) +} +inline void PartitionParam::set_tag(const char* value, size_t size) { + + tag_.SetNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), + ::std::string(reinterpret_cast(value), size)); + // @@protoc_insertion_point(field_set_pointer:milvus.grpc.PartitionParam.tag) +} +inline std::string* PartitionParam::mutable_tag() { + + // @@protoc_insertion_point(field_mutable:milvus.grpc.PartitionParam.tag) + return tag_.MutableNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); +} +inline std::string* PartitionParam::release_tag() { + // @@protoc_insertion_point(field_release:milvus.grpc.PartitionParam.tag) + + return tag_.ReleaseNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); +} +inline void PartitionParam::set_allocated_tag(std::string* tag) { + if (tag != nullptr) { + + } else { + + } + tag_.SetAllocatedNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), tag); + // @@protoc_insertion_point(field_set_allocated:milvus.grpc.PartitionParam.tag) +} + +// ------------------------------------------------------------------- + +// PartitionList + +// .milvus.grpc.Status status = 1; +inline bool PartitionList::has_status() const { + return this != internal_default_instance() && status_ != nullptr; +} +inline const ::milvus::grpc::Status& PartitionList::status() const { + const ::milvus::grpc::Status* p = status_; + // @@protoc_insertion_point(field_get:milvus.grpc.PartitionList.status) + return p != nullptr ? *p : *reinterpret_cast( + &::milvus::grpc::_Status_default_instance_); +} +inline ::milvus::grpc::Status* PartitionList::release_status() { + // @@protoc_insertion_point(field_release:milvus.grpc.PartitionList.status) + + ::milvus::grpc::Status* temp = status_; + status_ = nullptr; + return temp; +} +inline ::milvus::grpc::Status* PartitionList::mutable_status() { + + if (status_ == nullptr) { + auto* p = CreateMaybeMessage<::milvus::grpc::Status>(GetArenaNoVirtual()); + status_ = p; + } + // @@protoc_insertion_point(field_mutable:milvus.grpc.PartitionList.status) + return status_; +} +inline void PartitionList::set_allocated_status(::milvus::grpc::Status* status) { + ::PROTOBUF_NAMESPACE_ID::Arena* message_arena = GetArenaNoVirtual(); + if (message_arena == nullptr) { + delete reinterpret_cast< ::PROTOBUF_NAMESPACE_ID::MessageLite*>(status_); + } + if (status) { + ::PROTOBUF_NAMESPACE_ID::Arena* submessage_arena = nullptr; + if (message_arena != submessage_arena) { + status = ::PROTOBUF_NAMESPACE_ID::internal::GetOwnedMessage( + message_arena, status, submessage_arena); + } + + } else { + + } + status_ = status; + // @@protoc_insertion_point(field_set_allocated:milvus.grpc.PartitionList.status) +} + +// repeated .milvus.grpc.PartitionParam partition_array = 2; +inline int PartitionList::partition_array_size() const { + return partition_array_.size(); +} +inline void PartitionList::clear_partition_array() { + partition_array_.Clear(); +} +inline ::milvus::grpc::PartitionParam* PartitionList::mutable_partition_array(int index) { + // @@protoc_insertion_point(field_mutable:milvus.grpc.PartitionList.partition_array) + return partition_array_.Mutable(index); +} +inline ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::milvus::grpc::PartitionParam >* +PartitionList::mutable_partition_array() { + // @@protoc_insertion_point(field_mutable_list:milvus.grpc.PartitionList.partition_array) + return &partition_array_; +} +inline const ::milvus::grpc::PartitionParam& PartitionList::partition_array(int index) const { + // @@protoc_insertion_point(field_get:milvus.grpc.PartitionList.partition_array) + return partition_array_.Get(index); +} +inline ::milvus::grpc::PartitionParam* PartitionList::add_partition_array() { + // @@protoc_insertion_point(field_add:milvus.grpc.PartitionList.partition_array) + return partition_array_.Add(); +} +inline const ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::milvus::grpc::PartitionParam >& +PartitionList::partition_array() const { + // @@protoc_insertion_point(field_list:milvus.grpc.PartitionList.partition_array) + return partition_array_; +} + +// ------------------------------------------------------------------- + // Range // string start_value = 1; @@ -3146,14 +3922,9 @@ inline void RowRecord::set_vector_data(int index, float value) { vector_data_.Set(index, value); // @@protoc_insertion_point(field_set:milvus.grpc.RowRecord.vector_data) } -//inline void RowRecord::add_vector_data(float value) { -// vector_data_.Add(value); -// // @@protoc_insertion_point(field_add:milvus.grpc.RowRecord.vector_data) -//} -inline void RowRecord::add_vector_data(std::vector::const_iterator begin, - std::vector::const_iterator end) { - vector_data_.Add(begin, end); - // @@protoc_insertion_point(field_add:milvus.grpc.RowRecord.vector_data) +inline void RowRecord::add_vector_data(float value) { + vector_data_.Add(value); + // @@protoc_insertion_point(field_add:milvus.grpc.RowRecord.vector_data) } inline const ::PROTOBUF_NAMESPACE_ID::RepeatedField< float >& RowRecord::vector_data() const { @@ -3266,14 +4037,9 @@ inline void InsertParam::set_row_id_array(int index, ::PROTOBUF_NAMESPACE_ID::in row_id_array_.Set(index, value); // @@protoc_insertion_point(field_set:milvus.grpc.InsertParam.row_id_array) } -//inline void InsertParam::add_row_id_array(::PROTOBUF_NAMESPACE_ID::int64 value) { -// row_id_array_.Add(value); -// // @@protoc_insertion_point(field_add:milvus.grpc.InsertParam.row_id_array) -//} -inline void InsertParam::add_row_id_array(std::vector<::PROTOBUF_NAMESPACE_ID::int64>::const_iterator begin, - std::vector<::PROTOBUF_NAMESPACE_ID::int64>::const_iterator end) { - row_id_array_.Add(begin, end); - // @@protoc_insertion_point(field_add:milvus.grpc.InsertParam.row_id_array) +inline void InsertParam::add_row_id_array(::PROTOBUF_NAMESPACE_ID::int64 value) { + row_id_array_.Add(value); + // @@protoc_insertion_point(field_add:milvus.grpc.InsertParam.row_id_array) } inline const ::PROTOBUF_NAMESPACE_ID::RepeatedField< ::PROTOBUF_NAMESPACE_ID::int64 >& InsertParam::row_id_array() const { @@ -3286,6 +4052,57 @@ InsertParam::mutable_row_id_array() { return &row_id_array_; } +// string partition_tag = 4; +inline void InsertParam::clear_partition_tag() { + partition_tag_.ClearToEmptyNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); +} +inline const std::string& InsertParam::partition_tag() const { + // @@protoc_insertion_point(field_get:milvus.grpc.InsertParam.partition_tag) + return partition_tag_.GetNoArena(); +} +inline void InsertParam::set_partition_tag(const std::string& value) { + + partition_tag_.SetNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), value); + // @@protoc_insertion_point(field_set:milvus.grpc.InsertParam.partition_tag) +} +inline void InsertParam::set_partition_tag(std::string&& value) { + + partition_tag_.SetNoArena( + &::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), ::std::move(value)); + // @@protoc_insertion_point(field_set_rvalue:milvus.grpc.InsertParam.partition_tag) +} +inline void InsertParam::set_partition_tag(const char* value) { + GOOGLE_DCHECK(value != nullptr); + + partition_tag_.SetNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), ::std::string(value)); + // @@protoc_insertion_point(field_set_char:milvus.grpc.InsertParam.partition_tag) +} +inline void InsertParam::set_partition_tag(const char* value, size_t size) { + + partition_tag_.SetNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), + ::std::string(reinterpret_cast(value), size)); + // @@protoc_insertion_point(field_set_pointer:milvus.grpc.InsertParam.partition_tag) +} +inline std::string* InsertParam::mutable_partition_tag() { + + // @@protoc_insertion_point(field_mutable:milvus.grpc.InsertParam.partition_tag) + return partition_tag_.MutableNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); +} +inline std::string* InsertParam::release_partition_tag() { + // @@protoc_insertion_point(field_release:milvus.grpc.InsertParam.partition_tag) + + return partition_tag_.ReleaseNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); +} +inline void InsertParam::set_allocated_partition_tag(std::string* partition_tag) { + if (partition_tag != nullptr) { + + } else { + + } + partition_tag_.SetAllocatedNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), partition_tag); + // @@protoc_insertion_point(field_set_allocated:milvus.grpc.InsertParam.partition_tag) +} + // ------------------------------------------------------------------- // VectorIds @@ -3508,6 +4325,71 @@ inline void SearchParam::set_nprobe(::PROTOBUF_NAMESPACE_ID::int64 value) { // @@protoc_insertion_point(field_set:milvus.grpc.SearchParam.nprobe) } +// repeated string partition_tag_array = 6; +inline int SearchParam::partition_tag_array_size() const { + return partition_tag_array_.size(); +} +inline void SearchParam::clear_partition_tag_array() { + partition_tag_array_.Clear(); +} +inline const std::string& SearchParam::partition_tag_array(int index) const { + // @@protoc_insertion_point(field_get:milvus.grpc.SearchParam.partition_tag_array) + return partition_tag_array_.Get(index); +} +inline std::string* SearchParam::mutable_partition_tag_array(int index) { + // @@protoc_insertion_point(field_mutable:milvus.grpc.SearchParam.partition_tag_array) + return partition_tag_array_.Mutable(index); +} +inline void SearchParam::set_partition_tag_array(int index, const std::string& value) { + // @@protoc_insertion_point(field_set:milvus.grpc.SearchParam.partition_tag_array) + partition_tag_array_.Mutable(index)->assign(value); +} +inline void SearchParam::set_partition_tag_array(int index, std::string&& value) { + // @@protoc_insertion_point(field_set:milvus.grpc.SearchParam.partition_tag_array) + partition_tag_array_.Mutable(index)->assign(std::move(value)); +} +inline void SearchParam::set_partition_tag_array(int index, const char* value) { + GOOGLE_DCHECK(value != nullptr); + partition_tag_array_.Mutable(index)->assign(value); + // @@protoc_insertion_point(field_set_char:milvus.grpc.SearchParam.partition_tag_array) +} +inline void SearchParam::set_partition_tag_array(int index, const char* value, size_t size) { + partition_tag_array_.Mutable(index)->assign( + reinterpret_cast(value), size); + // @@protoc_insertion_point(field_set_pointer:milvus.grpc.SearchParam.partition_tag_array) +} +inline std::string* SearchParam::add_partition_tag_array() { + // @@protoc_insertion_point(field_add_mutable:milvus.grpc.SearchParam.partition_tag_array) + return partition_tag_array_.Add(); +} +inline void SearchParam::add_partition_tag_array(const std::string& value) { + partition_tag_array_.Add()->assign(value); + // @@protoc_insertion_point(field_add:milvus.grpc.SearchParam.partition_tag_array) +} +inline void SearchParam::add_partition_tag_array(std::string&& value) { + partition_tag_array_.Add(std::move(value)); + // @@protoc_insertion_point(field_add:milvus.grpc.SearchParam.partition_tag_array) +} +inline void SearchParam::add_partition_tag_array(const char* value) { + GOOGLE_DCHECK(value != nullptr); + partition_tag_array_.Add()->assign(value); + // @@protoc_insertion_point(field_add_char:milvus.grpc.SearchParam.partition_tag_array) +} +inline void SearchParam::add_partition_tag_array(const char* value, size_t size) { + partition_tag_array_.Add()->assign(reinterpret_cast(value), size); + // @@protoc_insertion_point(field_add_pointer:milvus.grpc.SearchParam.partition_tag_array) +} +inline const ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField& +SearchParam::partition_tag_array() const { + // @@protoc_insertion_point(field_list:milvus.grpc.SearchParam.partition_tag_array) + return partition_tag_array_; +} +inline ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField* +SearchParam::mutable_partition_tag_array() { + // @@protoc_insertion_point(field_mutable_list:milvus.grpc.SearchParam.partition_tag_array) + return &partition_tag_array_; +} + // ------------------------------------------------------------------- // SearchInFilesParam @@ -3706,14 +4588,9 @@ inline void TopKQueryResult::set_ids(int index, ::PROTOBUF_NAMESPACE_ID::int64 v ids_.Set(index, value); // @@protoc_insertion_point(field_set:milvus.grpc.TopKQueryResult.ids) } -//inline void TopKQueryResult::add_ids(::PROTOBUF_NAMESPACE_ID::int64 value) { -// ids_.Add(value); -// // @@protoc_insertion_point(field_add:milvus.grpc.TopKQueryResult.ids) -//} -inline void TopKQueryResult::add_ids(std::vector<::PROTOBUF_NAMESPACE_ID::int64>::const_iterator begin, - std::vector<::PROTOBUF_NAMESPACE_ID::int64>::const_iterator end) { - ids_.Add(begin,end); - // @@protoc_insertion_point(field_add:milvus.grpc.TopKQueryResult.ids) +inline void TopKQueryResult::add_ids(::PROTOBUF_NAMESPACE_ID::int64 value) { + ids_.Add(value); + // @@protoc_insertion_point(field_add:milvus.grpc.TopKQueryResult.ids) } inline const ::PROTOBUF_NAMESPACE_ID::RepeatedField< ::PROTOBUF_NAMESPACE_ID::int64 >& TopKQueryResult::ids() const { @@ -3741,13 +4618,9 @@ inline void TopKQueryResult::set_distances(int index, float value) { distances_.Set(index, value); // @@protoc_insertion_point(field_set:milvus.grpc.TopKQueryResult.distances) } -//inline void TopKQueryResult::add_distances(float value) { -// distances_.Add(value); -// // @@protoc_insertion_point(field_add:milvus.grpc.TopKQueryResult.distances) -//} -inline void TopKQueryResult::add_distances(std::vector::const_iterator begin, std::vector::const_iterator end) { - distances_.Add(begin, end); - // @@protoc_insertion_point(field_add:milvus.grpc.TopKQueryResult.distances) +inline void TopKQueryResult::add_distances(float value) { + distances_.Add(value); + // @@protoc_insertion_point(field_add:milvus.grpc.TopKQueryResult.distances) } inline const ::PROTOBUF_NAMESPACE_ID::RepeatedField< float >& TopKQueryResult::distances() const { @@ -4226,41 +5099,41 @@ inline void IndexParam::set_allocated_index(::milvus::grpc::Index* index) { // ------------------------------------------------------------------- -// DeleteByRangeParam +// DeleteByDateParam // .milvus.grpc.Range range = 1; -inline bool DeleteByRangeParam::has_range() const { +inline bool DeleteByDateParam::has_range() const { return this != internal_default_instance() && range_ != nullptr; } -inline void DeleteByRangeParam::clear_range() { +inline void DeleteByDateParam::clear_range() { if (GetArenaNoVirtual() == nullptr && range_ != nullptr) { delete range_; } range_ = nullptr; } -inline const ::milvus::grpc::Range& DeleteByRangeParam::range() const { +inline const ::milvus::grpc::Range& DeleteByDateParam::range() const { const ::milvus::grpc::Range* p = range_; - // @@protoc_insertion_point(field_get:milvus.grpc.DeleteByRangeParam.range) + // @@protoc_insertion_point(field_get:milvus.grpc.DeleteByDateParam.range) return p != nullptr ? *p : *reinterpret_cast( &::milvus::grpc::_Range_default_instance_); } -inline ::milvus::grpc::Range* DeleteByRangeParam::release_range() { - // @@protoc_insertion_point(field_release:milvus.grpc.DeleteByRangeParam.range) +inline ::milvus::grpc::Range* DeleteByDateParam::release_range() { + // @@protoc_insertion_point(field_release:milvus.grpc.DeleteByDateParam.range) ::milvus::grpc::Range* temp = range_; range_ = nullptr; return temp; } -inline ::milvus::grpc::Range* DeleteByRangeParam::mutable_range() { +inline ::milvus::grpc::Range* DeleteByDateParam::mutable_range() { if (range_ == nullptr) { auto* p = CreateMaybeMessage<::milvus::grpc::Range>(GetArenaNoVirtual()); range_ = p; } - // @@protoc_insertion_point(field_mutable:milvus.grpc.DeleteByRangeParam.range) + // @@protoc_insertion_point(field_mutable:milvus.grpc.DeleteByDateParam.range) return range_; } -inline void DeleteByRangeParam::set_allocated_range(::milvus::grpc::Range* range) { +inline void DeleteByDateParam::set_allocated_range(::milvus::grpc::Range* range) { ::PROTOBUF_NAMESPACE_ID::Arena* message_arena = GetArenaNoVirtual(); if (message_arena == nullptr) { delete range_; @@ -4276,58 +5149,58 @@ inline void DeleteByRangeParam::set_allocated_range(::milvus::grpc::Range* range } range_ = range; - // @@protoc_insertion_point(field_set_allocated:milvus.grpc.DeleteByRangeParam.range) + // @@protoc_insertion_point(field_set_allocated:milvus.grpc.DeleteByDateParam.range) } // string table_name = 2; -inline void DeleteByRangeParam::clear_table_name() { +inline void DeleteByDateParam::clear_table_name() { table_name_.ClearToEmptyNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); } -inline const std::string& DeleteByRangeParam::table_name() const { - // @@protoc_insertion_point(field_get:milvus.grpc.DeleteByRangeParam.table_name) +inline const std::string& DeleteByDateParam::table_name() const { + // @@protoc_insertion_point(field_get:milvus.grpc.DeleteByDateParam.table_name) return table_name_.GetNoArena(); } -inline void DeleteByRangeParam::set_table_name(const std::string& value) { +inline void DeleteByDateParam::set_table_name(const std::string& value) { table_name_.SetNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), value); - // @@protoc_insertion_point(field_set:milvus.grpc.DeleteByRangeParam.table_name) + // @@protoc_insertion_point(field_set:milvus.grpc.DeleteByDateParam.table_name) } -inline void DeleteByRangeParam::set_table_name(std::string&& value) { +inline void DeleteByDateParam::set_table_name(std::string&& value) { table_name_.SetNoArena( &::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), ::std::move(value)); - // @@protoc_insertion_point(field_set_rvalue:milvus.grpc.DeleteByRangeParam.table_name) + // @@protoc_insertion_point(field_set_rvalue:milvus.grpc.DeleteByDateParam.table_name) } -inline void DeleteByRangeParam::set_table_name(const char* value) { +inline void DeleteByDateParam::set_table_name(const char* value) { GOOGLE_DCHECK(value != nullptr); table_name_.SetNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), ::std::string(value)); - // @@protoc_insertion_point(field_set_char:milvus.grpc.DeleteByRangeParam.table_name) + // @@protoc_insertion_point(field_set_char:milvus.grpc.DeleteByDateParam.table_name) } -inline void DeleteByRangeParam::set_table_name(const char* value, size_t size) { +inline void DeleteByDateParam::set_table_name(const char* value, size_t size) { table_name_.SetNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), ::std::string(reinterpret_cast(value), size)); - // @@protoc_insertion_point(field_set_pointer:milvus.grpc.DeleteByRangeParam.table_name) + // @@protoc_insertion_point(field_set_pointer:milvus.grpc.DeleteByDateParam.table_name) } -inline std::string* DeleteByRangeParam::mutable_table_name() { +inline std::string* DeleteByDateParam::mutable_table_name() { - // @@protoc_insertion_point(field_mutable:milvus.grpc.DeleteByRangeParam.table_name) + // @@protoc_insertion_point(field_mutable:milvus.grpc.DeleteByDateParam.table_name) return table_name_.MutableNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); } -inline std::string* DeleteByRangeParam::release_table_name() { - // @@protoc_insertion_point(field_release:milvus.grpc.DeleteByRangeParam.table_name) +inline std::string* DeleteByDateParam::release_table_name() { + // @@protoc_insertion_point(field_release:milvus.grpc.DeleteByDateParam.table_name) return table_name_.ReleaseNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); } -inline void DeleteByRangeParam::set_allocated_table_name(std::string* table_name) { +inline void DeleteByDateParam::set_allocated_table_name(std::string* table_name) { if (table_name != nullptr) { } else { } table_name_.SetAllocatedNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), table_name); - // @@protoc_insertion_point(field_set_allocated:milvus.grpc.DeleteByRangeParam.table_name) + // @@protoc_insertion_point(field_set_allocated:milvus.grpc.DeleteByDateParam.table_name) } #ifdef __GNUC__ @@ -4365,6 +5238,12 @@ inline void DeleteByRangeParam::set_allocated_table_name(std::string* table_name // ------------------------------------------------------------------- +// ------------------------------------------------------------------- + +// ------------------------------------------------------------------- + +// ------------------------------------------------------------------- + // @@protoc_insertion_point(namespace_scope) diff --git a/core/src/grpc/milvus.proto b/core/src/grpc/milvus.proto index 74bdda3f29..4cfc1cfa0d 100644 --- a/core/src/grpc/milvus.proto +++ b/core/src/grpc/milvus.proto @@ -5,14 +5,21 @@ import "status.proto"; package milvus.grpc; /** - * @brief Table Name + * @brief Table name */ message TableName { string table_name = 1; } /** - * @brief Table Name List + * @brief Partition name + */ +message PartitionName { + string partition_name = 1; +} + +/** + * @brief Table name list */ message TableNameList { Status status = 1; @@ -20,7 +27,7 @@ message TableNameList { } /** - * @brief Table Schema + * @brief Table schema */ message TableSchema { Status status = 1; @@ -31,7 +38,24 @@ message TableSchema { } /** - * @brief Range Schema + * @brief Params of partition + */ +message PartitionParam { + string table_name = 1; + string partition_name = 2; + string tag = 3; +} + +/** + * @brief Partition list + */ +message PartitionList { + Status status = 1; + repeated PartitionParam partition_array = 2; +} + +/** + * @brief Range schema */ message Range { string start_value = 1; @@ -46,12 +70,13 @@ message RowRecord { } /** - * @brief params to be inserted + * @brief Params to be inserted */ message InsertParam { string table_name = 1; repeated RowRecord row_record_array = 2; repeated int64 row_id_array = 3; //optional + string partition_tag = 4; } /** @@ -63,7 +88,7 @@ message VectorIds { } /** - * @brief params for searching vector + * @brief Params for searching vector */ message SearchParam { string table_name = 1; @@ -71,10 +96,11 @@ message SearchParam { repeated Range query_range_array = 3; int64 topk = 4; int64 nprobe = 5; + repeated string partition_tag_array = 6; } /** - * @brief params for searching vector in files + * @brief Params for searching vector in files */ message SearchInFilesParam { repeated string file_id_array = 1; @@ -92,7 +118,7 @@ message TopKQueryResult { } /** - * @brief Server String Reply + * @brief Server string Reply */ message StringReply { Status status = 1; @@ -116,7 +142,7 @@ message TableRowCount { } /** - * @brief Give Server Command + * @brief Give server Command */ message Command { string cmd = 1; @@ -142,169 +168,173 @@ message IndexParam { } /** - * @brief table name and range for DeleteByRange + * @brief table name and range for DeleteByDate */ -message DeleteByRangeParam { +message DeleteByDateParam { Range range = 1; string table_name = 2; } service MilvusService { /** - * @brief Create table method + * @brief This method is used to create table * - * This method is used to create table - * - * @param param, use to provide table information to be created. + * @param TableSchema, use to provide table information to be created. * + * @return Status */ rpc CreateTable(TableSchema) returns (Status){} /** - * @brief Test table existence method + * @brief This method is used to test table existence. * - * This method is used to test table existence. - * - * @param table_name, table name is going to be tested. + * @param TableName, table name is going to be tested. * + * @return BoolReply */ rpc HasTable(TableName) returns (BoolReply) {} /** - * @brief Delete table method + * @brief This method is used to get table schema. * - * This method is used to delete table. + * @param TableName, target table name. * - * @param table_name, table name is going to be deleted. - * - */ - rpc DropTable(TableName) returns (Status) {} - - /** - * @brief Build index by table method - * - * This method is used to build index by table in sync mode. - * - * @param table_name, table is going to be built index. - * - */ - rpc CreateIndex(IndexParam) returns (Status) {} - - /** - * @brief Add vector array to table - * - * This method is used to add vector array to table. - * - * @param table_name, table_name is inserted. - * @param record_array, vector array is inserted. - * - * @return vector id array - */ - rpc Insert(InsertParam) returns (VectorIds) {} - - /** - * @brief Query vector - * - * This method is used to query vector in table. - * - * @param table_name, table_name is queried. - * @param query_record_array, all vector are going to be queried. - * @param query_range_array, optional ranges for conditional search. If not specified, search whole table - * @param topk, how many similarity vectors will be searched. - * - * @return query result array. - */ - rpc Search(SearchParam) returns (TopKQueryResult) {} - - /** - * @brief Internal use query interface - * - * This method is used to query vector in specified files. - * - * @param file_id_array, specified files id array, queried. - * @param query_record_array, all vector are going to be queried. - * @param query_range_array, optional ranges for conditional search. If not specified, search whole table - * @param topk, how many similarity vectors will be searched. - * - * @return query result array. - */ - rpc SearchInFiles(SearchInFilesParam) returns (TopKQueryResult) {} - - /** - * @brief Get table schema - * - * This method is used to get table schema. - * - * @param table_name, target table name. - * - * @return table schema + * @return TableSchema */ rpc DescribeTable(TableName) returns (TableSchema) {} /** - * @brief Get table schema + * @brief This method is used to get table schema. * - * This method is used to get table schema. + * @param TableName, target table name. * - * @param table_name, target table name. - * - * @return table schema + * @return TableRowCount */ rpc CountTable(TableName) returns (TableRowCount) {} /** - * @brief List all tables in database + * @brief This method is used to list all tables. * - * This method is used to list all tables. + * @param Command, dummy parameter. * - * - * @return table names. + * @return TableNameList */ rpc ShowTables(Command) returns (TableNameList) {} /** - * @brief Give the server status + * @brief This method is used to delete table. * - * This method is used to give the server status. + * @param TableName, table name is going to be deleted. * - * @return Server status. + * @return TableNameList + */ + rpc DropTable(TableName) returns (Status) {} + + /** + * @brief This method is used to build index by table in sync mode. + * + * @param IndexParam, index paramters. + * + * @return Status + */ + rpc CreateIndex(IndexParam) returns (Status) {} + + /** + * @brief This method is used to describe index + * + * @param TableName, target table name. + * + * @return IndexParam + */ + rpc DescribeIndex(TableName) returns (IndexParam) {} + + /** + * @brief This method is used to drop index + * + * @param TableName, target table name. + * + * @return Status + */ + rpc DropIndex(TableName) returns (Status) {} + + /** + * @brief This method is used to create partition + * + * @param PartitionParam, partition parameters. + * + * @return Status + */ + rpc CreatePartition(PartitionParam) returns (Status) {} + + /** + * @brief This method is used to show partition information + * + * @param TableName, target table name. + * + * @return PartitionList + */ + rpc ShowPartitions(TableName) returns (PartitionList) {} + + /** + * @brief This method is used to drop partition + * + * @param PartitionParam, target partition. + * + * @return Status + */ + rpc DropPartition(PartitionParam) returns (Status) {} + + /** + * @brief This method is used to add vector array to table. + * + * @param InsertParam, insert parameters. + * + * @return VectorIds + */ + rpc Insert(InsertParam) returns (VectorIds) {} + + /** + * @brief This method is used to query vector in table. + * + * @param SearchParam, search parameters. + * + * @return TopKQueryResult + */ + rpc Search(SearchParam) returns (TopKQueryResult) {} + + /** + * @brief This method is used to query vector in specified files. + * + * @param SearchInFilesParam, search in files paremeters. + * + * @return TopKQueryResult + */ + rpc SearchInFiles(SearchInFilesParam) returns (TopKQueryResult) {} + + /** + * @brief This method is used to give the server status. + * + * @param Command, command string + * + * @return StringReply */ rpc Cmd(Command) returns (StringReply) {} /** - * @brief delete table by range + * @brief This method is used to delete vector by date range * - * This method is used to delete vector by range + * @param DeleteByDateParam, delete parameters. * - * @return rpc status. + * @return status */ - rpc DeleteByRange(DeleteByRangeParam) returns (Status) {} + rpc DeleteByDate(DeleteByDateParam) returns (Status) {} /** - * @brief preload table + * @brief This method is used to preload table * - * This method is used to preload table + * @param TableName, target table name. * - * @return Status. + * @return Status */ rpc PreloadTable(TableName) returns (Status) {} - - /** - * @brief describe index - * - * This method is used to describe index - * - * @return Status. - */ - rpc DescribeIndex(TableName) returns (IndexParam) {} - - /** - * @brief drop index - * - * This method is used to drop index - * - * @return Status. - */ - rpc DropIndex(TableName) returns (Status) {} - } diff --git a/core/src/index/CMakeLists.txt b/core/src/index/CMakeLists.txt index f570752b68..53453d53aa 100644 --- a/core/src/index/CMakeLists.txt +++ b/core/src/index/CMakeLists.txt @@ -18,84 +18,97 @@ #------------------------------------------------------------------------------- -cmake_minimum_required(VERSION 3.14) -message(STATUS "---------------core--------------") +cmake_minimum_required(VERSION 3.12) +message(STATUS "------------------------------KNOWHERE-----------------------------------") message(STATUS "Building using CMake version: ${CMAKE_VERSION}") -set(KNOWHERE_VERSION "0.1.0") +set(KNOWHERE_VERSION "0.6.0") string(REGEX MATCH "^[0-9]+\\.[0-9]+\\.[0-9]+" KNOWHERE_BASE_VERSION "${KNOWHERE_VERSION}") -project(knowhere VERSION "${KNOWHERE_BASE_VERSION}" LANGUAGES CUDA C CXX) +project(knowhere VERSION "${KNOWHERE_BASE_VERSION}" LANGUAGES C CXX) set(CMAKE_CXX_STANDARD 14) set(KNOWHERE_VERSION_MAJOR "${knowhere_VERSION_MAJOR}") set(KNOWHERE_VERSION_MINOR "${knowhere_VERSION_MINOR}") set(KNOWHERE_VERSION_PATCH "${knowhere_VERSION_PATCH}") -if(KNOWHERE_VERSION_MAJOR STREQUAL "" +if (KNOWHERE_VERSION_MAJOR STREQUAL "" OR KNOWHERE_VERSION_MINOR STREQUAL "" OR KNOWHERE_VERSION_PATCH STREQUAL "") message(FATAL_ERROR "Failed to determine Knowhere version from '${KNOWHERE_VERSION}'") -endif() +endif () message(STATUS "Knowhere version: " "${KNOWHERE_VERSION_MAJOR}.${KNOWHERE_VERSION_MINOR}.${KNOWHERE_VERSION_PATCH} " "(full: '${KNOWHERE_VERSION}')") # if no build build type is specified, default to release builds -if(NOT CMAKE_BUILD_TYPE) +if (NOT CMAKE_BUILD_TYPE) set(CMAKE_BUILD_TYPE Release) -endif(NOT CMAKE_BUILD_TYPE) +endif (NOT CMAKE_BUILD_TYPE) -if(CMAKE_BUILD_TYPE STREQUAL "Release") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O3 -fPIC -DELPP_THREAD_SAFE -fopenmp") - set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -O3") -else() - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O0 -g -fPIC -DELPP_THREAD_SAFE -fopenmp") - set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -O0 -g") -endif() -MESSAGE(STATUS "CMAKE_CXX_FLAGS" ${CMAKE_CXX_FLAGS}) - -find_package(CUDA) - -if(CMAKE_SYSTEM_PROCESSOR MATCHES "(x86)|(X86)|(amd64)|(AMD64)") +if (CMAKE_SYSTEM_PROCESSOR MATCHES "(x86)|(X86)|(amd64)|(AMD64)") message(STATUS "building milvus_engine on x86 architecture") set(KNOWHERE_BUILD_ARCH x86_64) -elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "(ppc)") +elseif (CMAKE_SYSTEM_PROCESSOR MATCHES "(ppc)") message(STATUS "building milvus_engine on ppc architecture") set(KNOWHERE_BUILD_ARCH ppc64le) -else() +else () message(WARNING "unknown processor type") message(WARNING "CMAKE_SYSTEM_PROCESSOR=${CMAKE_SYSTEM_PROCESSOR}") set(KNOWHERE_BUILD_ARCH unknown) -endif() +endif () -if(CMAKE_BUILD_TYPE STREQUAL "Release") +if (CMAKE_BUILD_TYPE STREQUAL "Release") set(BUILD_TYPE "release") -else() +else () set(BUILD_TYPE "debug") -endif() +endif () message(STATUS "Build type = ${BUILD_TYPE}") set(INDEX_SOURCE_DIR ${PROJECT_SOURCE_DIR}) set(INDEX_BINARY_DIR ${PROJECT_BINARY_DIR}) -message(STATUS "Core source dir: ${PROJECT_SOURCE_DIR}") -message(STATUS "Core binary dir: ${PROJECT_BINARY_DIR}") set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${INDEX_SOURCE_DIR}/cmake") include(ExternalProject) include(DefineOptionsCore) include(BuildUtilsCore) + +set(KNOWHERE_CPU_VERSION false) +if (MILVUS_GPU_VERSION OR KNOWHERE_GPU_VERSION) + message(STATUS "Building Knowhere GPU version") + add_compile_definitions("MILVUS_GPU_VERSION") + enable_language(CUDA) + find_package(CUDA 10 REQUIRED) + set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -Xcompiler -fPIC -std=c++11 -D_FORCE_INLINES --expt-extended-lambda") +else () + message(STATUS "Building Knowhere CPU version") + set(KNOWHERE_CPU_VERSION true) + add_compile_definitions("MILVUS_CPU_VERSION") +endif () + include(ThirdPartyPackagesCore) +if (CMAKE_BUILD_TYPE STREQUAL "Release") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O3 -fPIC -DELPP_THREAD_SAFE -fopenmp -mavx -mf16c -msse4 -mpopcnt") + if (KNOWHERE_GPU_VERSION) + set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -O3") + endif () +else () + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O3 -g -fPIC -DELPP_THREAD_SAFE -fopenmp -mavx -mf16c -msse4 -mpopcnt") + if (KNOWHERE_GPU_VERSION) + set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -O3 -g") + endif () +endif () + add_subdirectory(knowhere) if (BUILD_COVERAGE STREQUAL "ON") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fprofile-arcs -ftest-coverage") -endif() +endif () set(INDEX_INCLUDE_DIRS ${INDEX_INCLUDE_DIRS} PARENT_SCOPE) -if(BUILD_UNIT_TEST STREQUAL "ON") +if (KNOWHERE_BUILD_TESTS) add_subdirectory(unittest) -endif() +endif () config_summary() diff --git a/core/src/index/build.sh b/core/src/index/build.sh index a77a16cf0b..357ac5693a 100644 --- a/core/src/index/build.sh +++ b/core/src/index/build.sh @@ -5,7 +5,7 @@ BUILD_UNITTEST="OFF" INSTALL_PREFIX=$(pwd)/cmake_build MAKE_CLEAN="OFF" PROFILING="OFF" -BUILD_FAISS_WITH_MKL="OFF" +FAISS_WITH_MKL="OFF" USE_JFROG_CACHE="OFF" while getopts "p:d:t:uhrcgmj" arg @@ -31,7 +31,7 @@ do PROFILING="ON" ;; m) - BUILD_FAISS_WITH_MKL="ON" + FAISS_WITH_MKL="ON" ;; j) USE_JFROG_CACHE="ON" @@ -74,7 +74,7 @@ if [[ ${MAKE_CLEAN} == "ON" ]]; then -DCMAKE_BUILD_TYPE=${BUILD_TYPE} \ -DCMAKE_CUDA_COMPILER=${CUDA_COMPILER} \ -DMILVUS_ENABLE_PROFILING=${PROFILING} \ - -DBUILD_FAISS_WITH_MKL=${BUILD_FAISS_WITH_MKL} \ + -DFAISS_WITH_MKL=${FAISS_WITH_MKL} \ -DUSE_JFROG_CACHE=${USE_JFROG_CACHE} \ ../" echo ${CMAKE_CMD} diff --git a/core/src/index/cmake/BuildUtilsCore.cmake b/core/src/index/cmake/BuildUtilsCore.cmake index 626229bd7b..1bf6ba2af4 100644 --- a/core/src/index/cmake/BuildUtilsCore.cmake +++ b/core/src/index/cmake/BuildUtilsCore.cmake @@ -1,50 +1,50 @@ # Define a function that check last file modification function(Check_Last_Modify cache_check_lists_file_path working_dir last_modified_commit_id) - if(EXISTS "${working_dir}") - if(EXISTS "${cache_check_lists_file_path}") + if (EXISTS "${working_dir}") + if (EXISTS "${cache_check_lists_file_path}") set(GIT_LOG_SKIP_NUM 0) set(_MATCH_ALL ON CACHE BOOL "Match all") set(_LOOP_STATUS ON CACHE BOOL "Whether out of loop") file(STRINGS ${cache_check_lists_file_path} CACHE_IGNORE_TXT) - while(_LOOP_STATUS) - foreach(_IGNORE_ENTRY ${CACHE_IGNORE_TXT}) - if(NOT _IGNORE_ENTRY MATCHES "^[^#]+") + while (_LOOP_STATUS) + foreach (_IGNORE_ENTRY ${CACHE_IGNORE_TXT}) + if (NOT _IGNORE_ENTRY MATCHES "^[^#]+") continue() - endif() + endif () set(_MATCH_ALL OFF) execute_process(COMMAND git log --no-merges -1 --skip=${GIT_LOG_SKIP_NUM} --name-status --pretty= WORKING_DIRECTORY ${working_dir} OUTPUT_VARIABLE CHANGE_FILES) - if(NOT CHANGE_FILES STREQUAL "") + if (NOT CHANGE_FILES STREQUAL "") string(REPLACE "\n" ";" _CHANGE_FILES ${CHANGE_FILES}) - foreach(_FILE_ENTRY ${_CHANGE_FILES}) + foreach (_FILE_ENTRY ${_CHANGE_FILES}) string(REGEX MATCH "[^ \t]+$" _FILE_NAME ${_FILE_ENTRY}) execute_process(COMMAND sh -c "echo ${_FILE_NAME} | grep ${_IGNORE_ENTRY}" RESULT_VARIABLE return_code) if (return_code EQUAL 0) execute_process(COMMAND git log --no-merges -1 --skip=${GIT_LOG_SKIP_NUM} --pretty=%H WORKING_DIRECTORY ${working_dir} OUTPUT_VARIABLE LAST_MODIFIED_COMMIT_ID) - set (${last_modified_commit_id} ${LAST_MODIFIED_COMMIT_ID} PARENT_SCOPE) + set(${last_modified_commit_id} ${LAST_MODIFIED_COMMIT_ID} PARENT_SCOPE) set(_LOOP_STATUS OFF) - endif() - endforeach() - else() + endif () + endforeach () + else () set(_LOOP_STATUS OFF) - endif() - endforeach() + endif () + endforeach () - if(_MATCH_ALL) + if (_MATCH_ALL) execute_process(COMMAND git log --no-merges -1 --skip=${GIT_LOG_SKIP_NUM} --pretty=%H WORKING_DIRECTORY ${working_dir} OUTPUT_VARIABLE LAST_MODIFIED_COMMIT_ID) - set (${last_modified_commit_id} ${LAST_MODIFIED_COMMIT_ID} PARENT_SCOPE) + set(${last_modified_commit_id} ${LAST_MODIFIED_COMMIT_ID} PARENT_SCOPE) set(_LOOP_STATUS OFF) - endif() + endif () math(EXPR GIT_LOG_SKIP_NUM "${GIT_LOG_SKIP_NUM} + 1") - endwhile(_LOOP_STATUS) - else() + endwhile (_LOOP_STATUS) + else () execute_process(COMMAND git log --no-merges -1 --skip=${GIT_LOG_SKIP_NUM} --pretty=%H WORKING_DIRECTORY ${working_dir} OUTPUT_VARIABLE LAST_MODIFIED_COMMIT_ID) - set (${last_modified_commit_id} ${LAST_MODIFIED_COMMIT_ID} PARENT_SCOPE) - endif() - else() + set(${last_modified_commit_id} ${LAST_MODIFIED_COMMIT_ID} PARENT_SCOPE) + endif () + else () message(FATAL_ERROR "The directory ${working_dir} does not exist") - endif() + endif () endfunction() # Define a function that extracts a cached package @@ -83,15 +83,15 @@ endfunction() # Define a function that to create a new cached package function(ExternalProject_Create_Cache project_name package_file install_path cache_username cache_password cache_path) - if(EXISTS ${package_file}) + if (EXISTS ${package_file}) message(STATUS "Removing existing package file: ${package_file}") file(REMOVE ${package_file}) - endif() + endif () string(REGEX REPLACE "(.+)/.+$" "\\1" package_dir ${package_file}) - if(NOT EXISTS ${package_dir}) + if (NOT EXISTS ${package_dir}) file(MAKE_DIRECTORY ${package_dir}) - endif() + endif () message(STATUS "Will create cached package file: ${package_file}") @@ -116,89 +116,89 @@ function(ADD_THIRDPARTY_LIB LIB_NAME) "${one_value_args}" "${multi_value_args}" ${ARGN}) - if(ARG_UNPARSED_ARGUMENTS) + if (ARG_UNPARSED_ARGUMENTS) message(SEND_ERROR "Error: unrecognized arguments: ${ARG_UNPARSED_ARGUMENTS}") - endif() + endif () - if(ARG_STATIC_LIB AND ARG_SHARED_LIB) - if(NOT ARG_STATIC_LIB) + if (ARG_STATIC_LIB AND ARG_SHARED_LIB) + if (NOT ARG_STATIC_LIB) message(FATAL_ERROR "No static or shared library provided for ${LIB_NAME}") - endif() + endif () set(AUG_LIB_NAME "${LIB_NAME}_static") add_library(${AUG_LIB_NAME} STATIC IMPORTED) set_target_properties(${AUG_LIB_NAME} PROPERTIES IMPORTED_LOCATION "${ARG_STATIC_LIB}") - if(ARG_DEPS) + if (ARG_DEPS) set_target_properties(${AUG_LIB_NAME} PROPERTIES INTERFACE_LINK_LIBRARIES "${ARG_DEPS}") - endif() + endif () message(STATUS "Added static library dependency ${AUG_LIB_NAME}: ${ARG_STATIC_LIB}") - if(ARG_INCLUDE_DIRECTORIES) + if (ARG_INCLUDE_DIRECTORIES) set_target_properties(${AUG_LIB_NAME} PROPERTIES INTERFACE_INCLUDE_DIRECTORIES "${ARG_INCLUDE_DIRECTORIES}") - endif() + endif () set(AUG_LIB_NAME "${LIB_NAME}_shared") add_library(${AUG_LIB_NAME} SHARED IMPORTED) - if(WIN32) + if (WIN32) # Mark the ".lib" location as part of a Windows DLL set_target_properties(${AUG_LIB_NAME} PROPERTIES IMPORTED_IMPLIB "${ARG_SHARED_LIB}") - else() + else () set_target_properties(${AUG_LIB_NAME} PROPERTIES IMPORTED_LOCATION "${ARG_SHARED_LIB}") - endif() - if(ARG_DEPS) + endif () + if (ARG_DEPS) set_target_properties(${AUG_LIB_NAME} PROPERTIES INTERFACE_LINK_LIBRARIES "${ARG_DEPS}") - endif() + endif () message(STATUS "Added shared library dependency ${AUG_LIB_NAME}: ${ARG_SHARED_LIB}") - if(ARG_INCLUDE_DIRECTORIES) + if (ARG_INCLUDE_DIRECTORIES) set_target_properties(${AUG_LIB_NAME} PROPERTIES INTERFACE_INCLUDE_DIRECTORIES "${ARG_INCLUDE_DIRECTORIES}") - endif() - elseif(ARG_STATIC_LIB) + endif () + elseif (ARG_STATIC_LIB) set(AUG_LIB_NAME "${LIB_NAME}_static") add_library(${AUG_LIB_NAME} STATIC IMPORTED) set_target_properties(${AUG_LIB_NAME} PROPERTIES IMPORTED_LOCATION "${ARG_STATIC_LIB}") - if(ARG_DEPS) + if (ARG_DEPS) set_target_properties(${AUG_LIB_NAME} PROPERTIES INTERFACE_LINK_LIBRARIES "${ARG_DEPS}") - endif() + endif () message(STATUS "Added static library dependency ${AUG_LIB_NAME}: ${ARG_STATIC_LIB}") - if(ARG_INCLUDE_DIRECTORIES) + if (ARG_INCLUDE_DIRECTORIES) set_target_properties(${AUG_LIB_NAME} PROPERTIES INTERFACE_INCLUDE_DIRECTORIES "${ARG_INCLUDE_DIRECTORIES}") - endif() - elseif(ARG_SHARED_LIB) + endif () + elseif (ARG_SHARED_LIB) set(AUG_LIB_NAME "${LIB_NAME}_shared") add_library(${AUG_LIB_NAME} SHARED IMPORTED) - if(WIN32) + if (WIN32) # Mark the ".lib" location as part of a Windows DLL set_target_properties(${AUG_LIB_NAME} PROPERTIES IMPORTED_IMPLIB "${ARG_SHARED_LIB}") - else() + else () set_target_properties(${AUG_LIB_NAME} PROPERTIES IMPORTED_LOCATION "${ARG_SHARED_LIB}") - endif() + endif () message(STATUS "Added shared library dependency ${AUG_LIB_NAME}: ${ARG_SHARED_LIB}") - if(ARG_DEPS) + if (ARG_DEPS) set_target_properties(${AUG_LIB_NAME} PROPERTIES INTERFACE_LINK_LIBRARIES "${ARG_DEPS}") - endif() - if(ARG_INCLUDE_DIRECTORIES) + endif () + if (ARG_INCLUDE_DIRECTORIES) set_target_properties(${AUG_LIB_NAME} PROPERTIES INTERFACE_INCLUDE_DIRECTORIES "${ARG_INCLUDE_DIRECTORIES}") - endif() - else() + endif () + else () message(FATAL_ERROR "No static or shared library provided for ${LIB_NAME}") - endif() + endif () endfunction() diff --git a/core/src/index/cmake/DefineOptionsCore.cmake b/core/src/index/cmake/DefineOptionsCore.cmake index cf5e8ea328..787a9c484f 100644 --- a/core/src/index/cmake/DefineOptionsCore.cmake +++ b/core/src/index/cmake/DefineOptionsCore.cmake @@ -13,16 +13,16 @@ macro(define_option name description default) endmacro() function(list_join lst glue out) - if("${${lst}}" STREQUAL "") + if ("${${lst}}" STREQUAL "") set(${out} "" PARENT_SCOPE) return() - endif() + endif () list(GET ${lst} 0 joined) list(REMOVE_AT ${lst} 0) - foreach(item ${${lst}}) + foreach (item ${${lst}}) set(joined "${joined}${glue}${item}") - endforeach() + endforeach () set(${out} ${joined} PARENT_SCOPE) endfunction() @@ -35,22 +35,31 @@ macro(define_option_string name description default) set("${name}_OPTION_ENUM" ${ARGN}) list_join("${name}_OPTION_ENUM" "|" "${name}_OPTION_ENUM") - if(NOT ("${${name}_OPTION_ENUM}" STREQUAL "")) + if (NOT ("${${name}_OPTION_ENUM}" STREQUAL "")) set_property(CACHE ${name} PROPERTY STRINGS ${ARGN}) - endif() + endif () endmacro() +#---------------------------------------------------------------------- +set_option_category("GPU version") + +if (MILVUS_GPU_VERSION) + define_option(KNOWHERE_GPU_VERSION "Build GPU version" ON) +else () + define_option(KNOWHERE_GPU_VERSION "Build GPU version" OFF) +endif () + #---------------------------------------------------------------------- set_option_category("Thirdparty") -set(KNOWHERE_DEPENDENCY_SOURCE_DEFAULT "AUTO") +set(KNOWHERE_DEPENDENCY_SOURCE_DEFAULT "BUNDLED") define_option_string(KNOWHERE_DEPENDENCY_SOURCE - "Method to use for acquiring KNOWHERE's build dependencies" - "${KNOWHERE_DEPENDENCY_SOURCE_DEFAULT}" - "AUTO" - "BUNDLED" - "SYSTEM") + "Method to use for acquiring KNOWHERE's build dependencies" + "${KNOWHERE_DEPENDENCY_SOURCE_DEFAULT}" + "AUTO" + "BUNDLED" + "SYSTEM") define_option(KNOWHERE_VERBOSE_THIRDPARTY_BUILD "Show output from ExternalProjects rather than just logging to files" ON) @@ -70,27 +79,16 @@ define_option(KNOWHERE_WITH_FAISS "Build with FAISS library" ON) define_option(KNOWHERE_WITH_FAISS_GPU_VERSION "Build with FAISS GPU version" ON) -define_option(KNOWHERE_WITH_OPENBLAS "Build with OpenBLAS library" ON) - -#---------------------------------------------------------------------- -if(MSVC) - set_option_category("MSVC") - - define_option(MSVC_LINK_VERBOSE - "Pass verbose linking options when linking libraries and executables" - OFF) - - define_option(KNOWHERE_USE_STATIC_CRT "Build KNOWHERE with statically linked CRT" OFF) -endif() +define_option(FAISS_WITH_MKL "Build FAISS with MKL" OFF) #---------------------------------------------------------------------- set_option_category("Test and benchmark") if (BUILD_UNIT_TEST) define_option(KNOWHERE_BUILD_TESTS "Build the KNOWHERE googletest unit tests" ON) -else() +else () define_option(KNOWHERE_BUILD_TESTS "Build the KNOWHERE googletest unit tests" OFF) -endif(BUILD_UNIT_TEST) +endif (BUILD_UNIT_TEST) #---------------------------------------------------------------------- macro(config_summary) @@ -102,12 +100,12 @@ macro(config_summary) message(STATUS " Generator: ${CMAKE_GENERATOR}") message(STATUS " Build type: ${CMAKE_BUILD_TYPE}") message(STATUS " Source directory: ${CMAKE_CURRENT_SOURCE_DIR}") - if(${CMAKE_EXPORT_COMPILE_COMMANDS}) + if (${CMAKE_EXPORT_COMPILE_COMMANDS}) message( STATUS " Compile commands: ${INDEX_BINARY_DIR}/compile_commands.json") - endif() + endif () - foreach(category ${KNOWHERE_OPTION_CATEGORIES}) + foreach (category ${KNOWHERE_OPTION_CATEGORIES}) message(STATUS) message(STATUS "${category} options:") @@ -115,50 +113,50 @@ macro(config_summary) set(option_names ${KNOWHERE_${category}_OPTION_NAMES}) set(max_value_length 0) - foreach(name ${option_names}) + foreach (name ${option_names}) string(LENGTH "\"${${name}}\"" value_length) - if(${max_value_length} LESS ${value_length}) + if (${max_value_length} LESS ${value_length}) set(max_value_length ${value_length}) - endif() - endforeach() + endif () + endforeach () - foreach(name ${option_names}) - if("${${name}_OPTION_TYPE}" STREQUAL "string") + foreach (name ${option_names}) + if ("${${name}_OPTION_TYPE}" STREQUAL "string") set(value "\"${${name}}\"") - else() + else () set(value "${${name}}") - endif() + endif () set(default ${${name}_OPTION_DEFAULT}) set(description ${${name}_OPTION_DESCRIPTION}) string(LENGTH ${description} description_length) - if(${description_length} LESS 70) + if (${description_length} LESS 70) string( SUBSTRING " " ${description_length} -1 description_padding) - else() + else () set(description_padding " ") - endif() + endif () set(comment "[${name}]") - if("${value}" STREQUAL "${default}") + if ("${value}" STREQUAL "${default}") set(comment "[default] ${comment}") - endif() + endif () - if(NOT ("${${name}_OPTION_ENUM}" STREQUAL "")) + if (NOT ("${${name}_OPTION_ENUM}" STREQUAL "")) set(comment "${comment} [${${name}_OPTION_ENUM}]") - endif() + endif () string( SUBSTRING "${value} " 0 ${max_value_length} value) message(STATUS " ${description} ${description_padding} ${value} ${comment}") - endforeach() + endforeach () - endforeach() + endforeach () endmacro() diff --git a/core/src/index/cmake/FindArrow.cmake b/core/src/index/cmake/FindArrow.cmake new file mode 100644 index 0000000000..fdf7c1437f --- /dev/null +++ b/core/src/index/cmake/FindArrow.cmake @@ -0,0 +1,431 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# - Find Arrow (arrow/api.h, libarrow.a, libarrow.so) +# This module defines +# ARROW_FOUND, whether Arrow has been found +# ARROW_FULL_SO_VERSION, full shared object version of found Arrow "100.0.0" +# ARROW_IMPORT_LIB, path to libarrow's import library (Windows only) +# ARROW_INCLUDE_DIR, directory containing headers +# ARROW_LIBS, deprecated. Use ARROW_LIB_DIR instead +# ARROW_LIB_DIR, directory containing Arrow libraries +# ARROW_SHARED_IMP_LIB, deprecated. Use ARROW_IMPORT_LIB instead +# ARROW_SHARED_LIB, path to libarrow's shared library +# ARROW_SO_VERSION, shared object version of found Arrow such as "100" +# ARROW_STATIC_LIB, path to libarrow.a +# ARROW_VERSION, version of found Arrow +# ARROW_VERSION_MAJOR, major version of found Arrow +# ARROW_VERSION_MINOR, minor version of found Arrow +# ARROW_VERSION_PATCH, patch version of found Arrow + +include(FindPkgConfig) +include(FindPackageHandleStandardArgs) + +set(ARROW_SEARCH_LIB_PATH_SUFFIXES) +if(CMAKE_LIBRARY_ARCHITECTURE) + list(APPEND ARROW_SEARCH_LIB_PATH_SUFFIXES "lib/${CMAKE_LIBRARY_ARCHITECTURE}") +endif() +list(APPEND ARROW_SEARCH_LIB_PATH_SUFFIXES + "lib64" + "lib32" + "lib" + "bin") +set(ARROW_CONFIG_SUFFIXES + "_RELEASE" + "_RELWITHDEBINFO" + "_MINSIZEREL" + "_DEBUG" + "") +if(CMAKE_BUILD_TYPE) + string(TOUPPER ${CMAKE_BUILD_TYPE} ARROW_CONFIG_SUFFIX_PREFERRED) + set(ARROW_CONFIG_SUFFIX_PREFERRED "_${ARROW_CONFIG_SUFFIX_PREFERRED}") + list(INSERT ARROW_CONFIG_SUFFIXES 0 "${ARROW_CONFIG_SUFFIX_PREFERRED}") +endif() + +if(NOT DEFINED ARROW_MSVC_STATIC_LIB_SUFFIX) + if(MSVC) + set(ARROW_MSVC_STATIC_LIB_SUFFIX "_static") + else() + set(ARROW_MSVC_STATIC_LIB_SUFFIX "") + endif() +endif() + +# Internal function. +# +# Set shared library name for ${base_name} to ${output_variable}. +# +# Example: +# arrow_build_shared_library_name(ARROW_SHARED_LIBRARY_NAME arrow) +# # -> ARROW_SHARED_LIBRARY_NAME=libarrow.so on Linux +# # -> ARROW_SHARED_LIBRARY_NAME=libarrow.dylib on macOS +# # -> ARROW_SHARED_LIBRARY_NAME=arrow.dll with MSVC on Windows +# # -> ARROW_SHARED_LIBRARY_NAME=libarrow.dll with MinGW on Windows +function(arrow_build_shared_library_name output_variable base_name) + set(${output_variable} + "${CMAKE_SHARED_LIBRARY_PREFIX}${base_name}${CMAKE_SHARED_LIBRARY_SUFFIX}" + PARENT_SCOPE) +endfunction() + +# Internal function. +# +# Set import library name for ${base_name} to ${output_variable}. +# This is useful only for MSVC build. Import library is used only +# with MSVC build. +# +# Example: +# arrow_build_import_library_name(ARROW_IMPORT_LIBRARY_NAME arrow) +# # -> ARROW_IMPORT_LIBRARY_NAME=arrow on Linux (meaningless) +# # -> ARROW_IMPORT_LIBRARY_NAME=arrow on macOS (meaningless) +# # -> ARROW_IMPORT_LIBRARY_NAME=arrow.lib with MSVC on Windows +# # -> ARROW_IMPORT_LIBRARY_NAME=libarrow.dll.a with MinGW on Windows +function(arrow_build_import_library_name output_variable base_name) + set(${output_variable} + "${CMAKE_IMPORT_LIBRARY_PREFIX}${base_name}${CMAKE_IMPORT_LIBRARY_SUFFIX}" + PARENT_SCOPE) +endfunction() + +# Internal function. +# +# Set static library name for ${base_name} to ${output_variable}. +# +# Example: +# arrow_build_static_library_name(ARROW_STATIC_LIBRARY_NAME arrow) +# # -> ARROW_STATIC_LIBRARY_NAME=libarrow.a on Linux +# # -> ARROW_STATIC_LIBRARY_NAME=libarrow.a on macOS +# # -> ARROW_STATIC_LIBRARY_NAME=arrow.lib with MSVC on Windows +# # -> ARROW_STATIC_LIBRARY_NAME=libarrow.dll.a with MinGW on Windows +function(arrow_build_static_library_name output_variable base_name) + set( + ${output_variable} + "${CMAKE_STATIC_LIBRARY_PREFIX}${base_name}${ARROW_MSVC_STATIC_LIB_SUFFIX}${CMAKE_STATIC_LIBRARY_SUFFIX}" + PARENT_SCOPE) +endfunction() + +# Internal function. +# +# Set macro value for ${macro_name} in ${header_content} to ${output_variable}. +# +# Example: +# arrow_extract_macro_value(version_major +# "ARROW_VERSION_MAJOR" +# "#define ARROW_VERSION_MAJOR 1.0.0") +# # -> version_major=1.0.0 +function(arrow_extract_macro_value output_variable macro_name header_content) + string(REGEX MATCH "#define +${macro_name} +[^\r\n]+" macro_definition + "${header_content}") + string(REGEX + REPLACE "^#define +${macro_name} +(.+)$" "\\1" macro_value "${macro_definition}") + set(${output_variable} "${macro_value}" PARENT_SCOPE) +endfunction() + +# Internal macro only for arrow_find_package. +# +# Find package in HOME. +macro(arrow_find_package_home) + find_path(${prefix}_include_dir "${header_path}" + PATHS "${home}" + PATH_SUFFIXES "include" + NO_DEFAULT_PATH) + set(include_dir "${${prefix}_include_dir}") + set(${prefix}_INCLUDE_DIR "${include_dir}" PARENT_SCOPE) + + if(MSVC) + set(CMAKE_SHARED_LIBRARY_SUFFIXES_ORIGINAL ${CMAKE_FIND_LIBRARY_SUFFIXES}) + # .dll isn't found by find_library with MSVC because .dll isn't included in + # CMAKE_FIND_LIBRARY_SUFFIXES. + list(APPEND CMAKE_FIND_LIBRARY_SUFFIXES "${CMAKE_SHARED_LIBRARY_SUFFIX}") + endif() + find_library(${prefix}_shared_lib + NAMES "${shared_lib_name}" + PATHS "${home}" + PATH_SUFFIXES ${ARROW_SEARCH_LIB_PATH_SUFFIXES} + NO_DEFAULT_PATH) + if(MSVC) + set(CMAKE_SHARED_LIBRARY_SUFFIXES ${CMAKE_FIND_LIBRARY_SUFFIXES_ORIGINAL}) + endif() + set(shared_lib "${${prefix}_shared_lib}") + set(${prefix}_SHARED_LIB "${shared_lib}" PARENT_SCOPE) + if(shared_lib) + add_library(${target_shared} SHARED IMPORTED) + set_target_properties(${target_shared} PROPERTIES IMPORTED_LOCATION "${shared_lib}") + if(include_dir) + set_target_properties(${target_shared} + PROPERTIES INTERFACE_INCLUDE_DIRECTORIES "${include_dir}") + endif() + find_library(${prefix}_import_lib + NAMES "${import_lib_name}" + PATHS "${home}" + PATH_SUFFIXES ${ARROW_SEARCH_LIB_PATH_SUFFIXES} + NO_DEFAULT_PATH) + set(import_lib "${${prefix}_import_lib}") + set(${prefix}_IMPORT_LIB "${import_lib}" PARENT_SCOPE) + if(import_lib) + set_target_properties(${target_shared} PROPERTIES IMPORTED_IMPLIB "${import_lib}") + endif() + endif() + + find_library(${prefix}_static_lib + NAMES "${static_lib_name}" + PATHS "${home}" + PATH_SUFFIXES ${ARROW_SEARCH_LIB_PATH_SUFFIXES} + NO_DEFAULT_PATH) + set(static_lib "${${prefix}_static_lib}") + set(${prefix}_STATIC_LIB "${static_lib}" PARENT_SCOPE) + if(static_lib) + add_library(${target_static} STATIC IMPORTED) + set_target_properties(${target_static} PROPERTIES IMPORTED_LOCATION "${static_lib}") + if(include_dir) + set_target_properties(${target_static} + PROPERTIES INTERFACE_INCLUDE_DIRECTORIES "${include_dir}") + endif() + endif() +endmacro() + +# Internal macro only for arrow_find_package. +# +# Find package by CMake package configuration. +macro(arrow_find_package_cmake_package_configuration) + # ARROW-5575: We need to split target files for each component + if(TARGET ${target_shared} OR TARGET ${target_static}) + set(${cmake_package_name}_FOUND TRUE) + else() + find_package(${cmake_package_name} CONFIG) + endif() + if(${cmake_package_name}_FOUND) + set(${prefix}_USE_CMAKE_PACKAGE_CONFIG TRUE PARENT_SCOPE) + if(TARGET ${target_shared}) + foreach(suffix ${ARROW_CONFIG_SUFFIXES}) + get_target_property(shared_lib ${target_shared} IMPORTED_LOCATION${suffix}) + if(shared_lib) + # Remove shared library version: + # libarrow.so.100.0.0 -> libarrow.so + # Because ARROW_HOME and pkg-config approaches don't add + # shared library version. + string(REGEX + REPLACE "(${CMAKE_SHARED_LIBRARY_SUFFIX})[.0-9]+$" "\\1" shared_lib + "${shared_lib}") + set(${prefix}_SHARED_LIB "${shared_lib}" PARENT_SCOPE) + break() + endif() + endforeach() + endif() + if(TARGET ${target_static}) + foreach(suffix ${ARROW_CONFIG_SUFFIXES}) + get_target_property(static_lib ${target_static} IMPORTED_LOCATION${suffix}) + if(static_lib) + set(${prefix}_STATIC_LIB "${static_lib}" PARENT_SCOPE) + break() + endif() + endforeach() + endif() + endif() +endmacro() + +# Internal macro only for arrow_find_package. +# +# Find package by pkg-config. +macro(arrow_find_package_pkg_config) + pkg_check_modules(${prefix}_PC ${pkg_config_name}) + if(${prefix}_PC_FOUND) + set(${prefix}_USE_PKG_CONFIG TRUE PARENT_SCOPE) + + set(include_dir "${${prefix}_PC_INCLUDEDIR}") + set(lib_dir "${${prefix}_PC_LIBDIR}") + set(shared_lib_paths "${${prefix}_PC_LINK_LIBRARIES}") + # Use the first shared library path as the IMPORTED_LOCATION + # for ${target_shared}. This assumes that the first shared library + # path is the shared library path for this module. + list(GET shared_lib_paths 0 first_shared_lib_path) + # Use the rest shared library paths as the INTERFACE_LINK_LIBRARIES + # for ${target_shared}. This assumes that the rest shared library + # paths are dependency library paths for this module. + list(LENGTH shared_lib_paths n_shared_lib_paths) + if(n_shared_lib_paths LESS_EQUAL 1) + set(rest_shared_lib_paths) + else() + list(SUBLIST + shared_lib_paths + 1 + -1 + rest_shared_lib_paths) + endif() + + set(${prefix}_VERSION "${${prefix}_PC_VERSION}" PARENT_SCOPE) + set(${prefix}_INCLUDE_DIR "${include_dir}" PARENT_SCOPE) + set(${prefix}_SHARED_LIB "${first_shared_lib_path}" PARENT_SCOPE) + + add_library(${target_shared} SHARED IMPORTED) + set_target_properties(${target_shared} + PROPERTIES INTERFACE_INCLUDE_DIRECTORIES + "${include_dir}" + INTERFACE_LINK_LIBRARIES + "${rest_shared_lib_paths}" + IMPORTED_LOCATION + "${first_shared_lib_path}") + + find_library(${prefix}_static_lib + NAMES "${static_lib_name}" + PATHS "${lib_dir}" + NO_DEFAULT_PATH) + set(static_lib "${${prefix}_static_lib}") + set(${prefix}_STATIC_LIB "${static_lib}" PARENT_SCOPE) + if(static_lib) + add_library(${target_static} STATIC IMPORTED) + set_target_properties(${target_static} + PROPERTIES INTERFACE_INCLUDE_DIRECTORIES "${include_dir}" + IMPORTED_LOCATION "${static_lib}") + endif() + endif() +endmacro() + +function(arrow_find_package + prefix + home + base_name + header_path + cmake_package_name + pkg_config_name) + arrow_build_shared_library_name(shared_lib_name ${base_name}) + arrow_build_import_library_name(import_lib_name ${base_name}) + arrow_build_static_library_name(static_lib_name ${base_name}) + + set(target_shared ${base_name}_shared) + set(target_static ${base_name}_static) + + if(home) + arrow_find_package_home() + set(${prefix}_FIND_APPROACH "HOME: ${home}" PARENT_SCOPE) + else() + arrow_find_package_cmake_package_configuration() + if(${cmake_package_name}_FOUND) + set(${prefix}_FIND_APPROACH + "CMake package configuration: ${cmake_package_name}" + PARENT_SCOPE) + else() + arrow_find_package_pkg_config() + set(${prefix}_FIND_APPROACH "pkg-config: ${pkg_config_name}" PARENT_SCOPE) + endif() + endif() + + if(NOT include_dir) + if(TARGET ${target_shared}) + get_target_property(include_dir ${target_shared} INTERFACE_INCLUDE_DIRECTORIES) + elseif(TARGET ${target_static}) + get_target_property(include_dir ${target_static} INTERFACE_INCLUDE_DIRECTORIES) + endif() + endif() + if(include_dir) + set(${prefix}_INCLUDE_DIR "${include_dir}" PARENT_SCOPE) + endif() + + if(shared_lib) + get_filename_component(lib_dir "${shared_lib}" DIRECTORY) + elseif(static_lib) + get_filename_component(lib_dir "${static_lib}" DIRECTORY) + else() + set(lib_dir NOTFOUND) + endif() + set(${prefix}_LIB_DIR "${lib_dir}" PARENT_SCOPE) + # For backward compatibility + set(${prefix}_LIBS "${lib_dir}" PARENT_SCOPE) +endfunction() + +if(NOT "$ENV{ARROW_HOME}" STREQUAL "") + file(TO_CMAKE_PATH "$ENV{ARROW_HOME}" ARROW_HOME) +endif() +arrow_find_package(ARROW + "${ARROW_HOME}" + arrow + arrow/api.h + Arrow + arrow) + +if(ARROW_HOME) + if(ARROW_INCLUDE_DIR) + file(READ "${ARROW_INCLUDE_DIR}/arrow/util/config.h" ARROW_CONFIG_H_CONTENT) + arrow_extract_macro_value(ARROW_VERSION_MAJOR "ARROW_VERSION_MAJOR" + "${ARROW_CONFIG_H_CONTENT}") + arrow_extract_macro_value(ARROW_VERSION_MINOR "ARROW_VERSION_MINOR" + "${ARROW_CONFIG_H_CONTENT}") + arrow_extract_macro_value(ARROW_VERSION_PATCH "ARROW_VERSION_PATCH" + "${ARROW_CONFIG_H_CONTENT}") + if("${ARROW_VERSION_MAJOR}" STREQUAL "" + OR "${ARROW_VERSION_MINOR}" STREQUAL "" + OR "${ARROW_VERSION_PATCH}" STREQUAL "") + set(ARROW_VERSION "0.0.0") + else() + set(ARROW_VERSION + "${ARROW_VERSION_MAJOR}.${ARROW_VERSION_MINOR}.${ARROW_VERSION_PATCH}") + endif() + + arrow_extract_macro_value(ARROW_SO_VERSION_QUOTED "ARROW_SO_VERSION" + "${ARROW_CONFIG_H_CONTENT}") + string(REGEX REPLACE "^\"(.+)\"$" "\\1" ARROW_SO_VERSION "${ARROW_SO_VERSION_QUOTED}") + arrow_extract_macro_value(ARROW_FULL_SO_VERSION_QUOTED "ARROW_FULL_SO_VERSION" + "${ARROW_CONFIG_H_CONTENT}") + string(REGEX + REPLACE "^\"(.+)\"$" "\\1" ARROW_FULL_SO_VERSION + "${ARROW_FULL_SO_VERSION_QUOTED}") + endif() +else() + if(ARROW_USE_CMAKE_PACKAGE_CONFIG) + find_package(Arrow CONFIG) + elseif(ARROW_USE_PKG_CONFIG) + pkg_get_variable(ARROW_SO_VERSION arrow so_version) + pkg_get_variable(ARROW_FULL_SO_VERSION arrow full_so_version) + endif() +endif() + +set(ARROW_ABI_VERSION ${ARROW_SO_VERSION}) + +mark_as_advanced(ARROW_ABI_VERSION + ARROW_CONFIG_SUFFIXES + ARROW_FULL_SO_VERSION + ARROW_IMPORT_LIB + ARROW_INCLUDE_DIR + ARROW_LIBS + ARROW_LIB_DIR + ARROW_SEARCH_LIB_PATH_SUFFIXES + ARROW_SHARED_IMP_LIB + ARROW_SHARED_LIB + ARROW_SO_VERSION + ARROW_STATIC_LIB + ARROW_VERSION + ARROW_VERSION_MAJOR + ARROW_VERSION_MINOR + ARROW_VERSION_PATCH) + +find_package_handle_standard_args(Arrow REQUIRED_VARS + # The first required variable is shown + # in the found message. So this list is + # not sorted alphabetically. + ARROW_INCLUDE_DIR + ARROW_LIB_DIR + ARROW_FULL_SO_VERSION + ARROW_SO_VERSION + VERSION_VAR + ARROW_VERSION) +set(ARROW_FOUND ${Arrow_FOUND}) + +if(Arrow_FOUND AND NOT Arrow_FIND_QUIETLY) + message(STATUS "Arrow version: ${ARROW_VERSION} (${ARROW_FIND_APPROACH})") + message(STATUS "Arrow SO and ABI version: ${ARROW_SO_VERSION}") + message(STATUS "Arrow full SO version: ${ARROW_FULL_SO_VERSION}") + message(STATUS "Found the Arrow core shared library: ${ARROW_SHARED_LIB}") + message(STATUS "Found the Arrow core import library: ${ARROW_IMPORT_LIB}") + message(STATUS "Found the Arrow core static library: ${ARROW_STATIC_LIB}") +endif() diff --git a/core/src/index/cmake/FindFAISS.cmake b/core/src/index/cmake/FindFAISS.cmake new file mode 100644 index 0000000000..e0e6337bce --- /dev/null +++ b/core/src/index/cmake/FindFAISS.cmake @@ -0,0 +1,44 @@ +set(FAISS_STATIC_LIB_NAME ${CMAKE_STATIC_LIBRARY_PREFIX}faiss${CMAKE_STATIC_LIBRARY_SUFFIX}) + +# First, find via if specified FAISS_ROOT +if (FAISS_ROOT) + find_library(FAISS_STATIC_LIB + NAMES ${FAISS_STATIC_LIB_NAME} + PATHS ${FAISS_ROOT} + PATH_SUFFIXES "lib" + NO_DEFAULT_PATH + ) + find_path(FAISS_INCLUDE_DIR + NAMES "faiss/Index.h" + PATHS ${FAISS_ROOT} + NO_DEFAULT_PATH + PATH_SUFFIXES "include" + ) +endif () + +find_package_handle_standard_args(FAISS REQUIRED_VARS FAISS_STATIC_LIB FAISS_INCLUDE_DIR) + +if (FAISS_FOUND) + if (NOT TARGET faiss) + add_library(faiss STATIC IMPORTED) + + set_target_properties( + faiss + PROPERTIES + IMPORTED_LOCATION "${FAISS_STATIC_LIB}" + INTERFACE_INCLUDE_DIRECTORIES "${FAISS_INCLUDE_DIR}" + ) + + if (FAISS_WITH_MKL) + set_target_properties( + faiss + PROPERTIES + INTERFACE_LINK_LIBRARIES "${MKL_LIBS}") + else () + set_target_properties( + faiss + PROPERTIES + INTERFACE_LINK_LIBRARIES "openblas;lapack") + endif () + endif () +endif () diff --git a/core/src/index/cmake/ThirdPartyPackagesCore.cmake b/core/src/index/cmake/ThirdPartyPackagesCore.cmake index 584283b49b..1a22a9d2be 100644 --- a/core/src/index/cmake/ThirdPartyPackagesCore.cmake +++ b/core/src/index/cmake/ThirdPartyPackagesCore.cmake @@ -16,43 +16,47 @@ set(KNOWHERE_THIRDPARTY_DEPENDENCIES - ARROW + Arrow FAISS GTest LAPACK OpenBLAS + MKL ) message(STATUS "Using ${KNOWHERE_DEPENDENCY_SOURCE} approach to find dependencies") # For each dependency, set dependency source to global default, if unset -foreach(DEPENDENCY ${KNOWHERE_THIRDPARTY_DEPENDENCIES}) - if("${${DEPENDENCY}_SOURCE}" STREQUAL "") +foreach (DEPENDENCY ${KNOWHERE_THIRDPARTY_DEPENDENCIES}) + if ("${${DEPENDENCY}_SOURCE}" STREQUAL "") set(${DEPENDENCY}_SOURCE ${KNOWHERE_DEPENDENCY_SOURCE}) - endif() -endforeach() + endif () +endforeach () macro(build_dependency DEPENDENCY_NAME) - if("${DEPENDENCY_NAME}" STREQUAL "ARROW") + if ("${DEPENDENCY_NAME}" STREQUAL "Arrow") build_arrow() - elseif("${DEPENDENCY_NAME}" STREQUAL "LAPACK") + elseif ("${DEPENDENCY_NAME}" STREQUAL "LAPACK") build_lapack() elseif ("${DEPENDENCY_NAME}" STREQUAL "GTest") build_gtest() elseif ("${DEPENDENCY_NAME}" STREQUAL "OpenBLAS") build_openblas() - elseif("${DEPENDENCY_NAME}" STREQUAL "FAISS") + elseif ("${DEPENDENCY_NAME}" STREQUAL "FAISS") build_faiss() - else() + elseif ("${DEPENDENCY_NAME}" STREQUAL "MKL") + build_mkl() + else () message(FATAL_ERROR "Unknown thirdparty dependency to build: ${DEPENDENCY_NAME}") endif () endmacro() macro(resolve_dependency DEPENDENCY_NAME) if (${DEPENDENCY_NAME}_SOURCE STREQUAL "AUTO") - #message(STATUS "Finding ${DEPENDENCY_NAME} package") - #message(STATUS "${DEPENDENCY_NAME} package not found") - build_dependency(${DEPENDENCY_NAME}) + find_package(${DEPENDENCY_NAME} MODULE) + if (NOT ${${DEPENDENCY_NAME}_FOUND}) + build_dependency(${DEPENDENCY_NAME}) + endif () elseif (${DEPENDENCY_NAME}_SOURCE STREQUAL "BUNDLED") build_dependency(${DEPENDENCY_NAME}) elseif (${DEPENDENCY_NAME}_SOURCE STREQUAL "SYSTEM") @@ -64,28 +68,28 @@ endmacro() # Identify OS if (UNIX) if (APPLE) - set (CMAKE_OS_NAME "osx" CACHE STRING "Operating system name" FORCE) + set(CMAKE_OS_NAME "osx" CACHE STRING "Operating system name" FORCE) else (APPLE) ## Check for Debian GNU/Linux ________________ - find_file (DEBIAN_FOUND debian_version debconf.conf + find_file(DEBIAN_FOUND debian_version debconf.conf PATHS /etc ) if (DEBIAN_FOUND) - set (CMAKE_OS_NAME "debian" CACHE STRING "Operating system name" FORCE) + set(CMAKE_OS_NAME "debian" CACHE STRING "Operating system name" FORCE) endif (DEBIAN_FOUND) ## Check for Fedora _________________________ - find_file (FEDORA_FOUND fedora-release + find_file(FEDORA_FOUND fedora-release PATHS /etc ) if (FEDORA_FOUND) - set (CMAKE_OS_NAME "fedora" CACHE STRING "Operating system name" FORCE) + set(CMAKE_OS_NAME "fedora" CACHE STRING "Operating system name" FORCE) endif (FEDORA_FOUND) ## Check for RedHat _________________________ - find_file (REDHAT_FOUND redhat-release inittab.RH + find_file(REDHAT_FOUND redhat-release inittab.RH PATHS /etc ) if (REDHAT_FOUND) - set (CMAKE_OS_NAME "redhat" CACHE STRING "Operating system name" FORCE) + set(CMAKE_OS_NAME "redhat" CACHE STRING "Operating system name" FORCE) endif (REDHAT_FOUND) ## Extra check for Ubuntu ____________________ if (DEBIAN_FOUND) @@ -94,18 +98,18 @@ if (UNIX) ## a first superficial inspection a system will ## be considered as Debian, which signifies an ## extra check is required. - find_file (UBUNTU_EXTRA legal issue + find_file(UBUNTU_EXTRA legal issue PATHS /etc ) if (UBUNTU_EXTRA) ## Scan contents of file - file (STRINGS ${UBUNTU_EXTRA} UBUNTU_FOUND + file(STRINGS ${UBUNTU_EXTRA} UBUNTU_FOUND REGEX Ubuntu ) ## Check result of string search if (UBUNTU_FOUND) - set (CMAKE_OS_NAME "ubuntu" CACHE STRING "Operating system name" FORCE) - set (DEBIAN_FOUND FALSE) + set(CMAKE_OS_NAME "ubuntu" CACHE STRING "Operating system name" FORCE) + set(DEBIAN_FOUND FALSE) endif (UBUNTU_FOUND) endif (UBUNTU_EXTRA) endif (DEBIAN_FOUND) @@ -119,28 +123,17 @@ set(THIRDPARTY_DIR "${INDEX_SOURCE_DIR}/thirdparty") # ---------------------------------------------------------------------- # JFrog -if(NOT DEFINED USE_JFROG_CACHE) +if (NOT DEFINED USE_JFROG_CACHE) set(USE_JFROG_CACHE "OFF") -endif() -if(USE_JFROG_CACHE STREQUAL "ON") +endif () +if (USE_JFROG_CACHE STREQUAL "ON") set(JFROG_ARTFACTORY_CACHE_URL "${JFROG_ARTFACTORY_URL}/milvus/thirdparty/cache/${CMAKE_OS_NAME}/${KNOWHERE_BUILD_ARCH}/${BUILD_TYPE}") set(THIRDPARTY_PACKAGE_CACHE "${THIRDPARTY_DIR}/cache") - if(NOT EXISTS ${THIRDPARTY_PACKAGE_CACHE}) + if (NOT EXISTS ${THIRDPARTY_PACKAGE_CACHE}) message(STATUS "Will create cached directory: ${THIRDPARTY_PACKAGE_CACHE}") file(MAKE_DIRECTORY ${THIRDPARTY_PACKAGE_CACHE}) - endif() -endif() - -macro(resolve_dependency DEPENDENCY_NAME) - if (${DEPENDENCY_NAME}_SOURCE STREQUAL "AUTO") - #disable find_package for now - build_dependency(${DEPENDENCY_NAME}) - elseif (${DEPENDENCY_NAME}_SOURCE STREQUAL "BUNDLED") - build_dependency(${DEPENDENCY_NAME}) - elseif (${DEPENDENCY_NAME}_SOURCE STREQUAL "SYSTEM") - find_package(${DEPENDENCY_NAME} REQUIRED) endif () -endmacro() +endif () # ---------------------------------------------------------------------- # ExternalProject options @@ -150,11 +143,11 @@ string(TOUPPER ${CMAKE_BUILD_TYPE} UPPERCASE_BUILD_TYPE) set(EP_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${CMAKE_CXX_FLAGS_${UPPERCASE_BUILD_TYPE}}") set(EP_C_FLAGS "${CMAKE_C_FLAGS} ${CMAKE_C_FLAGS_${UPPERCASE_BUILD_TYPE}}") -if(NOT MSVC) +if (NOT MSVC) # Set -fPIC on all external projects set(EP_CXX_FLAGS "${EP_CXX_FLAGS} -fPIC") set(EP_C_FLAGS "${EP_C_FLAGS} -fPIC") -endif() +endif () # CC/CXX environment variables are captured on the first invocation of the # builder (e.g make or ninja) instead of when CMake is invoked into to build @@ -164,13 +157,13 @@ endif() set(EP_COMMON_TOOLCHAIN -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}) -if(CMAKE_AR) +if (CMAKE_AR) set(EP_COMMON_TOOLCHAIN ${EP_COMMON_TOOLCHAIN} -DCMAKE_AR=${CMAKE_AR}) -endif() +endif () -if(CMAKE_RANLIB) +if (CMAKE_RANLIB) set(EP_COMMON_TOOLCHAIN ${EP_COMMON_TOOLCHAIN} -DCMAKE_RANLIB=${CMAKE_RANLIB}) -endif() +endif () # External projects are still able to override the following declarations. # cmake command line will favor the last defined variable when a duplicate is @@ -184,18 +177,18 @@ set(EP_COMMON_CMAKE_ARGS -DCMAKE_CXX_FLAGS=${EP_CXX_FLAGS} -DCMAKE_CXX_FLAGS_${UPPERCASE_BUILD_TYPE}=${EP_CXX_FLAGS}) -if(NOT KNOWHERE_VERBOSE_THIRDPARTY_BUILD) +if (NOT KNOWHERE_VERBOSE_THIRDPARTY_BUILD) set(EP_LOG_OPTIONS LOG_CONFIGURE 1 LOG_BUILD 1 LOG_INSTALL 1 LOG_DOWNLOAD 1) -else() +else () set(EP_LOG_OPTIONS) -endif() +endif () # Ensure that a default make is set -if("${MAKE}" STREQUAL "") - if(NOT MSVC) +if ("${MAKE}" STREQUAL "") + if (NOT MSVC) find_program(MAKE make) - endif() -endif() + endif () +endif () set(MAKE_BUILD_ARGS "-j8") @@ -212,71 +205,54 @@ find_package(Threads REQUIRED) # Read toolchain versions from cpp/thirdparty/versions.txt file(STRINGS "${THIRDPARTY_DIR}/versions.txt" TOOLCHAIN_VERSIONS_TXT) -foreach(_VERSION_ENTRY ${TOOLCHAIN_VERSIONS_TXT}) +foreach (_VERSION_ENTRY ${TOOLCHAIN_VERSIONS_TXT}) # Exclude comments - if(NOT _VERSION_ENTRY MATCHES "^[^#][A-Za-z0-9-_]+_VERSION=") + if (NOT _VERSION_ENTRY MATCHES "^[^#][A-Za-z0-9-_]+_VERSION=") continue() - endif() + endif () string(REGEX MATCH "^[^=]*" _LIB_NAME ${_VERSION_ENTRY}) string(REPLACE "${_LIB_NAME}=" "" _LIB_VERSION ${_VERSION_ENTRY}) # Skip blank or malformed lines - if(${_LIB_VERSION} STREQUAL "") + if (${_LIB_VERSION} STREQUAL "") continue() - endif() + endif () # For debugging #message(STATUS "${_LIB_NAME}: ${_LIB_VERSION}") set(${_LIB_NAME} "${_LIB_VERSION}") -endforeach() +endforeach () -if(CUSTOMIZATION) - execute_process(COMMAND wget -q --method HEAD ${FAISS_URL} RESULT_VARIABLE return_code) - message(STATUS "Check the remote cache file ${FAISS_URL}. return code = ${return_code}") - if (NOT return_code EQUAL 0) - MESSAGE(FATAL_ERROR "Can't access to ${FAISS_URL}") - else() - set(FAISS_SOURCE_URL ${FAISS_URL}) - # set(FAISS_MD5 "a589663865a8558205533c8ac414278c") - # set(FAISS_MD5 "57da9c4f599cc8fa4260488b1c96e1cc") # commit-id 6dbdf75987c34a2c853bd172ea0d384feea8358c branch-0.2.0 - # set(FAISS_MD5 "21deb1c708490ca40ecb899122c01403") # commit-id 643e48f479637fd947e7b93fa4ca72b38ecc9a39 branch-0.2.0 - # set(FAISS_MD5 "072db398351cca6e88f52d743bbb9fa0") # commit-id 3a2344d04744166af41ef1a74449d68a315bfe17 branch-0.2.1 - # set(FAISS_MD5 "c89ea8e655f5cdf58f42486f13614714") # commit-id 9c28a1cbb88f41fa03b03d7204106201ad33276b branch-0.2.1 - # set(FAISS_MD5 "87fdd86351ffcaf3f80dc26ade63c44b") # commit-id 841a156e67e8e22cd8088e1b58c00afbf2efc30b branch-0.2.1 - # set(FAISS_MD5 "f3b2ce3364c3fa7febd3aa7fdd0fe380") # commit-id 694e03458e6b69ce8a62502f71f69a614af5af8f branch-0.3.0 - # set(FAISS_MD5 "bb30722c22390ce5f6759ccb216c1b2a") # commit-id d324db297475286afe107847c7fb7a0f9dc7e90e branch-0.3.0 - # set(FAISS_MD5 "2293cdb209c3718e3b19f3edae8b32b3") # commit-id a13c1205dc52977a9ad3b33a14efa958604a8bff branch-0.3.0 - set(FAISS_MD5 "3632192f06b70e5f3fac8e2f08ced503") # commit-id fbd5b7566466c5665bb646f2b2f911c8544090f5 branch-0.3.0 - endif() -else() - set(FAISS_SOURCE_URL "https://github.com/JinHai-CN/faiss/archive/1.6.0.tar.gz") +if (DEFINED ENV{FAISS_SOURCE_URL}) + set(FAISS_SOURCE_URL "$ENV{FAISS_SOURCE_URL}") +else () + set(FAISS_SOURCE_URL "https://github.com/JinHai-CN/faiss/archive/${FAISS_VERSION}.tar.gz") set(FAISS_MD5 "b02c1a53234f5acc9bea1b0c55524f50") -endif() -message(STATUS "FAISS URL = ${FAISS_SOURCE_URL}") +endif () -if(DEFINED ENV{KNOWHERE_ARROW_URL}) +if (DEFINED ENV{KNOWHERE_ARROW_URL}) set(ARROW_SOURCE_URL "$ENV{KNOWHERE_ARROW_URL}") -else() +else () set(ARROW_SOURCE_URL "https://github.com/apache/arrow.git" ) -endif() +endif () if (DEFINED ENV{KNOWHERE_GTEST_URL}) set(GTEST_SOURCE_URL "$ENV{KNOWHERE_GTEST_URL}") else () set(GTEST_SOURCE_URL "https://github.com/google/googletest/archive/release-${GTEST_VERSION}.tar.gz") -endif() +endif () set(GTEST_MD5 "2e6fbeb6a91310a16efe181886c59596") -if(DEFINED ENV{KNOWHERE_LAPACK_URL}) +if (DEFINED ENV{KNOWHERE_LAPACK_URL}) set(LAPACK_SOURCE_URL "$ENV{KNOWHERE_LAPACK_URL}") -else() +else () set(LAPACK_SOURCE_URL "https://github.com/Reference-LAPACK/lapack/archive/${LAPACK_VERSION}.tar.gz") -endif() +endif () set(LAPACK_MD5 "96591affdbf58c450d45c1daa540dbd2") if (DEFINED ENV{KNOWHERE_OPENBLAS_URL}) @@ -284,7 +260,7 @@ if (DEFINED ENV{KNOWHERE_OPENBLAS_URL}) else () set(OPENBLAS_SOURCE_URL "https://github.com/xianyi/OpenBLAS/archive/${OPENBLAS_VERSION}.tar.gz") -endif() +endif () set(OPENBLAS_MD5 "8a110a25b819a4b94e8a9580702b6495") # ---------------------------------------------------------------------- @@ -294,10 +270,11 @@ set(ARROW_PREFIX "${INDEX_BINARY_DIR}/arrow_ep-prefix/src/arrow_ep/cpp") macro(build_arrow) message(STATUS "Building Apache ARROW-${ARROW_VERSION} from source") set(ARROW_STATIC_LIB_NAME arrow) - set(ARROW_STATIC_LIB + set(ARROW_STATIC_LIB "${ARROW_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}${ARROW_STATIC_LIB_NAME}${CMAKE_STATIC_LIBRARY_SUFFIX}" ) - set(ARROW_INCLUDE_DIR "${ARROW_PREFIX}/include") + set(ARROW_LIB_DIR "${ARROW_PREFIX}/lib") + set(ARROW_INCLUDE_DIR "${ARROW_PREFIX}/include") set(ARROW_CMAKE_ARGS ${EP_COMMON_CMAKE_ARGS} @@ -327,10 +304,10 @@ macro(build_arrow) -DBOOST_SOURCE=AUTO #try to find BOOST in the system default locations and build from source if not found ) - - if(USE_JFROG_CACHE STREQUAL "ON") + + if (USE_JFROG_CACHE STREQUAL "ON") execute_process(COMMAND sh -c "git ls-remote --heads --tags ${ARROW_SOURCE_URL} ${ARROW_VERSION} | cut -f 1" OUTPUT_VARIABLE ARROW_LAST_COMMIT_ID) - if(${ARROW_LAST_COMMIT_ID} MATCHES "^[^#][a-z0-9]+") + if (${ARROW_LAST_COMMIT_ID} MATCHES "^[^#][a-z0-9]+") string(MD5 ARROW_COMBINE_MD5 "${ARROW_LAST_COMMIT_ID}") set(ARROW_CACHE_PACKAGE_NAME "arrow_${ARROW_COMBINE_MD5}.tar.gz") set(ARROW_CACHE_URL "${JFROG_ARTFACTORY_CACHE_URL}/${ARROW_CACHE_PACKAGE_NAME}") @@ -360,18 +337,18 @@ macro(build_arrow) ) ExternalProject_Create_Cache(arrow_ep ${ARROW_CACHE_PACKAGE_PATH} "${INDEX_BINARY_DIR}/arrow_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${ARROW_CACHE_URL}) - else() + else () file(DOWNLOAD ${ARROW_CACHE_URL} ${ARROW_CACHE_PACKAGE_PATH} STATUS status) list(GET status 0 status_code) message(STATUS "DOWNLOADING FROM ${ARROW_CACHE_URL} TO ${ARROW_CACHE_PACKAGE_PATH}. STATUS = ${status_code}") if (status_code EQUAL 0) ExternalProject_Use_Cache(arrow_ep ${ARROW_CACHE_PACKAGE_PATH} ${INDEX_BINARY_DIR}) - endif() - endif() - else() + endif () + endif () + else () message(FATAL_ERROR "The last commit ID of \"${ARROW_SOURCE_URL}\" repository don't match!") - endif() - else() + endif () + else () externalproject_add(arrow_ep GIT_REPOSITORY ${ARROW_SOURCE_URL} @@ -391,31 +368,31 @@ macro(build_arrow) BUILD_BYPRODUCTS "${ARROW_STATIC_LIB}" ) - endif() + endif () - file(MAKE_DIRECTORY "${ARROW_PREFIX}/include") + file(MAKE_DIRECTORY "${ARROW_INCLUDE_DIR}") add_library(arrow STATIC IMPORTED) set_target_properties(arrow PROPERTIES IMPORTED_LOCATION "${ARROW_STATIC_LIB}" INTERFACE_INCLUDE_DIRECTORIES "${ARROW_INCLUDE_DIR}") - add_dependencies(arrow arrow_ep) + add_dependencies(arrow arrow_ep) set(JEMALLOC_PREFIX "${INDEX_BINARY_DIR}/arrow_ep-prefix/src/arrow_ep-build/jemalloc_ep-prefix/src/jemalloc_ep") add_custom_command(TARGET arrow_ep POST_BUILD - COMMAND ${CMAKE_COMMAND} -E make_directory ${ARROW_PREFIX}/lib/ - COMMAND ${CMAKE_COMMAND} -E copy ${JEMALLOC_PREFIX}/lib/libjemalloc_pic.a ${ARROW_PREFIX}/lib/ + COMMAND ${CMAKE_COMMAND} -E make_directory ${ARROW_LIB_DIR} + COMMAND ${CMAKE_COMMAND} -E copy ${JEMALLOC_PREFIX}/lib/libjemalloc_pic.a ${ARROW_LIB_DIR} DEPENDS ${JEMALLOC_PREFIX}/lib/libjemalloc_pic.a) endmacro() -if(KNOWHERE_WITH_ARROW AND NOT TARGET arrow_ep) +if (KNOWHERE_WITH_ARROW AND NOT TARGET arrow_ep) - resolve_dependency(ARROW) + resolve_dependency(Arrow) - link_directories(SYSTEM ${ARROW_PREFIX}/lib/) + link_directories(SYSTEM ${ARROW_LIB_DIR}) include_directories(SYSTEM ${ARROW_INCLUDE_DIR}) -endif() +endif () # ---------------------------------------------------------------------- # OpenBLAS @@ -429,7 +406,7 @@ macro(build_openblas) set(OPENBLAS_REAL_STATIC_LIB "${OPENBLAS_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}openblas_haswellp-r0.3.6${CMAKE_STATIC_LIBRARY_SUFFIX}") - if(USE_JFROG_CACHE STREQUAL "ON") + if (USE_JFROG_CACHE STREQUAL "ON") set(OPENBLAS_CACHE_PACKAGE_NAME "openblas_${OPENBLAS_MD5}.tar.gz") set(OPENBLAS_CACHE_URL "${JFROG_ARTFACTORY_CACHE_URL}/${OPENBLAS_CACHE_PACKAGE_NAME}") set(OPENBLAS_CACHE_PACKAGE_PATH "${THIRDPARTY_PACKAGE_CACHE}/${OPENBLAS_CACHE_PACKAGE_NAME}") @@ -456,15 +433,15 @@ macro(build_openblas) ${OPENBLAS_STATIC_LIB}) ExternalProject_Create_Cache(openblas_ep ${OPENBLAS_CACHE_PACKAGE_PATH} "${INDEX_BINARY_DIR}/openblas_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${OPENBLAS_CACHE_URL}) - else() + else () file(DOWNLOAD ${OPENBLAS_CACHE_URL} ${OPENBLAS_CACHE_PACKAGE_PATH} STATUS status) list(GET status 0 status_code) message(STATUS "DOWNLOADING FROM ${OPENBLAS_CACHE_URL} TO ${OPENBLAS_CACHE_PACKAGE_PATH}. STATUS = ${status_code}") if (status_code EQUAL 0) ExternalProject_Use_Cache(openblas_ep ${OPENBLAS_CACHE_PACKAGE_PATH} ${INDEX_BINARY_DIR}) - endif() - endif() - else() + endif () + endif () + else () externalproject_add(openblas_ep URL ${OPENBLAS_SOURCE_URL} @@ -482,7 +459,7 @@ macro(build_openblas) install BUILD_BYPRODUCTS ${OPENBLAS_STATIC_LIB}) - endif() + endif () file(MAKE_DIRECTORY "${OPENBLAS_INCLUDE_DIR}") add_library(openblas STATIC IMPORTED) @@ -511,7 +488,7 @@ macro(build_lapack) "-DCMAKE_INSTALL_PREFIX=${LAPACK_PREFIX}" -DCMAKE_INSTALL_LIBDIR=lib) - if(USE_JFROG_CACHE STREQUAL "ON") + if (USE_JFROG_CACHE STREQUAL "ON") set(LAPACK_CACHE_PACKAGE_NAME "lapack_${LAPACK_MD5}.tar.gz") set(LAPACK_CACHE_URL "${JFROG_ARTFACTORY_CACHE_URL}/${LAPACK_CACHE_PACKAGE_NAME}") set(LAPACK_CACHE_PACKAGE_PATH "${THIRDPARTY_PACKAGE_CACHE}/${LAPACK_CACHE_PACKAGE_NAME}") @@ -532,15 +509,15 @@ macro(build_lapack) ${LAPACK_STATIC_LIB}) ExternalProject_Create_Cache(lapack_ep ${LAPACK_CACHE_PACKAGE_PATH} "${INDEX_BINARY_DIR}/lapack_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${LAPACK_CACHE_URL}) - else() + else () file(DOWNLOAD ${LAPACK_CACHE_URL} ${LAPACK_CACHE_PACKAGE_PATH} STATUS status) list(GET status 0 status_code) message(STATUS "DOWNLOADING FROM ${LAPACK_CACHE_URL} TO ${LAPACK_CACHE_PACKAGE_PATH}. STATUS = ${status_code}") if (status_code EQUAL 0) ExternalProject_Use_Cache(lapack_ep ${LAPACK_CACHE_PACKAGE_PATH} ${INDEX_BINARY_DIR}) - endif() - endif() - else() + endif () + endif () + else () externalproject_add(lapack_ep URL ${LAPACK_SOURCE_URL} @@ -552,7 +529,7 @@ macro(build_lapack) ${MAKE_BUILD_ARGS} BUILD_BYPRODUCTS ${LAPACK_STATIC_LIB}) - endif() + endif () file(MAKE_DIRECTORY "${LAPACK_INCLUDE_DIR}") add_library(lapack STATIC IMPORTED) @@ -572,13 +549,13 @@ macro(build_gtest) set(GTEST_VENDORED TRUE) set(GTEST_CMAKE_CXX_FLAGS "${EP_CXX_FLAGS}") - if(APPLE) + if (APPLE) set(GTEST_CMAKE_CXX_FLAGS ${GTEST_CMAKE_CXX_FLAGS} -DGTEST_USE_OWN_TR1_TUPLE=1 -Wno-unused-value -Wno-ignored-attributes) - endif() + endif () set(GTEST_PREFIX "${INDEX_BINARY_DIR}/googletest_ep-prefix/src/googletest_ep") set(GTEST_INCLUDE_DIR "${GTEST_PREFIX}/include") @@ -597,10 +574,10 @@ macro(build_gtest) set(GMOCK_INCLUDE_DIR "${GTEST_PREFIX}/include") set(GMOCK_STATIC_LIB "${GTEST_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}gmock${CMAKE_STATIC_LIBRARY_SUFFIX}" - ) + ) - if(USE_JFROG_CACHE STREQUAL "ON") + if (USE_JFROG_CACHE STREQUAL "ON") set(GTEST_CACHE_PACKAGE_NAME "googletest_${GTEST_MD5}.tar.gz") set(GTEST_CACHE_URL "${JFROG_ARTFACTORY_CACHE_URL}/${GTEST_CACHE_PACKAGE_NAME}") set(GTEST_CACHE_PACKAGE_PATH "${THIRDPARTY_PACKAGE_CACHE}/${GTEST_CACHE_PACKAGE_NAME}") @@ -623,15 +600,15 @@ macro(build_gtest) ${EP_LOG_OPTIONS}) ExternalProject_Create_Cache(googletest_ep ${GTEST_CACHE_PACKAGE_PATH} "${INDEX_BINARY_DIR}/googletest_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${GTEST_CACHE_URL}) - else() + else () file(DOWNLOAD ${GTEST_CACHE_URL} ${GTEST_CACHE_PACKAGE_PATH} STATUS status) list(GET status 0 status_code) message(STATUS "DOWNLOADING FROM ${GTEST_CACHE_URL} TO ${GTEST_CACHE_PACKAGE_PATH}. STATUS = ${status_code}") if (status_code EQUAL 0) ExternalProject_Use_Cache(googletest_ep ${GTEST_CACHE_PACKAGE_PATH} ${INDEX_BINARY_DIR}) - endif() - endif() - else() + endif () + endif () + else () ExternalProject_Add(googletest_ep URL ${GTEST_SOURCE_URL} @@ -645,20 +622,20 @@ macro(build_gtest) CMAKE_ARGS ${GTEST_CMAKE_ARGS} ${EP_LOG_OPTIONS}) - endif() + endif () # The include directory must exist before it is referenced by a target. file(MAKE_DIRECTORY "${GTEST_INCLUDE_DIR}") add_library(gtest STATIC IMPORTED) set_target_properties(gtest - PROPERTIES IMPORTED_LOCATION "${GTEST_STATIC_LIB}" - INTERFACE_INCLUDE_DIRECTORIES "${GTEST_INCLUDE_DIR}") + PROPERTIES IMPORTED_LOCATION "${GTEST_STATIC_LIB}" + INTERFACE_INCLUDE_DIRECTORIES "${GTEST_INCLUDE_DIR}") add_library(gtest_main STATIC IMPORTED) set_target_properties(gtest_main - PROPERTIES IMPORTED_LOCATION "${GTEST_MAIN_STATIC_LIB}" - INTERFACE_INCLUDE_DIRECTORIES "${GTEST_INCLUDE_DIR}") + PROPERTIES IMPORTED_LOCATION "${GTEST_MAIN_STATIC_LIB}" + INTERFACE_INCLUDE_DIRECTORIES "${GTEST_INCLUDE_DIR}") add_library(gmock STATIC IMPORTED) set_target_properties(gmock @@ -674,46 +651,98 @@ endmacro() if (KNOWHERE_BUILD_TESTS AND NOT TARGET googletest_ep) resolve_dependency(GTest) - if(NOT GTEST_VENDORED) - endif() + if (NOT GTEST_VENDORED) + endif () # TODO: Don't use global includes but rather target_include_directories get_target_property(GTEST_INCLUDE_DIR gtest INTERFACE_INCLUDE_DIRECTORIES) link_directories(SYSTEM "${GTEST_PREFIX}/lib") include_directories(SYSTEM ${GTEST_INCLUDE_DIR}) -endif() +endif () + +# ---------------------------------------------------------------------- +# MKL + +macro(build_mkl) + + if (FAISS_WITH_MKL) + if (EXISTS "/proc/cpuinfo") + FILE(READ /proc/cpuinfo PROC_CPUINFO) + + SET(VENDOR_ID_RX "vendor_id[ \t]*:[ \t]*([a-zA-Z]+)\n") + STRING(REGEX MATCH "${VENDOR_ID_RX}" VENDOR_ID "${PROC_CPUINFO}") + STRING(REGEX REPLACE "${VENDOR_ID_RX}" "\\1" VENDOR_ID "${VENDOR_ID}") + + if (NOT ${VENDOR_ID} STREQUAL "GenuineIntel") + set(FAISS_WITH_MKL OFF) + endif () + endif () + + find_path(MKL_LIB_PATH + NAMES "libmkl_intel_ilp64.a" "libmkl_gnu_thread.a" "libmkl_core.a" + PATH_SUFFIXES "intel/compilers_and_libraries_${MKL_VERSION}/linux/mkl/lib/intel64/") + if (${MKL_LIB_PATH} STREQUAL "MKL_LIB_PATH-NOTFOUND") + message(FATAL_ERROR "Could not find MKL libraries") + endif () + message(STATUS "MKL lib path = ${MKL_LIB_PATH}") + + set(MKL_LIBS + ${MKL_LIB_PATH}/libmkl_intel_ilp64.a + ${MKL_LIB_PATH}/libmkl_gnu_thread.a + ${MKL_LIB_PATH}/libmkl_core.a + ) + endif () +endmacro() # ---------------------------------------------------------------------- # FAISS macro(build_faiss) message(STATUS "Building FAISS-${FAISS_VERSION} from source") + set(FAISS_PREFIX "${INDEX_BINARY_DIR}/faiss_ep-prefix/src/faiss_ep") set(FAISS_INCLUDE_DIR "${FAISS_PREFIX}/include") set(FAISS_STATIC_LIB "${FAISS_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}faiss${CMAKE_STATIC_LIBRARY_SUFFIX}") - set(FAISS_CONFIGURE_ARGS "--prefix=${FAISS_PREFIX}" "CFLAGS=${EP_C_FLAGS}" "CXXFLAGS=${EP_CXX_FLAGS}" - "LDFLAGS=-L${OPENBLAS_PREFIX}/lib -L${LAPACK_PREFIX}/lib -lopenblas -llapack" --without-python) - - if(${KNOWHERE_WITH_FAISS_GPU_VERSION} STREQUAL "ON") + if (FAISS_WITH_MKL) + set(FAISS_CONFIGURE_ARGS ${FAISS_CONFIGURE_ARGS} + "CPPFLAGS=-DFINTEGER=long -DMKL_ILP64 -m64 -I${MKL_LIB_PATH}/../../include" + "LDFLAGS=-L${MKL_LIB_PATH}" + ) + else () + message(STATUS "Build Faiss with OpenBlas/LAPACK") + set(FAISS_CONFIGURE_ARGS ${FAISS_CONFIGURE_ARGS} + "LDFLAGS=-L${OPENBLAS_PREFIX}/lib -L${LAPACK_PREFIX}/lib") + endif () + + if (KNOWHERE_GPU_VERSION) set(FAISS_CONFIGURE_ARGS ${FAISS_CONFIGURE_ARGS} "--with-cuda=${CUDA_TOOLKIT_ROOT_DIR}" "--with-cuda-arch=-gencode=arch=compute_60,code=sm_60 -gencode=arch=compute_61,code=sm_61 -gencode=arch=compute_70,code=sm_70 -gencode=arch=compute_75,code=sm_75" ) - else() + else () set(FAISS_CONFIGURE_ARGS ${FAISS_CONFIGURE_ARGS} --without-cuda) - endif() + endif () - if(USE_JFROG_CACHE STREQUAL "ON") + if (USE_JFROG_CACHE STREQUAL "ON") string(MD5 FAISS_COMBINE_MD5 "${FAISS_MD5}${LAPACK_MD5}${OPENBLAS_MD5}") - set(FAISS_CACHE_PACKAGE_NAME "faiss_${FAISS_COMBINE_MD5}.tar.gz") + if (KNOWHERE_GPU_VERSION) + set(FAISS_COMPUTE_TYPE "gpu") + else () + set(FAISS_COMPUTE_TYPE "cpu") + endif() + if (FAISS_WITH_MKL) + set(FAISS_CACHE_PACKAGE_NAME "faiss_${FAISS_COMPUTE_TYPE}_mkl_${FAISS_COMBINE_MD5}.tar.gz") + else () + set(FAISS_CACHE_PACKAGE_NAME "faiss_${FAISS_COMPUTE_TYPE}_openblas_${FAISS_COMBINE_MD5}.tar.gz") + endif() set(FAISS_CACHE_URL "${JFROG_ARTFACTORY_CACHE_URL}/${FAISS_CACHE_PACKAGE_NAME}") set(FAISS_CACHE_PACKAGE_PATH "${THIRDPARTY_PACKAGE_CACHE}/${FAISS_CACHE_PACKAGE_NAME}") @@ -736,18 +765,20 @@ macro(build_faiss) BUILD_BYPRODUCTS ${FAISS_STATIC_LIB}) - ExternalProject_Add_StepDependencies(faiss_ep build openblas_ep lapack_ep) + if (NOT FAISS_WITH_MKL) + ExternalProject_Add_StepDependencies(faiss_ep build openblas_ep lapack_ep) + endif () ExternalProject_Create_Cache(faiss_ep ${FAISS_CACHE_PACKAGE_PATH} "${INDEX_BINARY_DIR}/faiss_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${FAISS_CACHE_URL}) - else() + else () file(DOWNLOAD ${FAISS_CACHE_URL} ${FAISS_CACHE_PACKAGE_PATH} STATUS status) list(GET status 0 status_code) message(STATUS "DOWNLOADING FROM ${FAISS_CACHE_URL} TO ${FAISS_CACHE_PACKAGE_PATH}. STATUS = ${status_code}") if (status_code EQUAL 0) ExternalProject_Use_Cache(faiss_ep ${FAISS_CACHE_PACKAGE_PATH} ${INDEX_BINARY_DIR}) - endif() - endif() - else() + endif () + endif () + else () externalproject_add(faiss_ep URL ${FAISS_SOURCE_URL} @@ -764,35 +795,56 @@ macro(build_faiss) BUILD_BYPRODUCTS ${FAISS_STATIC_LIB}) - ExternalProject_Add_StepDependencies(faiss_ep build openblas_ep lapack_ep) - endif() + if (NOT FAISS_WITH_MKL) + ExternalProject_Add_StepDependencies(faiss_ep build openblas_ep lapack_ep) + endif () + + endif () file(MAKE_DIRECTORY "${FAISS_INCLUDE_DIR}") add_library(faiss STATIC IMPORTED) + set_target_properties( faiss - PROPERTIES IMPORTED_LOCATION "${FAISS_STATIC_LIB}" + PROPERTIES + IMPORTED_LOCATION "${FAISS_STATIC_LIB}" INTERFACE_INCLUDE_DIRECTORIES "${FAISS_INCLUDE_DIR}" - INTERFACE_LINK_LIBRARIES "openblas;lapack" ) + ) + if (FAISS_WITH_MKL) + set_target_properties( + faiss + PROPERTIES + INTERFACE_LINK_LIBRARIES "${MKL_LIBS}") + else () + set_target_properties( + faiss + PROPERTIES + INTERFACE_LINK_LIBRARIES "openblas;lapack") + endif () + add_dependencies(faiss faiss_ep) endmacro() -if(KNOWHERE_WITH_FAISS AND NOT TARGET faiss_ep) +if (KNOWHERE_WITH_FAISS AND NOT TARGET faiss_ep) - resolve_dependency(OpenBLAS) - get_target_property(OPENBLAS_INCLUDE_DIR openblas INTERFACE_INCLUDE_DIRECTORIES) - include_directories(SYSTEM "${OPENBLAS_INCLUDE_DIR}") - link_directories(SYSTEM ${OPENBLAS_PREFIX}/lib) + if (FAISS_WITH_MKL) + resolve_dependency(MKL) + else () + resolve_dependency(OpenBLAS) + get_target_property(OPENBLAS_INCLUDE_DIR openblas INTERFACE_INCLUDE_DIRECTORIES) + include_directories(SYSTEM "${OPENBLAS_INCLUDE_DIR}") + link_directories(SYSTEM ${OPENBLAS_PREFIX}/lib) - resolve_dependency(LAPACK) - get_target_property(LAPACK_INCLUDE_DIR lapack INTERFACE_INCLUDE_DIRECTORIES) - include_directories(SYSTEM "${LAPACK_INCLUDE_DIR}") - link_directories(SYSTEM "${LAPACK_PREFIX}/lib") + resolve_dependency(LAPACK) + get_target_property(LAPACK_INCLUDE_DIR lapack INTERFACE_INCLUDE_DIRECTORIES) + include_directories(SYSTEM "${LAPACK_INCLUDE_DIR}") + link_directories(SYSTEM "${LAPACK_PREFIX}/lib") + endif () resolve_dependency(FAISS) get_target_property(FAISS_INCLUDE_DIR faiss INTERFACE_INCLUDE_DIRECTORIES) include_directories(SYSTEM "${FAISS_INCLUDE_DIR}") link_directories(SYSTEM ${FAISS_PREFIX}/lib/) -endif() +endif () diff --git a/core/src/index/knowhere/CMakeLists.txt b/core/src/index/knowhere/CMakeLists.txt index bece9058a9..285461bdef 100644 --- a/core/src/index/knowhere/CMakeLists.txt +++ b/core/src/index/knowhere/CMakeLists.txt @@ -1,6 +1,3 @@ -include_directories(${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES}) -link_directories(${CUDA_TOOLKIT_ROOT_DIR}/lib64) - include_directories(${INDEX_SOURCE_DIR}/knowhere) include_directories(${INDEX_SOURCE_DIR}/thirdparty) include_directories(${INDEX_SOURCE_DIR}/thirdparty/SPTAG/AnnService) @@ -19,9 +16,9 @@ file(GLOB SRC_FILES ${SPTAG_SOURCE_DIR}/AnnService/src/Core/KDT/*.cpp ${SPTAG_SOURCE_DIR}/AnnService/src/Helper/*.cpp) -if(NOT TARGET SPTAGLibStatic) +if (NOT TARGET SPTAGLibStatic) add_library(SPTAGLibStatic STATIC ${SRC_FILES} ${HDR_FILES}) -endif() +endif () set(external_srcs knowhere/adapter/SptagAdapter.cpp @@ -33,22 +30,16 @@ set(external_srcs set(index_srcs knowhere/index/preprocessor/Normalize.cpp - knowhere/index/vector_index/IndexKDT.cpp + knowhere/index/vector_index/IndexSPTAG.cpp knowhere/index/vector_index/IndexIDMAP.cpp knowhere/index/vector_index/IndexIVF.cpp - knowhere/index/vector_index/IndexGPUIVF.cpp - knowhere/index/vector_index/helpers/KDTParameterMgr.cpp + knowhere/index/vector_index/helpers/SPTAGParameterMgr.cpp knowhere/index/vector_index/IndexNSG.cpp knowhere/index/vector_index/nsg/NSG.cpp knowhere/index/vector_index/nsg/NSGIO.cpp knowhere/index/vector_index/nsg/NSGHelper.cpp - knowhere/index/vector_index/helpers/Cloner.cpp - knowhere/index/vector_index/helpers/FaissGpuResourceMgr.cpp knowhere/index/vector_index/IndexIVFSQ.cpp - knowhere/index/vector_index/IndexGPUIVFSQ.cpp - knowhere/index/vector_index/IndexIVFSQHybrid.cpp knowhere/index/vector_index/IndexIVFPQ.cpp - knowhere/index/vector_index/IndexGPUIVFPQ.cpp knowhere/index/vector_index/FaissBaseIndex.cpp knowhere/index/vector_index/helpers/FaissIO.cpp knowhere/index/vector_index/helpers/IndexParameter.cpp @@ -57,24 +48,56 @@ set(index_srcs set(depend_libs SPTAGLibStatic faiss - openblas - lapack arrow - ${ARROW_PREFIX}/lib/libjemalloc_pic.a - cudart - cublas + ${ARROW_LIB_DIR}/libjemalloc_pic.a gomp gfortran pthread ) +if (FAISS_WITH_MKL) + set(depend_libs ${depend_libs} + "-Wl,--start-group \ + ${MKL_LIB_PATH}/libmkl_intel_ilp64.a \ + ${MKL_LIB_PATH}/libmkl_gnu_thread.a \ + ${MKL_LIB_PATH}/libmkl_core.a \ + -Wl,--end-group -lgomp -lpthread -lm -ldl" + ) +else () + set(depend_libs ${depend_libs} + lapack + openblas) +endif () -if(NOT TARGET knowhere) +if (KNOWHERE_GPU_VERSION) + include_directories(${CUDA_INCLUDE_DIRS}) + link_directories("${CUDA_TOOLKIT_ROOT_DIR}/lib64") + set(cuda_lib + cudart + cublas + ) + set(depend_libs ${depend_libs} + ${cuda_lib} + ) + + set(index_srcs ${index_srcs} + knowhere/index/vector_index/IndexGPUIVF.cpp + knowhere/index/vector_index/helpers/Cloner.cpp + knowhere/index/vector_index/helpers/FaissGpuResourceMgr.cpp + knowhere/index/vector_index/IndexGPUIVFSQ.cpp + knowhere/index/vector_index/IndexIVFSQHybrid.cpp + knowhere/index/vector_index/IndexGPUIVFPQ.cpp + knowhere/index/vector_index/IndexGPUIDMAP.cpp + ) + +endif () + +if (NOT TARGET knowhere) add_library( knowhere STATIC ${external_srcs} ${index_srcs} ) -endif() +endif () target_link_libraries( knowhere diff --git a/core/src/index/knowhere/knowhere/adapter/SptagAdapter.cpp b/core/src/index/knowhere/knowhere/adapter/SptagAdapter.cpp index b4c3910a01..db4a415261 100644 --- a/core/src/index/knowhere/knowhere/adapter/SptagAdapter.cpp +++ b/core/src/index/knowhere/knowhere/adapter/SptagAdapter.cpp @@ -89,34 +89,35 @@ ConvertToDataset(std::vector query_results) { } } - auto id_buf = MakeMutableBufferSmart((uint8_t*)p_id, sizeof(int64_t) * elems); - auto dist_buf = MakeMutableBufferSmart((uint8_t*)p_dist, sizeof(float) * elems); - - // TODO: magic - std::vector id_bufs{nullptr, id_buf}; - std::vector dist_bufs{nullptr, dist_buf}; - - auto int64_type = std::make_shared(); - auto float_type = std::make_shared(); - - auto id_array_data = arrow::ArrayData::Make(int64_type, elems, id_bufs); - auto dist_array_data = arrow::ArrayData::Make(float_type, elems, dist_bufs); - // auto id_array_data = std::make_shared(int64_type, sizeof(int64_t) * elems, id_bufs); - // auto dist_array_data = std::make_shared(float_type, sizeof(float) * elems, dist_bufs); - - // auto ids = ConstructInt64Array((uint8_t*)p_id, sizeof(int64_t) * elems); - // auto dists = ConstructFloatArray((uint8_t*)p_dist, sizeof(float) * elems); - - auto ids = std::make_shared>(id_array_data); - auto dists = std::make_shared>(dist_array_data); - std::vector array{ids, dists}; - - auto field_id = std::make_shared("id", std::make_shared()); - auto field_dist = std::make_shared("dist", std::make_shared()); - std::vector fields{field_id, field_dist}; - auto schema = std::make_shared(fields); - - return std::make_shared(array, schema); + // auto id_buf = MakeMutableBufferSmart((uint8_t*)p_id, sizeof(int64_t) * elems); + // auto dist_buf = MakeMutableBufferSmart((uint8_t*)p_dist, sizeof(float) * elems); + // + // // TODO: magic + // std::vector id_bufs{nullptr, id_buf}; + // std::vector dist_bufs{nullptr, dist_buf}; + // + // auto int64_type = std::make_shared(); + // auto float_type = std::make_shared(); + // + // auto id_array_data = arrow::ArrayData::Make(int64_type, elems, id_bufs); + // auto dist_array_data = arrow::ArrayData::Make(float_type, elems, dist_bufs); + // // auto id_array_data = std::make_shared(int64_type, sizeof(int64_t) * elems, id_bufs); + // // auto dist_array_data = std::make_shared(float_type, sizeof(float) * elems, dist_bufs); + // + // // auto ids = ConstructInt64Array((uint8_t*)p_id, sizeof(int64_t) * elems); + // // auto dists = ConstructFloatArray((uint8_t*)p_dist, sizeof(float) * elems); + // + // auto ids = std::make_shared>(id_array_data); + // auto dists = std::make_shared>(dist_array_data); + // std::vector array{ids, dists}; + // + // auto field_id = std::make_shared("id", std::make_shared()); + // auto field_dist = std::make_shared("dist", std::make_shared()); + // std::vector fields{field_id, field_dist}; + // auto schema = std::make_shared(fields); + // + // return std::make_shared(array, schema); + return std::make_shared((void*)p_id, (void*)p_dist); } } // namespace knowhere diff --git a/core/src/index/knowhere/knowhere/common/Config.h b/core/src/index/knowhere/knowhere/common/Config.h index 6191ecb771..48c2de8b1b 100644 --- a/core/src/index/knowhere/knowhere/common/Config.h +++ b/core/src/index/knowhere/knowhere/common/Config.h @@ -18,6 +18,8 @@ #pragma once #include +#include +#include "Log.h" namespace knowhere { @@ -50,6 +52,18 @@ struct Cfg { CheckValid() { return true; } + + void + Dump() { + KNOWHERE_LOG_DEBUG << DumpImpl().str(); + } + + virtual std::stringstream + DumpImpl() { + std::stringstream ss; + ss << "dim: " << d << ", metric: " << int(metric_type) << ", gpuid: " << gpu_id << ", k: " << k; + return ss; + } }; using Config = std::shared_ptr; diff --git a/core/src/index/knowhere/knowhere/common/Dataset.h b/core/src/index/knowhere/knowhere/common/Dataset.h index 1331239dd6..b101aba6a7 100644 --- a/core/src/index/knowhere/knowhere/common/Dataset.h +++ b/core/src/index/knowhere/knowhere/common/Dataset.h @@ -54,6 +54,9 @@ class Dataset { : tensor_(std::move(tensor)), tensor_schema_(std::move(tensor_schema)) { } + Dataset(void* ids, void* dists) : ids_(ids), dists_(dists) { + } + Dataset(const Dataset&) = delete; Dataset& operator=(const Dataset&) = delete; @@ -128,6 +131,16 @@ class Dataset { tensor_schema_ = std::move(tensor_schema); } + void* + ids() { + return ids_; + } + + void* + dist() { + return dists_; + } + // const Config & // meta() const { return meta_; } @@ -141,6 +154,9 @@ class Dataset { SchemaPtr array_schema_; std::vector tensor_; SchemaPtr tensor_schema_; + // TODO(yukun): using smart pointer + void* ids_; + void* dists_; // Config meta_; }; diff --git a/core/src/index/knowhere/knowhere/common/Log.h b/core/src/index/knowhere/knowhere/common/Log.h index 369e7143af..e50a28a15e 100644 --- a/core/src/index/knowhere/knowhere/common/Log.h +++ b/core/src/index/knowhere/knowhere/common/Log.h @@ -17,7 +17,7 @@ #pragma once -#include "external/easyloggingpp/easylogging++.h" +#include "easyloggingpp/easylogging++.h" namespace knowhere { diff --git a/core/src/index/knowhere/knowhere/index/vector_index/IndexGPUIDMAP.cpp b/core/src/index/knowhere/knowhere/index/vector_index/IndexGPUIDMAP.cpp new file mode 100644 index 0000000000..1aded3ddaa --- /dev/null +++ b/core/src/index/knowhere/knowhere/index/vector_index/IndexGPUIDMAP.cpp @@ -0,0 +1,129 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "knowhere/index/vector_index/IndexGPUIDMAP.h" + +#include +#include +#include +#include + +#ifdef MILVUS_GPU_VERSION + +#include + +#endif + +#include "knowhere/adapter/VectorAdapter.h" +#include "knowhere/common/Exception.h" +#include "knowhere/index/vector_index/IndexIDMAP.h" +#include "knowhere/index/vector_index/helpers/FaissIO.h" + +namespace knowhere { + +VectorIndexPtr +GPUIDMAP::CopyGpuToCpu(const Config& config) { + std::lock_guard lk(mutex_); + + faiss::Index* device_index = index_.get(); + faiss::Index* host_index = faiss::gpu::index_gpu_to_cpu(device_index); + + std::shared_ptr new_index; + new_index.reset(host_index); + return std::make_shared(new_index); +} + +VectorIndexPtr +GPUIDMAP::Clone() { + auto cpu_idx = CopyGpuToCpu(Config()); + + if (auto idmap = std::dynamic_pointer_cast(cpu_idx)) { + return idmap->CopyCpuToGpu(gpu_id_, Config()); + } else { + KNOWHERE_THROW_MSG("IndexType not Support GpuClone"); + } +} + +BinarySet +GPUIDMAP::SerializeImpl() { + try { + MemoryIOWriter writer; + { + faiss::Index* index = index_.get(); + faiss::Index* host_index = faiss::gpu::index_gpu_to_cpu(index); + + faiss::write_index(host_index, &writer); + delete host_index; + } + auto data = std::make_shared(); + data.reset(writer.data_); + + BinarySet res_set; + res_set.Append("IVF", data, writer.rp); + + return res_set; + } catch (std::exception& e) { + KNOWHERE_THROW_MSG(e.what()); + } +} + +void +GPUIDMAP::LoadImpl(const BinarySet& index_binary) { + auto binary = index_binary.GetByName("IVF"); + MemoryIOReader reader; + { + reader.total = binary->size; + reader.data_ = binary->data.get(); + + faiss::Index* index = faiss::read_index(&reader); + + if (auto res = FaissGpuResourceMgr::GetInstance().GetRes(gpu_id_)) { + ResScope rs(res, gpu_id_, false); + auto device_index = faiss::gpu::index_cpu_to_gpu(res->faiss_res.get(), gpu_id_, index); + index_.reset(device_index); + res_ = res; + } else { + KNOWHERE_THROW_MSG("Load error, can't get gpu resource"); + } + + delete index; + } +} + +VectorIndexPtr +GPUIDMAP::CopyGpuToGpu(const int64_t& device_id, const Config& config) { + auto cpu_index = CopyGpuToCpu(config); + return std::static_pointer_cast(cpu_index)->CopyCpuToGpu(device_id, config); +} + +float* +GPUIDMAP::GetRawVectors() { + KNOWHERE_THROW_MSG("Not support"); +} + +int64_t* +GPUIDMAP::GetRawIds() { + KNOWHERE_THROW_MSG("Not support"); +} + +void +GPUIDMAP::search_impl(int64_t n, const float* data, int64_t k, float* distances, int64_t* labels, const Config& cfg) { + ResScope rs(res_, gpu_id_); + index_->search(n, (float*)data, k, distances, labels); +} + +} // namespace knowhere diff --git a/core/src/index/knowhere/knowhere/index/vector_index/IndexGPUIDMAP.h b/core/src/index/knowhere/knowhere/index/vector_index/IndexGPUIDMAP.h new file mode 100644 index 0000000000..f1dfe2f21a --- /dev/null +++ b/core/src/index/knowhere/knowhere/index/vector_index/IndexGPUIDMAP.h @@ -0,0 +1,63 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include "IndexGPUIVF.h" +#include "IndexIDMAP.h" +#include "IndexIVF.h" + +#include +#include + +namespace knowhere { + +class GPUIDMAP : public IDMAP, public GPUIndex { + public: + explicit GPUIDMAP(std::shared_ptr index, const int64_t& device_id, ResPtr& res) + : IDMAP(std::move(index)), GPUIndex(device_id, res) { + } + + VectorIndexPtr + CopyGpuToCpu(const Config& config) override; + + float* + GetRawVectors() override; + + int64_t* + GetRawIds() override; + + VectorIndexPtr + Clone() override; + + VectorIndexPtr + CopyGpuToGpu(const int64_t& device_id, const Config& config) override; + + protected: + void + search_impl(int64_t n, const float* data, int64_t k, float* distances, int64_t* labels, const Config& cfg) override; + + BinarySet + SerializeImpl() override; + + void + LoadImpl(const BinarySet& index_binary) override; +}; + +using GPUIDMAPPtr = std::shared_ptr; + +} // namespace knowhere diff --git a/core/src/index/knowhere/knowhere/index/vector_index/IndexGPUIVFPQ.cpp b/core/src/index/knowhere/knowhere/index/vector_index/IndexGPUIVFPQ.cpp index b027539c37..cbd4f4f09c 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/IndexGPUIVFPQ.cpp +++ b/core/src/index/knowhere/knowhere/index/vector_index/IndexGPUIVFPQ.cpp @@ -39,17 +39,19 @@ GPUIVFPQ::Train(const DatasetPtr& dataset, const Config& config) { GETTENSOR(dataset) - // TODO(linxj): set device here. - // TODO(linxj): set gpu resource here. - faiss::gpu::StandardGpuResources res; - faiss::gpu::GpuIndexIVFPQ device_index(&res, dim, build_cfg->nlist, build_cfg->m, build_cfg->nbits, - GetMetricType(build_cfg->metric_type)); // IP not support - device_index.train(rows, (float*)p_data); - - std::shared_ptr host_index = nullptr; - host_index.reset(faiss::gpu::index_gpu_to_cpu(&device_index)); - - return std::make_shared(host_index); + auto temp_resource = FaissGpuResourceMgr::GetInstance().GetRes(gpu_id_); + if (temp_resource != nullptr) { + ResScope rs(temp_resource, gpu_id_, true); + auto device_index = new faiss::gpu::GpuIndexIVFPQ(temp_resource->faiss_res.get(), dim, build_cfg->nlist, + build_cfg->m, build_cfg->nbits, + GetMetricType(build_cfg->metric_type)); // IP not support + device_index->train(rows, (float*)p_data); + std::shared_ptr host_index = nullptr; + host_index.reset(faiss::gpu::index_gpu_to_cpu(device_index)); + return std::make_shared(host_index); + } else { + KNOWHERE_THROW_MSG("Build IVFSQ can't get gpu resource"); + } } std::shared_ptr @@ -66,7 +68,14 @@ GPUIVFPQ::GenParams(const Config& config) { VectorIndexPtr GPUIVFPQ::CopyGpuToCpu(const Config& config) { - KNOWHERE_THROW_MSG("not support yet"); + std::lock_guard lk(mutex_); + + faiss::Index* device_index = index_.get(); + faiss::Index* host_index = faiss::gpu::index_gpu_to_cpu(device_index); + + std::shared_ptr new_index; + new_index.reset(host_index); + return std::make_shared(new_index); } } // namespace knowhere diff --git a/core/src/index/knowhere/knowhere/index/vector_index/IndexGPUIVFPQ.h b/core/src/index/knowhere/knowhere/index/vector_index/IndexGPUIVFPQ.h index 13ea1075ca..f699445d65 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/IndexGPUIVFPQ.h +++ b/core/src/index/knowhere/knowhere/index/vector_index/IndexGPUIVFPQ.h @@ -18,6 +18,7 @@ #pragma once #include +#include #include "IndexGPUIVF.h" @@ -28,6 +29,10 @@ class GPUIVFPQ : public GPUIVF { explicit GPUIVFPQ(const int& device_id) : GPUIVF(device_id) { } + GPUIVFPQ(std::shared_ptr index, const int64_t& device_id, ResPtr& resource) + : GPUIVF(std::move(index), device_id, resource) { + } + IndexModelPtr Train(const DatasetPtr& dataset, const Config& config) override; diff --git a/core/src/index/knowhere/knowhere/index/vector_index/IndexIDMAP.cpp b/core/src/index/knowhere/knowhere/index/vector_index/IndexIDMAP.cpp index f926951736..7aedf98613 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/IndexIDMAP.cpp +++ b/core/src/index/knowhere/knowhere/index/vector_index/IndexIDMAP.cpp @@ -17,10 +17,18 @@ #include #include -#include + +#include +#include #include #include +#ifdef MILVUS_GPU_VERSION + +#include + +#endif + #include #include "knowhere/adapter/VectorAdapter.h" @@ -28,6 +36,13 @@ #include "knowhere/index/vector_index/IndexIDMAP.h" #include "knowhere/index/vector_index/helpers/FaissIO.h" +#ifdef MILVUS_GPU_VERSION + +#include "knowhere/index/vector_index/IndexGPUIDMAP.h" +#include "knowhere/index/vector_index/helpers/FaissGpuResourceMgr.h" + +#endif + namespace knowhere { BinarySet @@ -65,23 +80,24 @@ IDMAP::Search(const DatasetPtr& dataset, const Config& config) { search_impl(rows, (float*)p_data, config->k, res_dis, res_ids, Config()); - auto id_buf = MakeMutableBufferSmart((uint8_t*)res_ids, sizeof(int64_t) * elems); - auto dist_buf = MakeMutableBufferSmart((uint8_t*)res_dis, sizeof(float) * elems); - - std::vector id_bufs{nullptr, id_buf}; - std::vector dist_bufs{nullptr, dist_buf}; - - auto int64_type = std::make_shared(); - auto float_type = std::make_shared(); - - auto id_array_data = arrow::ArrayData::Make(int64_type, elems, id_bufs); - auto dist_array_data = arrow::ArrayData::Make(float_type, elems, dist_bufs); - - auto ids = std::make_shared>(id_array_data); - auto dists = std::make_shared>(dist_array_data); - std::vector array{ids, dists}; - - return std::make_shared(array, nullptr); + // auto id_buf = MakeMutableBufferSmart((uint8_t*)res_ids, sizeof(int64_t) * elems); + // auto dist_buf = MakeMutableBufferSmart((uint8_t*)res_dis, sizeof(float) * elems); + // + // std::vector id_bufs{nullptr, id_buf}; + // std::vector dist_bufs{nullptr, dist_buf}; + // + // auto int64_type = std::make_shared(); + // auto float_type = std::make_shared(); + // + // auto id_array_data = arrow::ArrayData::Make(int64_type, elems, id_bufs); + // auto dist_array_data = arrow::ArrayData::Make(float_type, elems, dist_bufs); + // + // auto ids = std::make_shared>(id_array_data); + // auto dists = std::make_shared>(dist_array_data); + // std::vector array{ids, dists}; + // + // return std::make_shared(array, nullptr); + return std::make_shared((void*)res_ids, (void*)res_dis); } void @@ -160,6 +176,8 @@ IDMAP::Clone() { VectorIndexPtr IDMAP::CopyCpuToGpu(const int64_t& device_id, const Config& config) { +#ifdef MILVUS_GPU_VERSION + if (auto res = FaissGpuResourceMgr::GetInstance().GetRes(device_id)) { ResScope rs(res, device_id, false); auto gpu_index = faiss::gpu::index_cpu_to_gpu(res->faiss_res.get(), device_id, index_.get()); @@ -170,6 +188,9 @@ IDMAP::CopyCpuToGpu(const int64_t& device_id, const Config& config) { } else { KNOWHERE_THROW_MSG("CopyCpuToGpu Error, can't get gpu_resource"); } +#else + KNOWHERE_THROW_MSG("Calling IDMAP::CopyCpuToGpu when we are using CPU version"); +#endif } void @@ -177,95 +198,4 @@ IDMAP::Seal() { // do nothing } -VectorIndexPtr -GPUIDMAP::CopyGpuToCpu(const Config& config) { - std::lock_guard lk(mutex_); - - faiss::Index* device_index = index_.get(); - faiss::Index* host_index = faiss::gpu::index_gpu_to_cpu(device_index); - - std::shared_ptr new_index; - new_index.reset(host_index); - return std::make_shared(new_index); -} - -VectorIndexPtr -GPUIDMAP::Clone() { - auto cpu_idx = CopyGpuToCpu(Config()); - - if (auto idmap = std::dynamic_pointer_cast(cpu_idx)) { - return idmap->CopyCpuToGpu(gpu_id_, Config()); - } else { - KNOWHERE_THROW_MSG("IndexType not Support GpuClone"); - } -} - -BinarySet -GPUIDMAP::SerializeImpl() { - try { - MemoryIOWriter writer; - { - faiss::Index* index = index_.get(); - faiss::Index* host_index = faiss::gpu::index_gpu_to_cpu(index); - - faiss::write_index(host_index, &writer); - delete host_index; - } - auto data = std::make_shared(); - data.reset(writer.data_); - - BinarySet res_set; - res_set.Append("IVF", data, writer.rp); - - return res_set; - } catch (std::exception& e) { - KNOWHERE_THROW_MSG(e.what()); - } -} - -void -GPUIDMAP::LoadImpl(const BinarySet& index_binary) { - auto binary = index_binary.GetByName("IVF"); - MemoryIOReader reader; - { - reader.total = binary->size; - reader.data_ = binary->data.get(); - - faiss::Index* index = faiss::read_index(&reader); - - if (auto res = FaissGpuResourceMgr::GetInstance().GetRes(gpu_id_)) { - ResScope rs(res, gpu_id_, false); - auto device_index = faiss::gpu::index_cpu_to_gpu(res->faiss_res.get(), gpu_id_, index); - index_.reset(device_index); - res_ = res; - } else { - KNOWHERE_THROW_MSG("Load error, can't get gpu resource"); - } - - delete index; - } -} - -VectorIndexPtr -GPUIDMAP::CopyGpuToGpu(const int64_t& device_id, const Config& config) { - auto cpu_index = CopyGpuToCpu(config); - return std::static_pointer_cast(cpu_index)->CopyCpuToGpu(device_id, config); -} - -float* -GPUIDMAP::GetRawVectors() { - KNOWHERE_THROW_MSG("Not support"); -} - -int64_t* -GPUIDMAP::GetRawIds() { - KNOWHERE_THROW_MSG("Not support"); -} - -void -GPUIDMAP::search_impl(int64_t n, const float* data, int64_t k, float* distances, int64_t* labels, const Config& cfg) { - ResScope rs(res_, gpu_id_); - index_->search(n, (float*)data, k, distances, labels); -} - } // namespace knowhere diff --git a/core/src/index/knowhere/knowhere/index/vector_index/IndexIDMAP.h b/core/src/index/knowhere/knowhere/index/vector_index/IndexIDMAP.h index ec1cbb9e77..0f66e8fac0 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/IndexIDMAP.h +++ b/core/src/index/knowhere/knowhere/index/vector_index/IndexIDMAP.h @@ -17,7 +17,6 @@ #pragma once -#include "IndexGPUIVF.h" #include "IndexIVF.h" #include @@ -67,32 +66,4 @@ class IDMAP : public VectorIndex, public FaissBaseIndex { using IDMAPPtr = std::shared_ptr; -class GPUIDMAP : public IDMAP, public GPUIndex { - public: - explicit GPUIDMAP(std::shared_ptr index, const int64_t& device_id, ResPtr& res) - : IDMAP(std::move(index)), GPUIndex(device_id, res) { - } - - VectorIndexPtr - CopyGpuToCpu(const Config& config) override; - float* - GetRawVectors() override; - int64_t* - GetRawIds() override; - VectorIndexPtr - Clone() override; - VectorIndexPtr - CopyGpuToGpu(const int64_t& device_id, const Config& config) override; - - protected: - void - search_impl(int64_t n, const float* data, int64_t k, float* distances, int64_t* labels, const Config& cfg) override; - BinarySet - SerializeImpl() override; - void - LoadImpl(const BinarySet& index_binary) override; -}; - -using GPUIDMAPPtr = std::shared_ptr; - } // namespace knowhere diff --git a/core/src/index/knowhere/knowhere/index/vector_index/IndexIVF.cpp b/core/src/index/knowhere/knowhere/index/vector_index/IndexIVF.cpp index 6da5db38ec..7f30a97ea0 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/IndexIVF.cpp +++ b/core/src/index/knowhere/knowhere/index/vector_index/IndexIVF.cpp @@ -15,11 +15,19 @@ // specific language governing permissions and limitations // under the License. +#include #include #include #include #include +#include +#include +#include +#include +#ifdef MILVUS_GPU_VERSION +#include #include +#endif #include #include @@ -29,7 +37,9 @@ #include "knowhere/adapter/VectorAdapter.h" #include "knowhere/common/Exception.h" #include "knowhere/common/Log.h" +#ifdef MILVUS_GPU_VERSION #include "knowhere/index/vector_index/IndexGPUIVF.h" +#endif #include "knowhere/index/vector_index/IndexIVF.h" namespace knowhere { @@ -129,23 +139,23 @@ IVF::Search(const DatasetPtr& dataset, const Config& config) { // std::cout << ss_res_id.str() << std::endl; // std::cout << ss_res_dist.str() << std::endl << std::endl; - auto id_buf = MakeMutableBufferSmart((uint8_t*)res_ids, sizeof(int64_t) * elems); - auto dist_buf = MakeMutableBufferSmart((uint8_t*)res_dis, sizeof(float) * elems); + // auto id_buf = MakeMutableBufferSmart((uint8_t*)res_ids, sizeof(int64_t) * elems); + // auto dist_buf = MakeMutableBufferSmart((uint8_t*)res_dis, sizeof(float) * elems); + // + // std::vector id_bufs{nullptr, id_buf}; + // std::vector dist_bufs{nullptr, dist_buf}; + // + // auto int64_type = std::make_shared(); + // auto float_type = std::make_shared(); + // + // auto id_array_data = arrow::ArrayData::Make(int64_type, elems, id_bufs); + // auto dist_array_data = arrow::ArrayData::Make(float_type, elems, dist_bufs); + // + // auto ids = std::make_shared>(id_array_data); + // auto dists = std::make_shared>(dist_array_data); + // std::vector array{ids, dists}; - std::vector id_bufs{nullptr, id_buf}; - std::vector dist_bufs{nullptr, dist_buf}; - - auto int64_type = std::make_shared(); - auto float_type = std::make_shared(); - - auto id_array_data = arrow::ArrayData::Make(int64_type, elems, id_bufs); - auto dist_array_data = arrow::ArrayData::Make(float_type, elems, dist_bufs); - - auto ids = std::make_shared>(id_array_data); - auto dists = std::make_shared>(dist_array_data); - std::vector array{ids, dists}; - - return std::make_shared(array, nullptr); + return std::make_shared((void*)res_ids, (void*)res_dis); } void @@ -221,16 +231,17 @@ IVF::search_impl(int64_t n, const float* data, int64_t k, float* distances, int6 faiss::ivflib::search_with_parameters(index_.get(), n, (float*)data, k, distances, labels, params.get()); stdclock::time_point after = stdclock::now(); double search_cost = (std::chrono::duration(after - before)).count(); - KNOWHERE_LOG_DEBUG << "K=" << k << " NQ=" << n << " NL=" << faiss::indexIVF_stats.nlist - << " ND=" << faiss::indexIVF_stats.ndis << " NH=" << faiss::indexIVF_stats.nheap_updates - << " Q=" << faiss::indexIVF_stats.quantization_time - << " S=" << faiss::indexIVF_stats.search_time; + KNOWHERE_LOG_DEBUG << "IVF search cost: " << search_cost + << ", quantization cost: " << faiss::indexIVF_stats.quantization_time + << ", data search cost: " << faiss::indexIVF_stats.search_time; faiss::indexIVF_stats.quantization_time = 0; faiss::indexIVF_stats.search_time = 0; } VectorIndexPtr IVF::CopyCpuToGpu(const int64_t& device_id, const Config& config) { +#ifdef MILVUS_GPU_VERSION + if (auto res = FaissGpuResourceMgr::GetInstance().GetRes(device_id)) { ResScope rs(res, device_id, false); auto gpu_index = faiss::gpu::index_cpu_to_gpu(res->faiss_res.get(), device_id, index_.get()); @@ -241,6 +252,10 @@ IVF::CopyCpuToGpu(const int64_t& device_id, const Config& config) { } else { KNOWHERE_THROW_MSG("CopyCpuToGpu Error, can't get gpu_resource"); } + +#else + KNOWHERE_THROW_MSG("Calling IVF::CopyCpuToGpu when we are using CPU version"); +#endif } VectorIndexPtr diff --git a/core/src/index/knowhere/knowhere/index/vector_index/IndexIVFPQ.cpp b/core/src/index/knowhere/knowhere/index/vector_index/IndexIVFPQ.cpp index 03acbf31d7..841ef63245 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/IndexIVFPQ.cpp +++ b/core/src/index/knowhere/knowhere/index/vector_index/IndexIVFPQ.cpp @@ -17,11 +17,19 @@ #include #include +#ifdef MILVUS_GPU_VERSION +#include +#endif + #include #include #include "knowhere/adapter/VectorAdapter.h" #include "knowhere/common/Exception.h" +#ifdef MILVUS_GPU_VERSION +#include "knowhere/index/vector_index/IndexGPUIVF.h" +#include "knowhere/index/vector_index/IndexGPUIVFPQ.h" +#endif #include "knowhere/index/vector_index/IndexIVFPQ.h" namespace knowhere { @@ -60,4 +68,22 @@ IVFPQ::Clone_impl(const std::shared_ptr& index) { return std::make_shared(index); } +VectorIndexPtr +IVFPQ::CopyCpuToGpu(const int64_t& device_id, const Config& config) { +#ifdef MILVUS_GPU_VERSION + if (auto res = FaissGpuResourceMgr::GetInstance().GetRes(device_id)) { + ResScope rs(res, device_id, false); + auto gpu_index = faiss::gpu::index_cpu_to_gpu(res->faiss_res.get(), device_id, index_.get()); + + std::shared_ptr device_index; + device_index.reset(gpu_index); + return std::make_shared(device_index, device_id, res); + } else { + KNOWHERE_THROW_MSG("CopyCpuToGpu Error, can't get gpu_resource"); + } +#else + KNOWHERE_THROW_MSG("Calling IVFPQ::CopyCpuToGpu when we are using CPU version"); +#endif +} + } // namespace knowhere diff --git a/core/src/index/knowhere/knowhere/index/vector_index/IndexIVFPQ.h b/core/src/index/knowhere/knowhere/index/vector_index/IndexIVFPQ.h index 69aaa5090b..fc50c68389 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/IndexIVFPQ.h +++ b/core/src/index/knowhere/knowhere/index/vector_index/IndexIVFPQ.h @@ -34,6 +34,9 @@ class IVFPQ : public IVF { IndexModelPtr Train(const DatasetPtr& dataset, const Config& config) override; + VectorIndexPtr + CopyCpuToGpu(const int64_t& device_id, const Config& config) override; + protected: std::shared_ptr GenParams(const Config& config) override; diff --git a/core/src/index/knowhere/knowhere/index/vector_index/IndexIVFSQ.cpp b/core/src/index/knowhere/knowhere/index/vector_index/IndexIVFSQ.cpp index 6e9a1d94da..7c86cd4dbd 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/IndexIVFSQ.cpp +++ b/core/src/index/knowhere/knowhere/index/vector_index/IndexIVFSQ.cpp @@ -15,15 +15,22 @@ // specific language governing permissions and limitations // under the License. +#ifdef MILVUS_GPU_VERSION +#include #include +#endif #include + #include #include "knowhere/adapter/VectorAdapter.h" #include "knowhere/common/Exception.h" -#include "knowhere/index/vector_index/IndexGPUIVFSQ.h" #include "knowhere/index/vector_index/IndexIVFSQ.h" + +#ifdef MILVUS_GPU_VERSION +#include "knowhere/index/vector_index/IndexGPUIVFSQ.h" #include "knowhere/index/vector_index/helpers/FaissGpuResourceMgr.h" +#endif namespace knowhere { @@ -54,6 +61,8 @@ IVFSQ::Clone_impl(const std::shared_ptr& index) { VectorIndexPtr IVFSQ::CopyCpuToGpu(const int64_t& device_id, const Config& config) { +#ifdef MILVUS_GPU_VERSION + if (auto res = FaissGpuResourceMgr::GetInstance().GetRes(device_id)) { ResScope rs(res, device_id, false); @@ -65,6 +74,10 @@ IVFSQ::CopyCpuToGpu(const int64_t& device_id, const Config& config) { } else { KNOWHERE_THROW_MSG("CopyCpuToGpu Error, can't get gpu_resource"); } + +#else + KNOWHERE_THROW_MSG("Calling IVFSQ::CopyCpuToGpu when we are using CPU version"); +#endif } } // namespace knowhere diff --git a/core/src/index/knowhere/knowhere/index/vector_index/IndexKDT.cpp b/core/src/index/knowhere/knowhere/index/vector_index/IndexKDT.cpp deleted file mode 100644 index 1bd45075e3..0000000000 --- a/core/src/index/knowhere/knowhere/index/vector_index/IndexKDT.cpp +++ /dev/null @@ -1,180 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include -#include -#include -#include -#include - -#undef mkdir - -#include "knowhere/index/vector_index/IndexKDT.h" -#include "knowhere/index/vector_index/helpers/Definitions.h" -//#include "knowhere/index/preprocessor/normalize.h" -#include "knowhere/adapter/SptagAdapter.h" -#include "knowhere/common/Exception.h" -#include "knowhere/index/vector_index/helpers/KDTParameterMgr.h" - -namespace knowhere { - -BinarySet -CPUKDTRNG::Serialize() { - std::vector index_blobs; - std::vector index_len; - - // TODO(zirui): dev - // index_ptr_->SaveIndexToMemory(index_blobs, index_len); - BinarySet binary_set; - - // - // auto sample = std::make_shared(); - // sample.reset(static_cast(index_blobs[0])); - // auto tree = std::make_shared(); - // tree.reset(static_cast(index_blobs[1])); - // auto graph = std::make_shared(); - // graph.reset(static_cast(index_blobs[2])); - // auto metadata = std::make_shared(); - // metadata.reset(static_cast(index_blobs[3])); - // - // binary_set.Append("samples", sample, index_len[0]); - // binary_set.Append("tree", tree, index_len[1]); - // binary_set.Append("graph", graph, index_len[2]); - // binary_set.Append("metadata", metadata, index_len[3]); - return binary_set; -} - -void -CPUKDTRNG::Load(const BinarySet& binary_set) { - // TODO(zirui): dev - - // std::vector index_blobs; - // - // auto samples = binary_set.GetByName("samples"); - // index_blobs.push_back(samples->data.get()); - // - // auto tree = binary_set.GetByName("tree"); - // index_blobs.push_back(tree->data.get()); - // - // auto graph = binary_set.GetByName("graph"); - // index_blobs.push_back(graph->data.get()); - // - // auto metadata = binary_set.GetByName("metadata"); - // index_blobs.push_back(metadata->data.get()); - // - // index_ptr_->LoadIndexFromMemory(index_blobs); -} - -// PreprocessorPtr -// CPUKDTRNG::BuildPreprocessor(const DatasetPtr &dataset, const Config &config) { -// return std::make_shared(); -//} - -IndexModelPtr -CPUKDTRNG::Train(const DatasetPtr& origin, const Config& train_config) { - SetParameters(train_config); - DatasetPtr dataset = origin->Clone(); - - // if (index_ptr_->GetDistCalcMethod() == SPTAG::DistCalcMethod::Cosine - // && preprocessor_) { - // preprocessor_->Preprocess(dataset); - //} - - auto vectorset = ConvertToVectorSet(dataset); - auto metaset = ConvertToMetadataSet(dataset); - index_ptr_->BuildIndex(vectorset, metaset); - - // TODO: return IndexModelPtr - return nullptr; -} - -void -CPUKDTRNG::Add(const DatasetPtr& origin, const Config& add_config) { - SetParameters(add_config); - DatasetPtr dataset = origin->Clone(); - - // if (index_ptr_->GetDistCalcMethod() == SPTAG::DistCalcMethod::Cosine - // && preprocessor_) { - // preprocessor_->Preprocess(dataset); - //} - - auto vectorset = ConvertToVectorSet(dataset); - auto metaset = ConvertToMetadataSet(dataset); - index_ptr_->AddIndex(vectorset, metaset); -} - -void -CPUKDTRNG::SetParameters(const Config& config) { - for (auto& para : KDTParameterMgr::GetInstance().GetKDTParameters()) { - // auto value = config.get_with_default(para.first, para.second); - index_ptr_->SetParameter(para.first, para.second); - } -} - -DatasetPtr -CPUKDTRNG::Search(const DatasetPtr& dataset, const Config& config) { - SetParameters(config); - auto tensor = dataset->tensor()[0]; - auto p = (float*)tensor->raw_mutable_data(); - for (auto i = 0; i < 10; ++i) { - for (auto j = 0; j < 10; ++j) { - std::cout << p[i * 10 + j] << " "; - } - std::cout << std::endl; - } - std::vector query_results = ConvertToQueryResult(dataset, config); - -#pragma omp parallel for - for (auto i = 0; i < query_results.size(); ++i) { - auto target = (float*)query_results[i].GetTarget(); - std::cout << target[0] << ", " << target[1] << ", " << target[2] << std::endl; - index_ptr_->SearchIndex(query_results[i]); - } - - return ConvertToDataset(query_results); -} - -int64_t -CPUKDTRNG::Count() { - index_ptr_->GetNumSamples(); -} - -int64_t -CPUKDTRNG::Dimension() { - index_ptr_->GetFeatureDim(); -} - -VectorIndexPtr -CPUKDTRNG::Clone() { - KNOWHERE_THROW_MSG("not support"); -} - -void -CPUKDTRNG::Seal() { - // do nothing -} - -// TODO(linxj): -BinarySet -CPUKDTRNGIndexModel::Serialize() { -} - -void -CPUKDTRNGIndexModel::Load(const BinarySet& binary) { -} - -} // namespace knowhere diff --git a/core/src/index/knowhere/knowhere/index/vector_index/IndexNSG.cpp b/core/src/index/knowhere/knowhere/index/vector_index/IndexNSG.cpp index f5519b8240..204819517a 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/IndexNSG.cpp +++ b/core/src/index/knowhere/knowhere/index/vector_index/IndexNSG.cpp @@ -19,8 +19,10 @@ #include "knowhere/adapter/VectorAdapter.h" #include "knowhere/common/Exception.h" #include "knowhere/common/Timer.h" +#ifdef MILVUS_GPU_VERSION #include "knowhere/index/vector_index/IndexGPUIVF.h" -#include "knowhere/index/vector_index/IndexIDMAP.h" +#endif + #include "knowhere/index/vector_index/IndexIVF.h" #include "knowhere/index/vector_index/nsg/NSG.h" #include "knowhere/index/vector_index/nsg/NSGIO.h" @@ -86,23 +88,24 @@ NSG::Search(const DatasetPtr& dataset, const Config& config) { s_params.search_length = build_cfg->search_length; index_->Search((float*)p_data, rows, dim, build_cfg->k, res_dis, res_ids, s_params); - auto id_buf = MakeMutableBufferSmart((uint8_t*)res_ids, sizeof(int64_t) * elems); - auto dist_buf = MakeMutableBufferSmart((uint8_t*)res_dis, sizeof(float) * elems); + // auto id_buf = MakeMutableBufferSmart((uint8_t*)res_ids, sizeof(int64_t) * elems); + // auto dist_buf = MakeMutableBufferSmart((uint8_t*)res_dis, sizeof(float) * elems); - std::vector id_bufs{nullptr, id_buf}; - std::vector dist_bufs{nullptr, dist_buf}; - - auto int64_type = std::make_shared(); - auto float_type = std::make_shared(); - - auto id_array_data = arrow::ArrayData::Make(int64_type, elems, id_bufs); - auto dist_array_data = arrow::ArrayData::Make(float_type, elems, dist_bufs); - - auto ids = std::make_shared>(id_array_data); - auto dists = std::make_shared>(dist_array_data); - std::vector array{ids, dists}; - - return std::make_shared(array, nullptr); + // std::vector id_bufs{nullptr, id_buf}; + // std::vector dist_bufs{nullptr, dist_buf}; + // + // auto int64_type = std::make_shared(); + // auto float_type = std::make_shared(); + // + // auto id_array_data = arrow::ArrayData::Make(int64_type, elems, id_bufs); + // auto dist_array_data = arrow::ArrayData::Make(float_type, elems, dist_bufs); + // + // auto ids = std::make_shared>(id_array_data); + // auto dists = std::make_shared>(dist_array_data); + // std::vector array{ids, dists}; + // + // return std::make_shared(array, nullptr); + return std::make_shared((void*)res_ids, (void*)res_dis); } IndexModelPtr @@ -117,7 +120,11 @@ NSG::Train(const DatasetPtr& dataset, const Config& config) { } // TODO(linxj): dev IndexFactory, support more IndexType +#ifdef MILVUS_GPU_VERSION auto preprocess_index = std::make_shared(build_cfg->gpu_id); +#else + auto preprocess_index = std::make_shared(); +#endif auto model = preprocess_index->Train(dataset, config); preprocess_index->set_index_model(model); preprocess_index->AddWithoutIds(dataset, config); diff --git a/core/src/index/knowhere/knowhere/index/vector_index/IndexSPTAG.cpp b/core/src/index/knowhere/knowhere/index/vector_index/IndexSPTAG.cpp new file mode 100644 index 0000000000..17a93fdcc7 --- /dev/null +++ b/core/src/index/knowhere/knowhere/index/vector_index/IndexSPTAG.cpp @@ -0,0 +1,348 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include +#include +#include + +#include +#include +#include + +#undef mkdir + +#include "knowhere/adapter/SptagAdapter.h" +#include "knowhere/common/Exception.h" +#include "knowhere/index/vector_index/IndexSPTAG.h" +#include "knowhere/index/vector_index/helpers/Definitions.h" +#include "knowhere/index/vector_index/helpers/SPTAGParameterMgr.h" + +namespace knowhere { + +CPUSPTAGRNG::CPUSPTAGRNG(const std::string& IndexType) { + if (IndexType == "KDT") { + index_ptr_ = SPTAG::VectorIndex::CreateInstance(SPTAG::IndexAlgoType::KDT, SPTAG::VectorValueType::Float); + index_ptr_->SetParameter("DistCalcMethod", "L2"); + index_type_ = SPTAG::IndexAlgoType::KDT; + } else { + index_ptr_ = SPTAG::VectorIndex::CreateInstance(SPTAG::IndexAlgoType::BKT, SPTAG::VectorValueType::Float); + index_ptr_->SetParameter("DistCalcMethod", "L2"); + index_type_ = SPTAG::IndexAlgoType::BKT; + } +} + +BinarySet +CPUSPTAGRNG::Serialize() { + std::string index_config; + std::vector index_blobs; + + std::shared_ptr> buffersize = index_ptr_->CalculateBufferSize(); + std::vector res(buffersize->size() + 1); + for (uint64_t i = 1; i < res.size(); i++) { + res[i] = new char[buffersize->at(i - 1)]; + auto ptr = &res[i][0]; + index_blobs.emplace_back(SPTAG::ByteArray((std::uint8_t*)ptr, buffersize->at(i - 1), false)); + } + + index_ptr_->SaveIndex(index_config, index_blobs); + + size_t length = index_config.length(); + char* cstr = new char[length]; + snprintf(cstr, length, "%s", index_config.c_str()); + + BinarySet binary_set; + auto sample = std::make_shared(); + sample.reset(static_cast(index_blobs[0].Data())); + auto tree = std::make_shared(); + tree.reset(static_cast(index_blobs[1].Data())); + auto graph = std::make_shared(); + graph.reset(static_cast(index_blobs[2].Data())); + auto deleteid = std::make_shared(); + deleteid.reset(static_cast(index_blobs[3].Data())); + auto metadata1 = std::make_shared(); + metadata1.reset(static_cast(index_blobs[4].Data())); + auto metadata2 = std::make_shared(); + metadata2.reset(static_cast(index_blobs[5].Data())); + auto config = std::make_shared(); + config.reset(static_cast((void*)cstr)); + + binary_set.Append("samples", sample, index_blobs[0].Length()); + binary_set.Append("tree", tree, index_blobs[1].Length()); + binary_set.Append("deleteid", deleteid, index_blobs[3].Length()); + binary_set.Append("metadata1", metadata1, index_blobs[4].Length()); + binary_set.Append("metadata2", metadata2, index_blobs[5].Length()); + binary_set.Append("config", config, length); + binary_set.Append("graph", graph, index_blobs[2].Length()); + + // MemoryIOWriter writer; + // size_t len = 0; + // for (int i = 0; i < 6; ++i) { + // len = index_blobs[i].Length(); + // assert(len != 0); + // writer(&len, sizeof(size_t), 1); + // writer(index_blobs[i].Data(), len, 1); + // len = 0; + // } + // writer(&length, sizeof(size_t), 1); + // writer(cstr, length, 1); + // auto data = std::make_shared(); + // data.reset(writer.data_); + // BinarySet binary_set; + // binary_set.Append("sptag", data, writer.total); + + // MemoryIOWriter writer; + // size_t len = 0; + // for (int i = 0; i < 6; ++i) { + // if (i == 2) continue; + // len = index_blobs[i].Length(); + // assert(len != 0); + // writer(&len, sizeof(size_t), 1); + // writer(index_blobs[i].Data(), len, 1); + // len = 0; + // } + // writer(&length, sizeof(size_t), 1); + // writer(cstr, length, 1); + // auto data = std::make_shared(); + // data.reset(writer.data_); + // BinarySet binary_set; + // binary_set.Append("sptag", data, writer.total); + // auto graph = std::make_shared(); + // graph.reset(static_cast(index_blobs[2].Data())); + // binary_set.Append("graph", graph, index_blobs[2].Length()); + + return binary_set; +} + +void +CPUSPTAGRNG::Load(const BinarySet& binary_set) { + std::string index_config; + std::vector index_blobs; + + auto samples = binary_set.GetByName("samples"); + index_blobs.push_back(SPTAG::ByteArray(samples->data.get(), samples->size, false)); + + auto tree = binary_set.GetByName("tree"); + index_blobs.push_back(SPTAG::ByteArray(tree->data.get(), tree->size, false)); + + auto graph = binary_set.GetByName("graph"); + index_blobs.push_back(SPTAG::ByteArray(graph->data.get(), graph->size, false)); + + auto deleteid = binary_set.GetByName("deleteid"); + index_blobs.push_back(SPTAG::ByteArray(deleteid->data.get(), deleteid->size, false)); + + auto metadata1 = binary_set.GetByName("metadata1"); + index_blobs.push_back(SPTAG::ByteArray(metadata1->data.get(), metadata1->size, false)); + + auto metadata2 = binary_set.GetByName("metadata2"); + index_blobs.push_back(SPTAG::ByteArray(metadata2->data.get(), metadata2->size, false)); + + auto config = binary_set.GetByName("config"); + index_config = reinterpret_cast(config->data.get()); + + // std::vector index_blobs; + // auto data = binary_set.GetByName("sptag"); + // MemoryIOReader reader; + // reader.total = data->size; + // reader.data_ = data->data.get(); + // size_t len = 0; + // for (int i = 0; i < 6; ++i) { + // reader(&len, sizeof(size_t), 1); + // assert(len != 0); + // auto binary = new uint8_t[len]; + // reader(binary, len, 1); + // index_blobs.emplace_back(SPTAG::ByteArray(binary, len, true)); + // len = 0; + // } + // reader(&len, sizeof(size_t), 1); + // assert(len != 0); + // auto config = new char[len]; + // reader(config, len, 1); + // std::string index_config = config; + // delete[] config; + + // std::vector index_blobs; + // auto data = binary_set.GetByName("sptag"); + // MemoryIOReader reader; + // reader.total = data->size; + // reader.data_ = data->data.get(); + // size_t len = 0; + // for (int i = 0; i < 6; ++i) { + // if (i == 2) { + // auto graph = binary_set.GetByName("graph"); + // index_blobs.emplace_back(SPTAG::ByteArray(graph->data.get(), graph->size, false)); + // continue; + // } + // reader(&len, sizeof(size_t), 1); + // assert(len != 0); + // auto binary = new uint8_t[len]; + // reader(binary, len, 1); + // index_blobs.emplace_back(SPTAG::ByteArray(binary, len, true)); + // len = 0; + // } + // reader(&len, sizeof(size_t), 1); + // assert(len != 0); + // auto config = new char[len]; + // reader(config, len, 1); + // std::string index_config = config; + // delete[] config; + index_ptr_->LoadIndex(index_config, index_blobs); +} + +// PreprocessorPtr +// CPUKDTRNG::BuildPreprocessor(const DatasetPtr &dataset, const Config &config) { +// return std::make_shared(); +//} + +IndexModelPtr +CPUSPTAGRNG::Train(const DatasetPtr& origin, const Config& train_config) { + SetParameters(train_config); + DatasetPtr dataset = origin->Clone(); + + // if (index_ptr_->GetDistCalcMethod() == SPTAG::DistCalcMethod::Cosine + // && preprocessor_) { + // preprocessor_->Preprocess(dataset); + //} + + auto vectorset = ConvertToVectorSet(dataset); + auto metaset = ConvertToMetadataSet(dataset); + index_ptr_->BuildIndex(vectorset, metaset); + + // TODO: return IndexModelPtr + return nullptr; +} + +void +CPUSPTAGRNG::Add(const DatasetPtr& origin, const Config& add_config) { + // SetParameters(add_config); + // DatasetPtr dataset = origin->Clone(); + // + // // if (index_ptr_->GetDistCalcMethod() == SPTAG::DistCalcMethod::Cosine + // // && preprocessor_) { + // // preprocessor_->Preprocess(dataset); + // //} + // + // auto vectorset = ConvertToVectorSet(dataset); + // auto metaset = ConvertToMetadataSet(dataset); + // index_ptr_->AddIndex(vectorset, metaset); +} + +void +CPUSPTAGRNG::SetParameters(const Config& config) { +#define Assign(param_name, str_name) \ + conf->param_name == INVALID_VALUE ? index_ptr_->SetParameter(str_name, std::to_string(build_cfg->param_name)) \ + : index_ptr_->SetParameter(str_name, std::to_string(conf->param_name)) + + if (index_type_ == SPTAG::IndexAlgoType::KDT) { + auto conf = std::dynamic_pointer_cast(config); + auto build_cfg = SPTAGParameterMgr::GetInstance().GetKDTParameters(); + + Assign(kdtnumber, "KDTNumber"); + Assign(numtopdimensionkdtsplit, "NumTopDimensionKDTSplit"); + Assign(samples, "Samples"); + Assign(tptnumber, "TPTNumber"); + Assign(tptleafsize, "TPTLeafSize"); + Assign(numtopdimensiontptsplit, "NumTopDimensionTPTSplit"); + Assign(neighborhoodsize, "NeighborhoodSize"); + Assign(graphneighborhoodscale, "GraphNeighborhoodScale"); + Assign(graphcefscale, "GraphCEFScale"); + Assign(refineiterations, "RefineIterations"); + Assign(cef, "CEF"); + Assign(maxcheckforrefinegraph, "MaxCheckForRefineGraph"); + Assign(numofthreads, "NumberOfThreads"); + Assign(maxcheck, "MaxCheck"); + Assign(thresholdofnumberofcontinuousnobetterpropagation, "ThresholdOfNumberOfContinuousNoBetterPropagation"); + Assign(numberofinitialdynamicpivots, "NumberOfInitialDynamicPivots"); + Assign(numberofotherdynamicpivots, "NumberOfOtherDynamicPivots"); + } else { + auto conf = std::dynamic_pointer_cast(config); + auto build_cfg = SPTAGParameterMgr::GetInstance().GetBKTParameters(); + + Assign(bktnumber, "BKTNumber"); + Assign(bktkmeansk, "BKTKMeansK"); + Assign(bktleafsize, "BKTLeafSize"); + Assign(samples, "Samples"); + Assign(tptnumber, "TPTNumber"); + Assign(tptleafsize, "TPTLeafSize"); + Assign(numtopdimensiontptsplit, "NumTopDimensionTPTSplit"); + Assign(neighborhoodsize, "NeighborhoodSize"); + Assign(graphneighborhoodscale, "GraphNeighborhoodScale"); + Assign(graphcefscale, "GraphCEFScale"); + Assign(refineiterations, "RefineIterations"); + Assign(cef, "CEF"); + Assign(maxcheckforrefinegraph, "MaxCheckForRefineGraph"); + Assign(numofthreads, "NumberOfThreads"); + Assign(maxcheck, "MaxCheck"); + Assign(thresholdofnumberofcontinuousnobetterpropagation, "ThresholdOfNumberOfContinuousNoBetterPropagation"); + Assign(numberofinitialdynamicpivots, "NumberOfInitialDynamicPivots"); + Assign(numberofotherdynamicpivots, "NumberOfOtherDynamicPivots"); + } +} + +DatasetPtr +CPUSPTAGRNG::Search(const DatasetPtr& dataset, const Config& config) { + SetParameters(config); + auto tensor = dataset->tensor()[0]; + auto p = (float*)tensor->raw_mutable_data(); + for (auto i = 0; i < 10; ++i) { + for (auto j = 0; j < 10; ++j) { + std::cout << p[i * 10 + j] << " "; + } + std::cout << std::endl; + } + std::vector query_results = ConvertToQueryResult(dataset, config); + +#pragma omp parallel for + for (auto i = 0; i < query_results.size(); ++i) { + auto target = (float*)query_results[i].GetTarget(); + std::cout << target[0] << ", " << target[1] << ", " << target[2] << std::endl; + index_ptr_->SearchIndex(query_results[i]); + } + + return ConvertToDataset(query_results); +} + +int64_t +CPUSPTAGRNG::Count() { + return index_ptr_->GetNumSamples(); +} + +int64_t +CPUSPTAGRNG::Dimension() { + return index_ptr_->GetFeatureDim(); +} + +VectorIndexPtr +CPUSPTAGRNG::Clone() { + KNOWHERE_THROW_MSG("not support"); +} + +void +CPUSPTAGRNG::Seal() { + return; // do nothing +} + +BinarySet +CPUSPTAGRNGIndexModel::Serialize() { + // KNOWHERE_THROW_MSG("not support"); // not support +} + +void +CPUSPTAGRNGIndexModel::Load(const BinarySet& binary) { + // KNOWHERE_THROW_MSG("not support"); // not support +} + +} // namespace knowhere diff --git a/core/src/index/knowhere/knowhere/index/vector_index/IndexKDT.h b/core/src/index/knowhere/knowhere/index/vector_index/IndexSPTAG.h similarity index 83% rename from core/src/index/knowhere/knowhere/index/vector_index/IndexKDT.h rename to core/src/index/knowhere/knowhere/index/vector_index/IndexSPTAG.h index f6d436995b..01380ce943 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/IndexKDT.h +++ b/core/src/index/knowhere/knowhere/index/vector_index/IndexSPTAG.h @@ -18,33 +18,37 @@ #pragma once #include + #include #include +#include + #include "VectorIndex.h" #include "knowhere/index/IndexModel.h" namespace knowhere { -class CPUKDTRNG : public VectorIndex { +class CPUSPTAGRNG : public VectorIndex { public: - CPUKDTRNG() { - index_ptr_ = SPTAG::VectorIndex::CreateInstance(SPTAG::IndexAlgoType::KDT, SPTAG::VectorValueType::Float); - index_ptr_->SetParameter("DistCalcMethod", "L2"); - } + explicit CPUSPTAGRNG(const std::string& IndexType); public: BinarySet Serialize() override; + VectorIndexPtr Clone() override; + void Load(const BinarySet& index_array) override; public: // PreprocessorPtr // BuildPreprocessor(const DatasetPtr &dataset, const Config &config) override; + int64_t Count() override; + int64_t Dimension() override; @@ -56,6 +60,7 @@ class CPUKDTRNG : public VectorIndex { DatasetPtr Search(const DatasetPtr& dataset, const Config& config) override; + void Seal() override; @@ -66,11 +71,12 @@ class CPUKDTRNG : public VectorIndex { private: PreprocessorPtr preprocessor_; std::shared_ptr index_ptr_; + SPTAG::IndexAlgoType index_type_; }; -using CPUKDTRNGPtr = std::shared_ptr; +using CPUSPTAGRNGPtr = std::shared_ptr; -class CPUKDTRNGIndexModel : public IndexModel { +class CPUSPTAGRNGIndexModel : public IndexModel { public: BinarySet Serialize() override; @@ -82,6 +88,6 @@ class CPUKDTRNGIndexModel : public IndexModel { std::shared_ptr index_; }; -using CPUKDTRNGIndexModelPtr = std::shared_ptr; +using CPUSPTAGRNGIndexModelPtr = std::shared_ptr; } // namespace knowhere diff --git a/core/src/index/knowhere/knowhere/index/vector_index/helpers/Cloner.cpp b/core/src/index/knowhere/knowhere/index/vector_index/helpers/Cloner.cpp index 5ff2bfc2e3..363e629eca 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/helpers/Cloner.cpp +++ b/core/src/index/knowhere/knowhere/index/vector_index/helpers/Cloner.cpp @@ -49,7 +49,7 @@ CopyCpuToGpu(const VectorIndexPtr& index, const int64_t& device_id, const Config if (auto cpu_index = std::dynamic_pointer_cast(index)) { return cpu_index->CopyCpuToGpu(device_id, config); } else if (auto cpu_index = std::dynamic_pointer_cast(index)) { - KNOWHERE_THROW_MSG("IVFPQ not support transfer to gpu"); + return cpu_index->CopyCpuToGpu(device_id, config); } else if (auto cpu_index = std::dynamic_pointer_cast(index)) { return cpu_index->CopyCpuToGpu(device_id, config); } else if (auto cpu_index = std::dynamic_pointer_cast(index)) { diff --git a/core/src/index/knowhere/knowhere/index/vector_index/helpers/IndexParameter.cpp b/core/src/index/knowhere/knowhere/index/vector_index/helpers/IndexParameter.cpp index 20f3388174..6e9ee55658 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/helpers/IndexParameter.cpp +++ b/core/src/index/knowhere/knowhere/index/vector_index/helpers/IndexParameter.cpp @@ -34,4 +34,26 @@ GetMetricType(METRICTYPE& type) { KNOWHERE_THROW_MSG("Metric type is invalid"); } +std::stringstream +IVFCfg::DumpImpl() { + auto ss = Cfg::DumpImpl(); + ss << ", nlist: " << nlist << ", nprobe: " << nprobe; + return ss; +} + +std::stringstream +IVFSQCfg::DumpImpl() { + auto ss = IVFCfg::DumpImpl(); + ss << ", nbits: " << nbits; + return ss; +} + +std::stringstream +NSGCfg::DumpImpl() { + auto ss = IVFCfg::DumpImpl(); + ss << ", knng: " << knng << ", search_length: " << search_length << ", out_degree: " << out_degree + << ", candidate: " << candidate_pool_size; + return ss; +} + } // namespace knowhere diff --git a/core/src/index/knowhere/knowhere/index/vector_index/helpers/IndexParameter.h b/core/src/index/knowhere/knowhere/index/vector_index/helpers/IndexParameter.h index b2854abef8..e30088ecdf 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/helpers/IndexParameter.h +++ b/core/src/index/knowhere/knowhere/index/vector_index/helpers/IndexParameter.h @@ -42,6 +42,32 @@ constexpr int64_t DEFAULT_OUT_DEGREE = INVALID_VALUE; constexpr int64_t DEFAULT_CANDIDATE_SISE = INVALID_VALUE; constexpr int64_t DEFAULT_NNG_K = INVALID_VALUE; +// SPTAG Config +constexpr int64_t DEFAULT_SAMPLES = INVALID_VALUE; +constexpr int64_t DEFAULT_TPTNUMBER = INVALID_VALUE; +constexpr int64_t DEFAULT_TPTLEAFSIZE = INVALID_VALUE; +constexpr int64_t DEFAULT_NUMTOPDIMENSIONTPTSPLIT = INVALID_VALUE; +constexpr int64_t DEFAULT_NEIGHBORHOODSIZE = INVALID_VALUE; +constexpr int64_t DEFAULT_GRAPHNEIGHBORHOODSCALE = INVALID_VALUE; +constexpr int64_t DEFAULT_GRAPHCEFSCALE = INVALID_VALUE; +constexpr int64_t DEFAULT_REFINEITERATIONS = INVALID_VALUE; +constexpr int64_t DEFAULT_CEF = INVALID_VALUE; +constexpr int64_t DEFAULT_MAXCHECKFORREFINEGRAPH = INVALID_VALUE; +constexpr int64_t DEFAULT_NUMOFTHREADS = INVALID_VALUE; +constexpr int64_t DEFAULT_MAXCHECK = INVALID_VALUE; +constexpr int64_t DEFAULT_THRESHOLDOFNUMBEROFCONTINUOUSNOBETTERPROPAGATION = INVALID_VALUE; +constexpr int64_t DEFAULT_NUMBEROFINITIALDYNAMICPIVOTS = INVALID_VALUE; +constexpr int64_t DEFAULT_NUMBEROFOTHERDYNAMICPIVOTS = INVALID_VALUE; + +// KDT Config +constexpr int64_t DEFAULT_KDTNUMBER = INVALID_VALUE; +constexpr int64_t DEFAULT_NUMTOPDIMENSIONKDTSPLIT = INVALID_VALUE; + +// BKT Config +constexpr int64_t DEFAULT_BKTNUMBER = INVALID_VALUE; +constexpr int64_t DEFAULT_BKTKMEANSK = INVALID_VALUE; +constexpr int64_t DEFAULT_BKTLEAFSIZE = INVALID_VALUE; + struct IVFCfg : public Cfg { int64_t nlist = DEFAULT_NLIST; int64_t nprobe = DEFAULT_NPROBE; @@ -53,6 +79,9 @@ struct IVFCfg : public Cfg { IVFCfg() = default; + std::stringstream + DumpImpl() override; + bool CheckValid() override { return true; @@ -69,6 +98,9 @@ struct IVFSQCfg : public IVFCfg { : IVFCfg(dim, k, gpu_id, nlist, nprobe, type), nbits(nbits) { } + std::stringstream + DumpImpl() override; + IVFSQCfg() = default; bool @@ -119,6 +151,9 @@ struct NSGCfg : public IVFCfg { NSGCfg() = default; + std::stringstream + DumpImpl() override; + bool CheckValid() override { return true; @@ -126,8 +161,57 @@ struct NSGCfg : public IVFCfg { }; using NSGConfig = std::shared_ptr; -struct KDTCfg : public Cfg { - int64_t tptnubmber = -1; +struct SPTAGCfg : public Cfg { + int64_t samples = DEFAULT_SAMPLES; + int64_t tptnumber = DEFAULT_TPTNUMBER; + int64_t tptleafsize = DEFAULT_TPTLEAFSIZE; + int64_t numtopdimensiontptsplit = DEFAULT_NUMTOPDIMENSIONTPTSPLIT; + int64_t neighborhoodsize = DEFAULT_NEIGHBORHOODSIZE; + int64_t graphneighborhoodscale = DEFAULT_GRAPHNEIGHBORHOODSCALE; + int64_t graphcefscale = DEFAULT_GRAPHCEFSCALE; + int64_t refineiterations = DEFAULT_REFINEITERATIONS; + int64_t cef = DEFAULT_CEF; + int64_t maxcheckforrefinegraph = DEFAULT_MAXCHECKFORREFINEGRAPH; + int64_t numofthreads = DEFAULT_NUMOFTHREADS; + int64_t maxcheck = DEFAULT_MAXCHECK; + int64_t thresholdofnumberofcontinuousnobetterpropagation = DEFAULT_THRESHOLDOFNUMBEROFCONTINUOUSNOBETTERPROPAGATION; + int64_t numberofinitialdynamicpivots = DEFAULT_NUMBEROFINITIALDYNAMICPIVOTS; + int64_t numberofotherdynamicpivots = DEFAULT_NUMBEROFOTHERDYNAMICPIVOTS; + + SPTAGCfg() = default; + + bool + CheckValid() override { + return true; + }; }; +using SPTAGConfig = std::shared_ptr; + +struct KDTCfg : public SPTAGCfg { + int64_t kdtnumber = DEFAULT_KDTNUMBER; + int64_t numtopdimensionkdtsplit = DEFAULT_NUMTOPDIMENSIONKDTSPLIT; + + KDTCfg() = default; + + bool + CheckValid() override { + return true; + }; +}; +using KDTConfig = std::shared_ptr; + +struct BKTCfg : public SPTAGCfg { + int64_t bktnumber = DEFAULT_BKTNUMBER; + int64_t bktkmeansk = DEFAULT_BKTKMEANSK; + int64_t bktleafsize = DEFAULT_BKTLEAFSIZE; + + BKTCfg() = default; + + bool + CheckValid() override { + return true; + }; +}; +using BKTConfig = std::shared_ptr; } // namespace knowhere diff --git a/core/src/index/knowhere/knowhere/index/vector_index/helpers/KDTParameterMgr.cpp b/core/src/index/knowhere/knowhere/index/vector_index/helpers/KDTParameterMgr.cpp deleted file mode 100644 index 19bf070dba..0000000000 --- a/core/src/index/knowhere/knowhere/index/vector_index/helpers/KDTParameterMgr.cpp +++ /dev/null @@ -1,55 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include - -#include "knowhere/index/vector_index/helpers/KDTParameterMgr.h" - -namespace knowhere { - -const std::vector& -KDTParameterMgr::GetKDTParameters() { - return kdt_parameters_; -} - -KDTParameterMgr::KDTParameterMgr() { - kdt_parameters_ = std::vector{ - {"KDTNumber", "1"}, - {"NumTopDimensionKDTSplit", "5"}, - {"NumSamplesKDTSplitConsideration", "100"}, - - {"TPTNumber", "1"}, - {"TPTLeafSize", "2000"}, - {"NumTopDimensionTPTSplit", "5"}, - - {"NeighborhoodSize", "32"}, - {"GraphNeighborhoodScale", "2"}, - {"GraphCEFScale", "2"}, - {"RefineIterations", "0"}, - {"CEF", "1000"}, - {"MaxCheckForRefineGraph", "10000"}, - - {"NumberOfThreads", "1"}, - - {"MaxCheck", "8192"}, - {"ThresholdOfNumberOfContinuousNoBetterPropagation", "3"}, - {"NumberOfInitialDynamicPivots", "50"}, - {"NumberOfOtherDynamicPivots", "4"}, - }; -} - -} // namespace knowhere diff --git a/core/src/index/knowhere/knowhere/index/vector_index/helpers/SPTAGParameterMgr.cpp b/core/src/index/knowhere/knowhere/index/vector_index/helpers/SPTAGParameterMgr.cpp new file mode 100644 index 0000000000..836f204c77 --- /dev/null +++ b/core/src/index/knowhere/knowhere/index/vector_index/helpers/SPTAGParameterMgr.cpp @@ -0,0 +1,75 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include + +#include "knowhere/index/vector_index/helpers/SPTAGParameterMgr.h" + +namespace knowhere { + +const KDTConfig& +SPTAGParameterMgr::GetKDTParameters() { + return kdt_config_; +} + +const BKTConfig& +SPTAGParameterMgr::GetBKTParameters() { + return bkt_config_; +} + +SPTAGParameterMgr::SPTAGParameterMgr() { + kdt_config_ = std::make_shared(); + kdt_config_->kdtnumber = 1; + kdt_config_->numtopdimensionkdtsplit = 5; + kdt_config_->samples = 100; + kdt_config_->tptnumber = 1; + kdt_config_->tptleafsize = 2000; + kdt_config_->numtopdimensiontptsplit = 5; + kdt_config_->neighborhoodsize = 32; + kdt_config_->graphneighborhoodscale = 2; + kdt_config_->graphcefscale = 2; + kdt_config_->refineiterations = 0; + kdt_config_->cef = 1000; + kdt_config_->maxcheckforrefinegraph = 10000; + kdt_config_->numofthreads = 1; + kdt_config_->maxcheck = 8192; + kdt_config_->thresholdofnumberofcontinuousnobetterpropagation = 3; + kdt_config_->numberofinitialdynamicpivots = 50; + kdt_config_->numberofotherdynamicpivots = 4; + + bkt_config_ = std::make_shared(); + bkt_config_->bktnumber = 1; + bkt_config_->bktkmeansk = 32; + bkt_config_->bktleafsize = 8; + bkt_config_->samples = 100; + bkt_config_->tptnumber = 1; + bkt_config_->tptleafsize = 2000; + bkt_config_->numtopdimensiontptsplit = 5; + bkt_config_->neighborhoodsize = 32; + bkt_config_->graphneighborhoodscale = 2; + bkt_config_->graphcefscale = 2; + bkt_config_->refineiterations = 0; + bkt_config_->cef = 1000; + bkt_config_->maxcheckforrefinegraph = 10000; + bkt_config_->numofthreads = 1; + bkt_config_->maxcheck = 8192; + bkt_config_->thresholdofnumberofcontinuousnobetterpropagation = 3; + bkt_config_->numberofinitialdynamicpivots = 50; + bkt_config_->numberofotherdynamicpivots = 4; +} + +} // namespace knowhere diff --git a/core/src/index/knowhere/knowhere/index/vector_index/helpers/KDTParameterMgr.h b/core/src/index/knowhere/knowhere/index/vector_index/helpers/SPTAGParameterMgr.h similarity index 66% rename from core/src/index/knowhere/knowhere/index/vector_index/helpers/KDTParameterMgr.h rename to core/src/index/knowhere/knowhere/index/vector_index/helpers/SPTAGParameterMgr.h index fe90761e17..6a6f7c48d1 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/helpers/KDTParameterMgr.h +++ b/core/src/index/knowhere/knowhere/index/vector_index/helpers/SPTAGParameterMgr.h @@ -22,31 +22,40 @@ #include #include +#include +#include "IndexParameter.h" + namespace knowhere { -using KDTParameter = std::pair; +using KDTConfig = std::shared_ptr; +using BKTConfig = std::shared_ptr; -class KDTParameterMgr { +class SPTAGParameterMgr { public: - const std::vector& + const KDTConfig& GetKDTParameters(); + const BKTConfig& + GetBKTParameters(); + public: - static KDTParameterMgr& + static SPTAGParameterMgr& GetInstance() { - static KDTParameterMgr instance; + static SPTAGParameterMgr instance; return instance; } - KDTParameterMgr(const KDTParameterMgr&) = delete; - KDTParameterMgr& - operator=(const KDTParameterMgr&) = delete; + SPTAGParameterMgr(const SPTAGParameterMgr&) = delete; + + SPTAGParameterMgr& + operator=(const SPTAGParameterMgr&) = delete; private: - KDTParameterMgr(); + SPTAGParameterMgr(); private: - std::vector kdt_parameters_; + KDTConfig kdt_config_; + BKTConfig bkt_config_; }; } // namespace knowhere diff --git a/core/src/index/thirdparty/SPTAG/AnnService/inc/Core/Common/Dataset.h b/core/src/index/thirdparty/SPTAG/AnnService/inc/Core/Common/Dataset.h index d00ea45365..0208f6d983 100644 --- a/core/src/index/thirdparty/SPTAG/AnnService/inc/Core/Common/Dataset.h +++ b/core/src/index/thirdparty/SPTAG/AnnService/inc/Core/Common/Dataset.h @@ -195,7 +195,7 @@ namespace SPTAG C = *((DimensionType*)pDataPointsMemFile); pDataPointsMemFile += sizeof(DimensionType); - Initialize(R, C, (T*)pDataPointsMemFile); + Initialize(R, C, (T*)pDataPointsMemFile, false); std::cout << "Load " << name << " (" << R << ", " << C << ") Finish!" << std::endl; return true; } diff --git a/core/src/index/thirdparty/versions.txt b/core/src/index/thirdparty/versions.txt index a2b16414c2..efcef26fa9 100644 --- a/core/src/index/thirdparty/versions.txt +++ b/core/src/index/thirdparty/versions.txt @@ -1,6 +1,7 @@ -ARROW_VERSION=apache-arrow-0.14.0 +ARROW_VERSION=apache-arrow-0.15.1 BOOST_VERSION=1.70.0 GTEST_VERSION=1.8.1 LAPACK_VERSION=v3.8.0 OPENBLAS_VERSION=v0.3.6 -FAISS_VERSION=branch-0.3.0 \ No newline at end of file +FAISS_VERSION=1.6.0 +MKL_VERSION=2019.5.281 diff --git a/core/src/index/unittest/CMakeLists.txt b/core/src/index/unittest/CMakeLists.txt index 2e84908cd7..93ae63a9ec 100644 --- a/core/src/index/unittest/CMakeLists.txt +++ b/core/src/index/unittest/CMakeLists.txt @@ -2,26 +2,32 @@ include_directories(${INDEX_SOURCE_DIR}/thirdparty) include_directories(${INDEX_SOURCE_DIR}/thirdparty/SPTAG/AnnService) include_directories(${INDEX_SOURCE_DIR}/knowhere) include_directories(${INDEX_SOURCE_DIR}) -include_directories(/usr/local/cuda/include) -link_directories(/usr/local/cuda/lib64) - -message(STATUS "arrow prefix: ${ARROW_PREFIX}") -message(STATUS "libjemalloc_pic path: ${ARROW_PREFIX}/lib/libjemalloc_pic.a") set(depend_libs gtest gmock gtest_main gmock_main - faiss openblas lapack - arrow "${ARROW_PREFIX}/lib/libjemalloc_pic.a" + faiss + arrow "${ARROW_LIB_DIR}/libjemalloc_pic.a" ) +if (FAISS_WITH_MKL) + set(depend_libs ${depend_libs} + "-Wl,--start-group \ + ${MKL_LIB_PATH}/libmkl_intel_ilp64.a \ + ${MKL_LIB_PATH}/libmkl_gnu_thread.a \ + ${MKL_LIB_PATH}/libmkl_core.a \ + -Wl,--end-group -lgomp -lpthread -lm -ldl" + ) +else () + set(depend_libs ${depend_libs} + lapack + openblas) +endif () set(basic_libs - cudart cublas gomp gfortran pthread ) set(util_srcs - ${MILVUS_ENGINE_SRC}/external/easyloggingpp/easylogging++.cc - ${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_index/helpers/FaissGpuResourceMgr.cpp + ${MILVUS_THIRDPARTY_SRC}/easyloggingpp/easylogging++.cc ${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_index/helpers/FaissIO.cpp ${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_index/helpers/IndexParameter.cpp ${INDEX_SOURCE_DIR}/knowhere/knowhere/adapter/Structure.cpp @@ -31,60 +37,80 @@ set(util_srcs ${INDEX_SOURCE_DIR}/unittest/utils.cpp ) +if (KNOWHERE_GPU_VERSION) + include_directories(${CUDA_INCLUDE_DIRS}) + link_directories("${CUDA_TOOLKIT_ROOT_DIR}/lib64") + set(cuda_lib + cudart + cublas + ) + set(basic_libs ${basic_libs} + ${cuda_lib} + ) + set(util_srcs ${util_srcs} + ${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_index/helpers/FaissGpuResourceMgr.cpp + ) +endif () + # set(ivf_srcs - ${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_index/helpers/Cloner.cpp ${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_index/IndexIVF.cpp - ${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_index/IndexGPUIVF.cpp ${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_index/IndexIVFSQ.cpp - ${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_index/IndexGPUIVFSQ.cpp ${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_index/IndexIVFPQ.cpp - ${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_index/IndexGPUIVFPQ.cpp - ${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_index/IndexIVFSQHybrid.cpp ${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_index/IndexIDMAP.cpp ${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_index/FaissBaseIndex.cpp ) -if(NOT TARGET test_ivf) +if (KNOWHERE_GPU_VERSION) + set(ivf_srcs ${ivf_srcs} + ${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_index/IndexGPUIDMAP.cpp + ${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_index/helpers/Cloner.cpp + ${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_index/IndexGPUIVF.cpp + ${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_index/IndexGPUIVFSQ.cpp + ${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_index/IndexGPUIVFPQ.cpp + ${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_index/IndexIVFSQHybrid.cpp + ) +endif () +if (NOT TARGET test_ivf) add_executable(test_ivf test_ivf.cpp ${ivf_srcs} ${util_srcs}) -endif() +endif () target_link_libraries(test_ivf ${depend_libs} ${unittest_libs} ${basic_libs}) # -set(idmap_srcs - ${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_index/IndexIDMAP.cpp - ) -if(NOT TARGET test_idmap) - add_executable(test_idmap test_idmap.cpp ${idmap_srcs} ${ivf_srcs} ${util_srcs}) -endif() +if (NOT TARGET test_idmap) + add_executable(test_idmap test_idmap.cpp ${ivf_srcs} ${util_srcs}) +endif () target_link_libraries(test_idmap ${depend_libs} ${unittest_libs} ${basic_libs}) -# -set(kdt_srcs +# +set(sptag_srcs ${INDEX_SOURCE_DIR}/knowhere/knowhere/adapter/SptagAdapter.cpp ${INDEX_SOURCE_DIR}/knowhere/knowhere/index/preprocessor/Normalize.cpp - ${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_index/helpers/KDTParameterMgr.cpp - ${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_index/IndexKDT.cpp + ${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_index/helpers/SPTAGParameterMgr.cpp + ${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_index/IndexSPTAG.cpp ) -if(NOT TARGET test_kdt) - add_executable(test_kdt test_kdt.cpp ${kdt_srcs} ${util_srcs}) -endif() -target_link_libraries(test_kdt +if (NOT TARGET test_sptag) + add_executable(test_sptag test_sptag.cpp ${sptag_srcs} ${util_srcs}) +endif () +target_link_libraries(test_sptag SPTAGLibStatic ${depend_libs} ${unittest_libs} ${basic_libs}) -add_executable(test_gpuresource test_gpuresource.cpp ${util_srcs} ${ivf_srcs}) -target_link_libraries(test_gpuresource ${depend_libs} ${unittest_libs} ${basic_libs}) +if (KNOWHERE_GPU_VERSION) + add_executable(test_gpuresource test_gpuresource.cpp ${util_srcs} ${ivf_srcs}) + target_link_libraries(test_gpuresource ${depend_libs} ${unittest_libs} ${basic_libs}) -add_executable(test_customized_index test_customized_index.cpp ${util_srcs} ${ivf_srcs}) -target_link_libraries(test_customized_index ${depend_libs} ${unittest_libs} ${basic_libs}) + add_executable(test_customized_index test_customized_index.cpp ${util_srcs} ${ivf_srcs}) + target_link_libraries(test_customized_index ${depend_libs} ${unittest_libs} ${basic_libs}) +endif () install(TARGETS test_ivf DESTINATION unittest) install(TARGETS test_idmap DESTINATION unittest) -install(TARGETS test_kdt DESTINATION unittest) -install(TARGETS test_gpuresource DESTINATION unittest) -install(TARGETS test_customized_index DESTINATION unittest) - +install(TARGETS test_sptag DESTINATION unittest) +if (KNOWHERE_GPU_VERSION) + install(TARGETS test_gpuresource DESTINATION unittest) + install(TARGETS test_customized_index DESTINATION unittest) +endif () #add_subdirectory(faiss_ori) #add_subdirectory(faiss_benchmark) add_subdirectory(test_nsg) diff --git a/core/src/index/unittest/Helper.h b/core/src/index/unittest/Helper.h index 8d4bb0f4ae..5d7df41b69 100644 --- a/core/src/index/unittest/Helper.h +++ b/core/src/index/unittest/Helper.h @@ -18,13 +18,16 @@ #include #include -#include "knowhere/index/vector_index/IndexGPUIVF.h" -#include "knowhere/index/vector_index/IndexGPUIVFPQ.h" -#include "knowhere/index/vector_index/IndexGPUIVFSQ.h" #include "knowhere/index/vector_index/IndexIVF.h" #include "knowhere/index/vector_index/IndexIVFPQ.h" #include "knowhere/index/vector_index/IndexIVFSQ.h" + +#ifdef MILVUS_GPU_VERSION +#include "knowhere/index/vector_index/IndexGPUIVF.h" +#include "knowhere/index/vector_index/IndexGPUIVFPQ.h" +#include "knowhere/index/vector_index/IndexGPUIVFSQ.h" #include "knowhere/index/vector_index/IndexIVFSQHybrid.h" +#endif int DEVICEID = 0; constexpr int64_t DIM = 128; @@ -41,16 +44,18 @@ IndexFactory(const std::string& type) { return std::make_shared(); } else if (type == "IVFPQ") { return std::make_shared(); + } else if (type == "IVFSQ") { + return std::make_shared(); +#ifdef MILVUS_GPU_VERSION } else if (type == "GPUIVF") { return std::make_shared(DEVICEID); } else if (type == "GPUIVFPQ") { return std::make_shared(DEVICEID); - } else if (type == "IVFSQ") { - return std::make_shared(); } else if (type == "GPUIVFSQ") { return std::make_shared(DEVICEID); } else if (type == "IVFSQHybrid") { return std::make_shared(DEVICEID); +#endif } } @@ -110,11 +115,15 @@ class TestGpuIndexBase : public ::testing::Test { protected: void SetUp() override { +#ifdef MILVUS_GPU_VERSION knowhere::FaissGpuResourceMgr::GetInstance().InitDevice(DEVICEID, PINMEM, TEMPMEM, RESNUM); +#endif } void TearDown() override { +#ifdef MILVUS_GPU_VERSION knowhere::FaissGpuResourceMgr::GetInstance().Free(); +#endif } }; diff --git a/core/src/index/unittest/faiss_benchmark/CMakeLists.txt b/core/src/index/unittest/faiss_benchmark/CMakeLists.txt index 556364b68a..e2fd7a6c85 100644 --- a/core/src/index/unittest/faiss_benchmark/CMakeLists.txt +++ b/core/src/index/unittest/faiss_benchmark/CMakeLists.txt @@ -1,24 +1,50 @@ -include_directories(${INDEX_SOURCE_DIR}/thirdparty) -include_directories(${INDEX_SOURCE_DIR}/include) -include_directories(/usr/local/cuda/include) -include_directories(/usr/local/hdf5/include) +if (KNOWHERE_GPU_VERSION) -link_directories(/usr/local/cuda/lib64) -link_directories(/usr/local/hdf5/lib) + include_directories(${INDEX_SOURCE_DIR}/thirdparty) + include_directories(${INDEX_SOURCE_DIR}/include) + include_directories(/usr/local/cuda/include) + include_directories(/usr/local/hdf5/include) -set(unittest_libs - gtest gmock gtest_main gmock_main) + link_directories(/usr/local/cuda/lib64) + link_directories(/usr/local/hdf5/lib) -set(depend_libs - faiss openblas lapack hdf5 - arrow ${ARROW_PREFIX}/lib/libjemalloc_pic.a - ) + set(unittest_libs + gtest gmock gtest_main gmock_main) -set(basic_libs - cudart cublas - gomp gfortran pthread - ) + set(depend_libs + faiss hdf5 + arrow ${ARROW_LIB_DIR}/libjemalloc_pic.a + ) + if (FAISS_WITH_MKL) + set(depend_libs ${depend_libs} + "-Wl,--start-group \ + ${MKL_LIB_PATH}/libmkl_intel_ilp64.a \ + ${MKL_LIB_PATH}/libmkl_gnu_thread.a \ + ${MKL_LIB_PATH}/libmkl_core.a \ + -Wl,--end-group -lgomp -lpthread -lm -ldl" + ) + else () + set(depend_libs ${depend_libs} + lapack + openblas) + endif () -add_executable(test_faiss_benchmark faiss_benchmark_test.cpp) -target_link_libraries(test_faiss_benchmark ${depend_libs} ${unittest_libs} ${basic_libs}) -install(TARGETS test_faiss_benchmark DESTINATION unittest) + set(basic_libs + gomp gfortran pthread + ) + + include_directories(${CUDA_INCLUDE_DIRS}) + link_directories("${CUDA_TOOLKIT_ROOT_DIR}/lib64") + set(cuda_lib + cudart + cublas + ) + set(basic_libs ${basic_libs} + ${cuda_lib} + ) + + add_executable(test_faiss_benchmark faiss_benchmark_test.cpp) + target_link_libraries(test_faiss_benchmark ${depend_libs} ${unittest_libs} ${basic_libs}) + install(TARGETS test_faiss_benchmark DESTINATION unittest) + +endif () diff --git a/core/src/index/unittest/faiss_ori/CMakeLists.txt b/core/src/index/unittest/faiss_ori/CMakeLists.txt index d01463aaab..907076ca6e 100644 --- a/core/src/index/unittest/faiss_ori/CMakeLists.txt +++ b/core/src/index/unittest/faiss_ori/CMakeLists.txt @@ -1,26 +1,49 @@ -include_directories(${INDEX_SOURCE_DIR}/thirdparty) -include_directories(${INDEX_SOURCE_DIR}/include) -include_directories(/usr/local/cuda/include) -link_directories(/usr/local/cuda/lib64) +if (KNOWHERE_GPU_VERSION) -set(unittest_libs - gtest gmock gtest_main gmock_main) + include_directories(${INDEX_SOURCE_DIR}/thirdparty) + include_directories(${INDEX_SOURCE_DIR}/include) -set(depend_libs - faiss openblas lapack - arrow ${ARROW_PREFIX}/lib/libjemalloc_pic.a - ) + set(unittest_libs + gtest gmock gtest_main gmock_main) -set(basic_libs - cudart cublas - gomp gfortran pthread - ) + set(depend_libs + faiss + arrow ${ARROW_LIB_DIR}/libjemalloc_pic.a + ) + if (FAISS_WITH_MKL) + set(depend_libs ${depend_libs} + "-Wl,--start-group \ + ${MKL_LIB_PATH}/libmkl_intel_ilp64.a \ + ${MKL_LIB_PATH}/libmkl_gnu_thread.a \ + ${MKL_LIB_PATH}/libmkl_core.a \ + -Wl,--end-group -lgomp -lpthread -lm -ldl" + ) + else () + set(depend_libs ${depend_libs} + lapack + openblas) + endif () + set(basic_libs + gomp gfortran pthread + ) -# -if(NOT TARGET test_gpu) - add_executable(test_gpu gpuresource_test.cpp) -endif() -target_link_libraries(test_gpu ${depend_libs} ${unittest_libs} ${basic_libs}) + include_directories(${CUDA_INCLUDE_DIRS}) + link_directories("${CUDA_TOOLKIT_ROOT_DIR}/lib64") + set(cuda_lib + cudart + cublas + ) + set(basic_libs ${basic_libs} + ${cuda_lib} + ) -install(TARGETS test_gpu DESTINATION unittest) \ No newline at end of file + # + if (NOT TARGET test_gpu) + add_executable(test_gpu gpuresource_test.cpp) + endif () + target_link_libraries(test_gpu ${depend_libs} ${unittest_libs} ${basic_libs}) + + install(TARGETS test_gpu DESTINATION unittest) + +endif () \ No newline at end of file diff --git a/core/src/index/unittest/test_idmap.cpp b/core/src/index/unittest/test_idmap.cpp index d1ff3ee046..98a554199f 100644 --- a/core/src/index/unittest/test_idmap.cpp +++ b/core/src/index/unittest/test_idmap.cpp @@ -21,8 +21,10 @@ #include "knowhere/adapter/Structure.h" #include "knowhere/common/Exception.h" #include "knowhere/index/vector_index/IndexIDMAP.h" +#ifdef MILVUS_GPU_VERSION +#include "knowhere/index/vector_index/IndexGPUIDMAP.h" #include "knowhere/index/vector_index/helpers/Cloner.h" - +#endif #include "Helper.h" #include "unittest/utils.h" @@ -116,6 +118,7 @@ TEST_F(IDMAPTest, idmap_serialize) { } } +#ifdef MILVUS_GPU_VERSION TEST_F(IDMAPTest, copy_test) { ASSERT_TRUE(!xb.empty()); @@ -175,3 +178,4 @@ TEST_F(IDMAPTest, copy_test) { AssertAnns(device_result, nq, k); } } +#endif diff --git a/core/src/index/unittest/test_ivf.cpp b/core/src/index/unittest/test_ivf.cpp index 8b17e08272..79ef1aa170 100644 --- a/core/src/index/unittest/test_ivf.cpp +++ b/core/src/index/unittest/test_ivf.cpp @@ -20,13 +20,24 @@ #include #include +#ifdef MILVUS_GPU_VERSION #include +#endif #include "knowhere/common/Exception.h" #include "knowhere/common/Timer.h" -#include "knowhere/index/vector_index/IndexGPUIVF.h" + #include "knowhere/index/vector_index/IndexIVF.h" +#include "knowhere/index/vector_index/IndexIVFPQ.h" +#include "knowhere/index/vector_index/IndexIVFSQ.h" + +#ifdef MILVUS_GPU_VERSION +#include "knowhere/index/vector_index/IndexGPUIVF.h" +#include "knowhere/index/vector_index/IndexGPUIVFPQ.h" +#include "knowhere/index/vector_index/IndexGPUIVFSQ.h" +#include "knowhere/index/vector_index/IndexIVFSQHybrid.h" #include "knowhere/index/vector_index/helpers/Cloner.h" +#endif #include "unittest/Helper.h" #include "unittest/utils.h" @@ -39,8 +50,9 @@ class IVFTest : public DataGen, public TestWithParam<::std::tuplearray()[0]; - auto ids_p2 = p2->array()[0]; + auto ids_p1 = p1->ids(); + auto ids_p2 = p2->ids(); for (int i = 0; i < nq * k; ++i) { - EXPECT_EQ(*(ids_p2->data()->GetValues(1, i)), *(ids_p1->data()->GetValues(1, i))); + EXPECT_EQ(*((int64_t*)(ids_p2) + i), *((int64_t*)(ids_p1) + i)); + // EXPECT_EQ(*(ids_p2->data()->GetValues(1, i)), *(ids_p1->data()->GetValues(1, + // i))); } }; @@ -194,7 +213,7 @@ TEST_P(IVFTest, clone_test) { { // copy from gpu to cpu - std::vector support_idx_vec{"GPUIVF", "GPUIVFSQ", "IVFSQHybrid"}; + std::vector support_idx_vec{"GPUIVF", "GPUIVFSQ", "GPUIVFPQ", "IVFSQHybrid"}; auto finder = std::find(support_idx_vec.cbegin(), support_idx_vec.cend(), index_type); if (finder != support_idx_vec.cend()) { EXPECT_NO_THROW({ @@ -219,7 +238,7 @@ TEST_P(IVFTest, clone_test) { { // copy to gpu - std::vector support_idx_vec{"IVF", "GPUIVF", "IVFSQ", "GPUIVFSQ"}; + std::vector support_idx_vec{"IVF", "GPUIVF", "IVFSQ", "GPUIVFSQ", "IVFPQ", "GPUIVFPQ"}; auto finder = std::find(support_idx_vec.cbegin(), support_idx_vec.cend(), index_type); if (finder != support_idx_vec.cend()) { EXPECT_NO_THROW({ @@ -238,7 +257,9 @@ TEST_P(IVFTest, clone_test) { } } } +#endif +#ifdef MILVUS_GPU_VERSION #ifdef CUSTOMIZATION TEST_P(IVFTest, gpu_seal_test) { std::vector support_idx_vec{"GPUIVF", "GPUIVFSQ", "IVFSQHybrid"}; @@ -271,5 +292,5 @@ TEST_P(IVFTest, gpu_seal_test) { auto with_seal = tc.RecordSection("With seal"); ASSERT_GE(without_seal, with_seal); } - +#endif #endif diff --git a/core/src/index/unittest/test_kdt.cpp b/core/src/index/unittest/test_kdt.cpp deleted file mode 100644 index 5400881875..0000000000 --- a/core/src/index/unittest/test_kdt.cpp +++ /dev/null @@ -1,140 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include - -#include -#include - -#include "knowhere/adapter/SptagAdapter.h" -#include "knowhere/adapter/Structure.h" -#include "knowhere/common/Exception.h" -#include "knowhere/index/vector_index/IndexKDT.h" -#include "knowhere/index/vector_index/helpers/Definitions.h" - -#include "unittest/utils.h" - -using ::testing::Combine; -using ::testing::TestWithParam; -using ::testing::Values; - -class KDTTest : public DataGen, public ::testing::Test { - protected: - void - SetUp() override { - Generate(96, 1000, 10); - index_ = std::make_shared(); - - auto tempconf = std::make_shared(); - tempconf->tptnubmber = 1; - tempconf->k = 10; - conf = tempconf; - - Init_with_default(); - } - - protected: - knowhere::Config conf; - std::shared_ptr index_ = nullptr; -}; - -// TODO(lxj): add test about count() and dimension() -TEST_F(KDTTest, kdt_basic) { - assert(!xb.empty()); - - auto preprocessor = index_->BuildPreprocessor(base_dataset, conf); - index_->set_preprocessor(preprocessor); - - auto model = index_->Train(base_dataset, conf); - index_->set_index_model(model); - index_->Add(base_dataset, conf); - auto result = index_->Search(query_dataset, conf); - AssertAnns(result, nq, k); - - { - auto ids = result->array()[0]; - auto dists = result->array()[1]; - - std::stringstream ss_id; - std::stringstream ss_dist; - for (auto i = 0; i < nq; i++) { - for (auto j = 0; j < k; ++j) { - ss_id << *ids->data()->GetValues(1, i * k + j) << " "; - ss_dist << *dists->data()->GetValues(1, i * k + j) << " "; - } - ss_id << std::endl; - ss_dist << std::endl; - } - std::cout << "id\n" << ss_id.str() << std::endl; - std::cout << "dist\n" << ss_dist.str() << std::endl; - } -} - -// TODO(zirui): enable test -// TEST_F(KDTTest, kdt_serialize) { -// assert(!xb.empty()); -// -// auto preprocessor = index_->BuildPreprocessor(base_dataset, conf); -// index_->set_preprocessor(preprocessor); -// -// auto model = index_->Train(base_dataset, conf); -// // index_->Add(base_dataset, conf); -// auto binaryset = index_->Serialize(); -// auto new_index = std::make_shared(); -// new_index->Load(binaryset); -// auto result = new_index->Search(query_dataset, conf); -// AssertAnns(result, nq, k); -// PrintResult(result, nq, k); -// ASSERT_EQ(new_index->Count(), nb); -// ASSERT_EQ(new_index->Dimension(), dim); -// ASSERT_THROW({ new_index->Clone(); }, knowhere::KnowhereException); -// ASSERT_NO_THROW({ new_index->Seal(); }); -// -// { -// int fileno = 0; -// const std::string& base_name = "/tmp/kdt_serialize_test_bin_"; -// std::vector filename_list; -// std::vector> meta_list; -// for (auto& iter : binaryset.binary_map_) { -// const std::string& filename = base_name + std::to_string(fileno); -// FileIOWriter writer(filename); -// writer(iter.second->data.get(), iter.second->size); -// -// meta_list.emplace_back(std::make_pair(iter.first, iter.second->size)); -// filename_list.push_back(filename); -// ++fileno; -// } -// -// knowhere::BinarySet load_data_list; -// for (int i = 0; i < filename_list.size() && i < meta_list.size(); ++i) { -// auto bin_size = meta_list[i].second; -// FileIOReader reader(filename_list[i]); -// -// auto load_data = new uint8_t[bin_size]; -// reader(load_data, bin_size); -// auto data = std::make_shared(); -// data.reset(load_data); -// load_data_list.Append(meta_list[i].first, data, bin_size); -// } -// -// auto new_index = std::make_shared(); -// new_index->Load(load_data_list); -// auto result = new_index->Search(query_dataset, conf); -// AssertAnns(result, nq, k); -// PrintResult(result, nq, k); -// } -//} diff --git a/core/src/index/unittest/test_nsg/CMakeLists.txt b/core/src/index/unittest/test_nsg/CMakeLists.txt index 3d22051d82..82dbb5110c 100644 --- a/core/src/index/unittest/test_nsg/CMakeLists.txt +++ b/core/src/index/unittest/test_nsg/CMakeLists.txt @@ -4,14 +4,13 @@ add_definitions(-std=c++11 -O3 -lboost -march=native -Wall -DINFO) -find_package(OpenMP) -if (OPENMP_FOUND) +find_package(OpenMP REQUIRED) +if (OpenMP_FOUND) set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${OpenMP_C_FLAGS}") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS}") else () message(FATAL_ERROR "no OpenMP supprot") endif () -message(${OpenMP_CXX_FLAGS}) include_directories(${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_index/nsg) aux_source_directory(${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_index/nsg nsg_src) @@ -20,9 +19,9 @@ set(interface_src ${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_index/IndexNSG.cpp ) -if(NOT TARGET test_nsg) +if (NOT TARGET test_nsg) add_executable(test_nsg test_nsg.cpp ${interface_src} ${nsg_src} ${util_srcs} ${ivf_srcs}) -endif() +endif () target_link_libraries(test_nsg ${depend_libs} ${unittest_libs} ${basic_libs}) ############################## diff --git a/core/src/index/unittest/test_nsg/test_nsg.cpp b/core/src/index/unittest/test_nsg/test_nsg.cpp index 11b9becce4..47c014e691 100644 --- a/core/src/index/unittest/test_nsg/test_nsg.cpp +++ b/core/src/index/unittest/test_nsg/test_nsg.cpp @@ -21,7 +21,9 @@ #include "knowhere/common/Exception.h" #include "knowhere/index/vector_index/FaissBaseIndex.h" #include "knowhere/index/vector_index/IndexNSG.h" +#ifdef MILVUS_GPU_VERSION #include "knowhere/index/vector_index/helpers/FaissGpuResourceMgr.h" +#endif #include "knowhere/index/vector_index/nsg/NSGIO.h" #include "unittest/utils.h" @@ -37,7 +39,9 @@ class NSGInterfaceTest : public DataGen, public ::testing::Test { void SetUp() override { // Init_with_default(); +#ifdef MILVUS_GPU_VERSION knowhere::FaissGpuResourceMgr::GetInstance().InitDevice(DEVICEID, 1024 * 1024 * 200, 1024 * 1024 * 600, 2); +#endif Generate(256, 1000000 / 100, 1); index_ = std::make_shared(); @@ -60,7 +64,9 @@ class NSGInterfaceTest : public DataGen, public ::testing::Test { void TearDown() override { +#ifdef MILVUS_GPU_VERSION knowhere::FaissGpuResourceMgr::GetInstance().Free(); +#endif } protected: diff --git a/core/src/index/unittest/test_sptag.cpp b/core/src/index/unittest/test_sptag.cpp new file mode 100644 index 0000000000..d472dab0d7 --- /dev/null +++ b/core/src/index/unittest/test_sptag.cpp @@ -0,0 +1,154 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include + +#include +#include + +#include "knowhere/adapter/SptagAdapter.h" +#include "knowhere/adapter/Structure.h" +#include "knowhere/common/Exception.h" +#include "knowhere/index/vector_index/IndexSPTAG.h" +#include "knowhere/index/vector_index/helpers/Definitions.h" + +#include "unittest/utils.h" + +using ::testing::Combine; +using ::testing::TestWithParam; +using ::testing::Values; + +class SPTAGTest : public DataGen, public TestWithParam { + protected: + void + SetUp() override { + IndexType = GetParam(); + Generate(128, 100, 5); + index_ = std::make_shared(IndexType); + if (IndexType == "KDT") { + auto tempconf = std::make_shared(); + tempconf->tptnumber = 1; + tempconf->k = 10; + conf = tempconf; + } else { + auto tempconf = std::make_shared(); + tempconf->tptnumber = 1; + tempconf->k = 10; + conf = tempconf; + } + + Init_with_default(); + } + + protected: + knowhere::Config conf; + std::shared_ptr index_ = nullptr; + std::string IndexType; +}; + +INSTANTIATE_TEST_CASE_P(SPTAGParameters, SPTAGTest, Values("KDT", "BKT")); + +// TODO(lxj): add test about count() and dimension() +TEST_P(SPTAGTest, sptag_basic) { + assert(!xb.empty()); + + auto preprocessor = index_->BuildPreprocessor(base_dataset, conf); + index_->set_preprocessor(preprocessor); + + auto model = index_->Train(base_dataset, conf); + index_->set_index_model(model); + index_->Add(base_dataset, conf); + auto result = index_->Search(query_dataset, conf); + AssertAnns(result, nq, k); + + { + // auto ids = result->array()[0]; + // auto dists = result->array()[1]; + auto ids = result->ids(); + auto dists = result->dist(); + + std::stringstream ss_id; + std::stringstream ss_dist; + for (auto i = 0; i < nq; i++) { + for (auto j = 0; j < k; ++j) { + // ss_id << *ids->data()->GetValues(1, i * k + j) << " "; + // ss_dist << *dists->data()->GetValues(1, i * k + j) << " "; + ss_id << *((int64_t*)(ids) + i * k + j) << " "; + ss_dist << *((float*)(dists) + i * k + j) << " "; + } + ss_id << std::endl; + ss_dist << std::endl; + } + std::cout << "id\n" << ss_id.str() << std::endl; + std::cout << "dist\n" << ss_dist.str() << std::endl; + } +} + +TEST_P(SPTAGTest, sptag_serialize) { + assert(!xb.empty()); + + auto preprocessor = index_->BuildPreprocessor(base_dataset, conf); + index_->set_preprocessor(preprocessor); + + auto model = index_->Train(base_dataset, conf); + + index_->Add(base_dataset, conf); + auto binaryset = index_->Serialize(); + auto new_index = std::make_shared(IndexType); + new_index->Load(binaryset); + auto result = new_index->Search(query_dataset, conf); + AssertAnns(result, nq, k); + PrintResult(result, nq, k); + ASSERT_EQ(new_index->Count(), nb); + ASSERT_EQ(new_index->Dimension(), dim); + // ASSERT_THROW({ new_index->Clone(); }, knowhere::KnowhereException); + // ASSERT_NO_THROW({ new_index->Seal(); }); + + { + int fileno = 0; + const std::string& base_name = "/tmp/sptag_serialize_test_bin_"; + std::vector filename_list; + std::vector> meta_list; + for (auto& iter : binaryset.binary_map_) { + const std::string& filename = base_name + std::to_string(fileno); + FileIOWriter writer(filename); + writer(iter.second->data.get(), iter.second->size); + + meta_list.emplace_back(std::make_pair(iter.first, iter.second->size)); + filename_list.push_back(filename); + ++fileno; + } + + knowhere::BinarySet load_data_list; + for (int i = 0; i < filename_list.size() && i < meta_list.size(); ++i) { + auto bin_size = meta_list[i].second; + FileIOReader reader(filename_list[i]); + + auto load_data = new uint8_t[bin_size]; + reader(load_data, bin_size); + auto data = std::make_shared(); + data.reset(load_data); + load_data_list.Append(meta_list[i].first, data, bin_size); + } + + auto new_index = std::make_shared(IndexType); + new_index->Load(load_data_list); + auto result = new_index->Search(query_dataset, conf); + AssertAnns(result, nq, k); + PrintResult(result, nq, k); + } +} diff --git a/core/src/index/unittest/utils.cpp b/core/src/index/unittest/utils.cpp index d4a59bafbb..11dad4a8b9 100644 --- a/core/src/index/unittest/utils.cpp +++ b/core/src/index/unittest/utils.cpp @@ -151,23 +151,26 @@ generate_query_dataset(int64_t nb, int64_t dim, float* xb) { void AssertAnns(const knowhere::DatasetPtr& result, const int& nq, const int& k) { - auto ids = result->array()[0]; + auto ids = result->ids(); for (auto i = 0; i < nq; i++) { - EXPECT_EQ(i, *(ids->data()->GetValues(1, i * k))); + EXPECT_EQ(i, *((int64_t*)(ids) + i * k)); + // EXPECT_EQ(i, *(ids->data()->GetValues(1, i * k))); } } void PrintResult(const knowhere::DatasetPtr& result, const int& nq, const int& k) { - auto ids = result->array()[0]; - auto dists = result->array()[1]; + auto ids = result->ids(); + auto dists = result->dist(); std::stringstream ss_id; std::stringstream ss_dist; - for (auto i = 0; i < 10; i++) { + for (auto i = 0; i < nq; i++) { for (auto j = 0; j < k; ++j) { - ss_id << *(ids->data()->GetValues(1, i * k + j)) << " "; - ss_dist << *(dists->data()->GetValues(1, i * k + j)) << " "; + // ss_id << *(ids->data()->GetValues(1, i * k + j)) << " "; + // ss_dist << *(dists->data()->GetValues(1, i * k + j)) << " "; + ss_id << *((int64_t*)(ids) + i * k + j) << " "; + ss_dist << *((float*)(dists) + i * k + j) << " "; } ss_id << std::endl; ss_dist << std::endl; diff --git a/core/src/main.cpp b/core/src/main.cpp index 85f91469bb..b39ba87997 100644 --- a/core/src/main.cpp +++ b/core/src/main.cpp @@ -22,7 +22,7 @@ #include #include -#include "external/easyloggingpp/easylogging++.h" +#include "easyloggingpp/easylogging++.h" #include "metrics/Metrics.h" #include "server/Server.h" #include "src/version.h" @@ -51,7 +51,18 @@ print_banner() { std::cout << " /_/ /_/___/____/___/\\____/___/ " << std::endl; std::cout << std::endl; std::cout << "Welcome to Milvus!" << std::endl; - std::cout << "Milvus " << BUILD_TYPE << " version: v" << MILVUS_VERSION << ", built at " << BUILD_TIME << std::endl; + std::cout << "Milvus " << BUILD_TYPE << " version: v" << MILVUS_VERSION << ", built at " << BUILD_TIME << ", with " +#ifdef WITH_MKL + << "MKL" +#else + << "OpenBLAS" +#endif + << " library." << std::endl; +#ifdef MILVUS_CPU_VERSION + std::cout << "You are using Milvus CPU version" << std::endl; +#else + std::cout << "You are using Milvus GPU version" << std::endl; +#endif std::cout << std::endl; } @@ -95,7 +106,7 @@ main(int argc, char* argv[]) { char* log_filename_ptr = strdup(optarg); log_config_file = log_filename_ptr; free(log_filename_ptr); - std::cout << "Initial log config from: " << log_config_file << std::endl; + std::cout << "Initializing log config from: " << log_config_file << std::endl; break; } case 'p': { @@ -132,7 +143,7 @@ main(int argc, char* argv[]) { s = server.Start(); if (s.ok()) { - std::cout << "Milvus server start successfully." << std::endl; + std::cout << "Milvus server started successfully!" << std::endl; } else { goto FAIL; } diff --git a/core/src/metrics/Metrics.cpp b/core/src/metrics/Metrics.cpp index 51db5555b8..5fd3553cdc 100644 --- a/core/src/metrics/Metrics.cpp +++ b/core/src/metrics/Metrics.cpp @@ -16,8 +16,10 @@ // under the License. #include "metrics/Metrics.h" -#include "PrometheusMetrics.h" #include "server/Config.h" +#ifdef MILVUS_WITH_PROMETHEUS +#include "metrics/prometheus/PrometheusMetrics.h" +#endif #include @@ -37,11 +39,15 @@ Metrics::CreateMetricsCollector() { config.GetMetricConfigCollector(collector_type_str); +#ifdef MILVUS_WITH_PROMETHEUS if (collector_type_str == "prometheus") { return PrometheusMetrics::GetInstance(); } else { return MetricsBase::GetInstance(); } +#else + return MetricsBase::GetInstance(); +#endif } } // namespace server diff --git a/core/src/metrics/SystemInfo.cpp b/core/src/metrics/SystemInfo.cpp index 154f7b0797..1421cc1c95 100644 --- a/core/src/metrics/SystemInfo.cpp +++ b/core/src/metrics/SystemInfo.cpp @@ -19,7 +19,6 @@ #include "utils/Log.h" #include -#include #include #include #include @@ -29,6 +28,10 @@ #include #include +#ifdef MILVUS_GPU_VERSION +#include +#endif + namespace milvus { namespace server { @@ -60,6 +63,7 @@ SystemInfo::Init() { total_ram_ = GetPhysicalMemory(); fclose(file); +#ifdef MILVUS_GPU_VERSION // initialize GPU information nvmlReturn_t nvmlresult; nvmlresult = nvmlInit(); @@ -72,6 +76,7 @@ SystemInfo::Init() { SERVER_LOG_ERROR << "Unable to get devidce number"; return; } +#endif // initialize network traffic information std::pair in_and_out_octets = Octets(); @@ -213,6 +218,9 @@ SystemInfo::GPUMemoryTotal() { if (!initialized_) Init(); std::vector result; + +#ifdef MILVUS_GPU_VERSION + nvmlMemory_t nvmlMemory; for (int i = 0; i < num_device_; ++i) { nvmlDevice_t device; @@ -220,6 +228,8 @@ SystemInfo::GPUMemoryTotal() { nvmlDeviceGetMemoryInfo(device, &nvmlMemory); result.push_back(nvmlMemory.total); } +#endif + return result; } @@ -228,6 +238,9 @@ SystemInfo::GPUTemperature() { if (!initialized_) Init(); std::vector result; + +#ifdef MILVUS_GPU_VERSION + for (int i = 0; i < num_device_; i++) { nvmlDevice_t device; nvmlDeviceGetHandleByIndex(i, &device); @@ -235,6 +248,9 @@ SystemInfo::GPUTemperature() { nvmlDeviceGetTemperature(device, NVML_TEMPERATURE_GPU, &temp); result.push_back(temp); } + +#endif + return result; } @@ -283,6 +299,9 @@ SystemInfo::GPUMemoryUsed() { Init(); std::vector result; + +#ifdef MILVUS_GPU_VERSION + nvmlMemory_t nvmlMemory; for (int i = 0; i < num_device_; ++i) { nvmlDevice_t device; @@ -290,6 +309,9 @@ SystemInfo::GPUMemoryUsed() { nvmlDeviceGetMemoryInfo(device, &nvmlMemory); result.push_back(nvmlMemory.used); } + +#endif + return result; } diff --git a/core/src/metrics/PrometheusMetrics.cpp b/core/src/metrics/prometheus/PrometheusMetrics.cpp similarity index 98% rename from core/src/metrics/PrometheusMetrics.cpp rename to core/src/metrics/prometheus/PrometheusMetrics.cpp index 770b34dc47..19b2683280 100644 --- a/core/src/metrics/PrometheusMetrics.cpp +++ b/core/src/metrics/prometheus/PrometheusMetrics.cpp @@ -15,9 +15,9 @@ // specific language governing permissions and limitations // under the License. -#include "metrics/PrometheusMetrics.h" -#include "SystemInfo.h" +#include "metrics/prometheus/PrometheusMetrics.h" #include "cache/GpuCacheMgr.h" +#include "metrics/SystemInfo.h" #include "server/Config.h" #include "utils/Log.h" diff --git a/core/src/metrics/PrometheusMetrics.h b/core/src/metrics/prometheus/PrometheusMetrics.h similarity index 99% rename from core/src/metrics/PrometheusMetrics.h rename to core/src/metrics/prometheus/PrometheusMetrics.h index ef60f9a231..5a452ca02c 100644 --- a/core/src/metrics/PrometheusMetrics.h +++ b/core/src/metrics/prometheus/PrometheusMetrics.h @@ -24,7 +24,7 @@ #include #include -#include "MetricBase.h" +#include "metrics/MetricBase.h" #include "utils/Error.h" #define METRICS_NOW_TIME std::chrono::system_clock::now() diff --git a/core/src/scheduler/JobMgr.cpp b/core/src/scheduler/JobMgr.cpp index 4404d95763..76c07fe459 100644 --- a/core/src/scheduler/JobMgr.cpp +++ b/core/src/scheduler/JobMgr.cpp @@ -85,7 +85,7 @@ JobMgr::worker_function() { } for (auto& task : tasks) { - calculate_path(task); + calculate_path(res_mgr_, task); } // disk resources NEVER be empty. @@ -103,26 +103,21 @@ JobMgr::build_task(const JobPtr& job) { } void -JobMgr::calculate_path(const TaskPtr& task) { - if (task->type_ == TaskType::SearchTask) { - if (task->label()->Type() != TaskLabelType::SPECIFIED_RESOURCE) { - return; - } - - std::vector path; - auto spec_label = std::static_pointer_cast(task->label()); - auto src = res_mgr_->GetDiskResources()[0]; - auto dest = spec_label->resource(); - ShortestPath(src.lock(), dest.lock(), res_mgr_, path); - task->path() = Path(path, path.size() - 1); - } else if (task->type_ == TaskType::BuildIndexTask) { - auto spec_label = std::static_pointer_cast(task->label()); - auto src = res_mgr_->GetDiskResources()[0]; - auto dest = spec_label->resource(); - std::vector path; - ShortestPath(src.lock(), dest.lock(), res_mgr_, path); - task->path() = Path(path, path.size() - 1); +JobMgr::calculate_path(const ResourceMgrPtr& res_mgr, const TaskPtr& task) { + if (task->type_ != TaskType::SearchTask && task->type_ != TaskType::BuildIndexTask) { + return; } + + if (task->label()->Type() != TaskLabelType::SPECIFIED_RESOURCE) { + return; + } + + std::vector path; + auto spec_label = std::static_pointer_cast(task->label()); + auto src = res_mgr->GetDiskResources()[0]; + auto dest = spec_label->resource(); + ShortestPath(src.lock(), dest.lock(), res_mgr, path); + task->path() = Path(path, path.size() - 1); } } // namespace scheduler diff --git a/core/src/scheduler/JobMgr.h b/core/src/scheduler/JobMgr.h index fbd6c0ee45..af072614b5 100644 --- a/core/src/scheduler/JobMgr.h +++ b/core/src/scheduler/JobMgr.h @@ -59,8 +59,9 @@ class JobMgr : public interface::dumpable { static std::vector build_task(const JobPtr& job); - void - calculate_path(const TaskPtr& task); + public: + static void + calculate_path(const ResourceMgrPtr& res_mgr, const TaskPtr& task); private: bool running_ = false; diff --git a/core/src/scheduler/SchedInst.cpp b/core/src/scheduler/SchedInst.cpp index f0c00c2d2a..69d293f986 100644 --- a/core/src/scheduler/SchedInst.cpp +++ b/core/src/scheduler/SchedInst.cpp @@ -18,7 +18,6 @@ #include "scheduler/SchedInst.h" #include "ResourceFactory.h" #include "Utils.h" -#include "knowhere/index/vector_index/IndexGPUIVF.h" #include "server/Config.h" #include @@ -46,18 +45,6 @@ std::mutex BuildMgrInst::mutex_; void load_simple_config() { - server::Config& config = server::Config::GetInstance(); - std::string mode; - config.GetResourceConfigMode(mode); - std::vector pool; - config.GetResourceConfigSearchResources(pool); - - // get resources - auto gpu_ids = get_gpu_pool(); - - int32_t index_build_device_id; - config.GetResourceConfigIndexBuildDevice(index_build_device_id); - // create and connect ResMgrInst::GetInstance()->Add(ResourceFactory::Create("disk", "DISK", 0, true, false)); @@ -65,26 +52,46 @@ load_simple_config() { ResMgrInst::GetInstance()->Add(ResourceFactory::Create("cpu", "CPU", 0, true, true)); ResMgrInst::GetInstance()->Connect("disk", "cpu", io); + // get resources +#ifdef MILVUS_GPU_VERSION + server::Config& config = server::Config::GetInstance(); + std::vector gpu_ids; + config.GetGpuResourceConfigSearchResources(gpu_ids); + std::vector build_gpu_ids; + config.GetGpuResourceConfigBuildIndexResources(build_gpu_ids); auto pcie = Connection("pcie", 12000); - bool find_build_gpu_id = false; - for (auto& gpu_id : gpu_ids) { - ResMgrInst::GetInstance()->Add(ResourceFactory::Create(std::to_string(gpu_id), "GPU", gpu_id, true, true)); - ResMgrInst::GetInstance()->Connect("cpu", std::to_string(gpu_id), pcie); - if (index_build_device_id == gpu_id) { - find_build_gpu_id = true; + + std::vector not_find_build_ids; + for (auto& build_id : build_gpu_ids) { + bool find_gpu_id = false; + for (auto& gpu_id : gpu_ids) { + if (gpu_id == build_id) { + find_gpu_id = true; + break; + } + } + if (not find_gpu_id) { + not_find_build_ids.emplace_back(build_id); } } - if (not find_build_gpu_id && index_build_device_id != server::CPU_DEVICE_ID) { - ResMgrInst::GetInstance()->Add( - ResourceFactory::Create(std::to_string(index_build_device_id), "GPU", index_build_device_id, true, true)); - ResMgrInst::GetInstance()->Connect("cpu", std::to_string(index_build_device_id), pcie); + for (auto& gpu_id : gpu_ids) { + ResMgrInst::GetInstance()->Add(ResourceFactory::Create(std::to_string(gpu_id), "GPU", gpu_id, true, true)); + ResMgrInst::GetInstance()->Connect("cpu", std::to_string(gpu_id), pcie); } + + for (auto& not_find_id : not_find_build_ids) { + ResMgrInst::GetInstance()->Add( + ResourceFactory::Create(std::to_string(not_find_id), "GPU", not_find_id, true, true)); + ResMgrInst::GetInstance()->Connect("cpu", std::to_string(not_find_id), pcie); + } +#endif } void StartSchedulerService() { load_simple_config(); + OptimizerInst::GetInstance()->Init(); ResMgrInst::GetInstance()->Start(); SchedInst::GetInstance()->Start(); JobMgrInst::GetInstance()->Start(); diff --git a/core/src/scheduler/SchedInst.h b/core/src/scheduler/SchedInst.h index d51611af26..dc2d5ade35 100644 --- a/core/src/scheduler/SchedInst.h +++ b/core/src/scheduler/SchedInst.h @@ -21,10 +21,13 @@ #include "JobMgr.h" #include "ResourceMgr.h" #include "Scheduler.h" -#include "optimizer/HybridPass.h" -#include "optimizer/LargeSQ8HPass.h" -#include "optimizer/OnlyCPUPass.h" -#include "optimizer/OnlyGPUPass.h" +#include "Utils.h" +#include "optimizer/BuildIndexPass.h" +#include "optimizer/FaissFlatPass.h" +#include "optimizer/FaissIVFFlatPass.h" +#include "optimizer/FaissIVFSQ8HPass.h" +#include "optimizer/FaissIVFSQ8Pass.h" +#include "optimizer/FallbackPass.h" #include "optimizer/Optimizer.h" #include "server/Config.h" @@ -97,20 +100,15 @@ class OptimizerInst { if (instance == nullptr) { std::lock_guard lock(mutex_); if (instance == nullptr) { - server::Config& config = server::Config::GetInstance(); - std::vector search_resources; - bool has_cpu = false; - config.GetResourceConfigSearchResources(search_resources); - for (auto& resource : search_resources) { - if (resource == "cpu") { - has_cpu = true; - } - } std::vector pass_list; - pass_list.push_back(std::make_shared()); - pass_list.push_back(std::make_shared()); - pass_list.push_back(std::make_shared()); - pass_list.push_back(std::make_shared(has_cpu)); +#ifdef MILVUS_GPU_VERSION + pass_list.push_back(std::make_shared()); + pass_list.push_back(std::make_shared()); + pass_list.push_back(std::make_shared()); + pass_list.push_back(std::make_shared()); + pass_list.push_back(std::make_shared()); +#endif + pass_list.push_back(std::make_shared()); instance = std::make_shared(pass_list); } } diff --git a/core/src/scheduler/Scheduler.cpp b/core/src/scheduler/Scheduler.cpp index 8d2d4406f8..68d7457aa9 100644 --- a/core/src/scheduler/Scheduler.cpp +++ b/core/src/scheduler/Scheduler.cpp @@ -108,10 +108,6 @@ Scheduler::OnLoadCompleted(const EventPtr& event) { auto task_table_type = load_completed_event->task_table_item_->task->label()->Type(); switch (task_table_type) { - case TaskLabelType::DEFAULT: { - Action::DefaultLabelTaskScheduler(res_mgr_, resource, load_completed_event); - break; - } case TaskLabelType::SPECIFIED_RESOURCE: { Action::SpecifiedResourceLabelTaskScheduler(res_mgr_, resource, load_completed_event); break; diff --git a/core/src/scheduler/TaskCreator.cpp b/core/src/scheduler/TaskCreator.cpp index 9f3bc2ae9a..30b76cc5bb 100644 --- a/core/src/scheduler/TaskCreator.cpp +++ b/core/src/scheduler/TaskCreator.cpp @@ -18,7 +18,6 @@ #include "scheduler/TaskCreator.h" #include "SchedInst.h" #include "tasklabel/BroadcastLabel.h" -#include "tasklabel/DefaultLabel.h" #include "tasklabel/SpecResLabel.h" namespace milvus { @@ -47,8 +46,7 @@ std::vector TaskCreator::Create(const SearchJobPtr& job) { std::vector tasks; for (auto& index_file : job->index_files()) { - auto label = std::make_shared(); - auto task = std::make_shared(index_file.second, label); + auto task = std::make_shared(index_file.second, nullptr); task->job_ = job; tasks.emplace_back(task); } @@ -70,19 +68,8 @@ TaskCreator::Create(const DeleteJobPtr& job) { std::vector TaskCreator::Create(const BuildIndexJobPtr& job) { std::vector tasks; - server::Config& config = server::Config::GetInstance(); - int32_t build_index_id; - Status stat = config.GetResourceConfigIndexBuildDevice(build_index_id); - ResourcePtr res_ptr; - if (build_index_id == server::CPU_DEVICE_ID) { - res_ptr = ResMgrInst::GetInstance()->GetResource("cpu"); - } else { - res_ptr = ResMgrInst::GetInstance()->GetResource(ResourceType::GPU, build_index_id); - } - for (auto& to_index_file : job->to_index_files()) { - auto label = std::make_shared(std::weak_ptr(res_ptr)); - auto task = std::make_shared(to_index_file.second, label); + auto task = std::make_shared(to_index_file.second, nullptr); task->job_ = job; tasks.emplace_back(task); } diff --git a/core/src/scheduler/Utils.cpp b/core/src/scheduler/Utils.cpp index 998e545ba5..527b9adf03 100644 --- a/core/src/scheduler/Utils.cpp +++ b/core/src/scheduler/Utils.cpp @@ -16,10 +16,10 @@ // under the License. #include "scheduler/Utils.h" -#include "server/Config.h" -#include "utils/Log.h" +#ifdef MILVUS_GPU_VERSION #include +#endif #include #include #include @@ -38,46 +38,11 @@ get_current_timestamp() { uint64_t get_num_gpu() { int n_devices = 0; +#ifdef MILVUS_GPU_VERSION cudaGetDeviceCount(&n_devices); +#endif return n_devices; } -std::vector -get_gpu_pool() { - std::vector gpu_pool; - - server::Config& config = server::Config::GetInstance(); - std::vector pool; - Status s = config.GetResourceConfigSearchResources(pool); - if (!s.ok()) { - SERVER_LOG_ERROR << s.message(); - } - - std::set gpu_ids; - - for (auto& resource : pool) { - if (resource == "cpu") { - continue; - } else { - if (resource.length() < 4 || resource.substr(0, 3) != "gpu") { - // error - exit(-1); - } - auto gpu_id = std::stoi(resource.substr(3)); - if (gpu_id >= scheduler::get_num_gpu()) { - // error - exit(-1); - } - gpu_ids.insert(gpu_id); - } - } - - for (auto& gpu_id : gpu_ids) { - gpu_pool.push_back(gpu_id); - } - - return gpu_pool; -} - } // namespace scheduler } // namespace milvus diff --git a/core/src/scheduler/Utils.h b/core/src/scheduler/Utils.h index 24876eeb96..bf88cf0345 100644 --- a/core/src/scheduler/Utils.h +++ b/core/src/scheduler/Utils.h @@ -27,8 +27,5 @@ get_current_timestamp(); uint64_t get_num_gpu(); -std::vector -get_gpu_pool(); - } // namespace scheduler } // namespace milvus diff --git a/core/src/scheduler/action/Action.h b/core/src/scheduler/action/Action.h index f5f828cbf6..7391d287a8 100644 --- a/core/src/scheduler/action/Action.h +++ b/core/src/scheduler/action/Action.h @@ -36,10 +36,6 @@ class Action { static void PushTaskToResource(TaskTableItemPtr task_item, const ResourcePtr& dest); - static void - DefaultLabelTaskScheduler(const ResourceMgrPtr& res_mgr, ResourcePtr resource, - std::shared_ptr event); - static void SpecifiedResourceLabelTaskScheduler(const ResourceMgrPtr& res_mgr, ResourcePtr resource, std::shared_ptr event); diff --git a/core/src/scheduler/action/PushTaskToNeighbour.cpp b/core/src/scheduler/action/PushTaskToNeighbour.cpp index 9aed678937..562041cee1 100644 --- a/core/src/scheduler/action/PushTaskToNeighbour.cpp +++ b/core/src/scheduler/action/PushTaskToNeighbour.cpp @@ -101,78 +101,11 @@ Action::PushTaskToResource(TaskTableItemPtr task_item, const ResourcePtr& dest) dest->task_table().Put(task_item->task, task_item); } -void -Action::DefaultLabelTaskScheduler(const ResourceMgrPtr& res_mgr, ResourcePtr resource, - std::shared_ptr event) { - if (not resource->HasExecutor() && event->task_table_item_->Move()) { - auto task_item = event->task_table_item_; - auto task = event->task_table_item_->task; - auto search_task = std::static_pointer_cast(task); - bool moved = false; - - // to support test task, REFACTOR - if (resource->type() == ResourceType::CPU) { - if (auto index_engine = search_task->index_engine_) { - auto location = index_engine->GetLocation(); - - for (auto i = 0; i < res_mgr->GetNumGpuResource(); ++i) { - auto index = milvus::cache::GpuCacheMgr::GetInstance(i)->GetIndex(location); - if (index != nullptr) { - moved = true; - auto dest_resource = res_mgr->GetResource(ResourceType::GPU, i); - PushTaskToResource(event->task_table_item_, dest_resource); - break; - } - } - } - } - - if (not moved) { - PushTaskToNeighbourRandomly(task_item, resource); - } - } -} - void Action::SpecifiedResourceLabelTaskScheduler(const ResourceMgrPtr& res_mgr, ResourcePtr resource, std::shared_ptr event) { auto task_item = event->task_table_item_; auto task = event->task_table_item_->task; - // if (resource->type() == ResourceType::DISK) { - // // step 1: calculate shortest path per resource, from disk to compute resource - // auto compute_resources = res_mgr->GetComputeResources(); - // std::vector> paths; - // std::vector transport_costs; - // for (auto& res : compute_resources) { - // std::vector path; - // uint64_t transport_cost = ShortestPath(resource, res, res_mgr, path); - // transport_costs.push_back(transport_cost); - // paths.emplace_back(path); - // } - // if (task->job_.lock()->type() == JobType::BUILD) { - // // step2: Read device id in config - // // get build index gpu resource - // server::Config& config = server::Config::GetInstance(); - // int32_t build_index_gpu; - // Status stat = config.GetResourceConfigIndexBuildDevice(build_index_gpu); - // - // bool find_gpu_res = false; - // if (res_mgr->GetResource(ResourceType::GPU, build_index_gpu) != nullptr) { - // for (uint64_t i = 0; i < compute_resources.size(); ++i) { - // if (compute_resources[i]->name() == - // res_mgr->GetResource(ResourceType::GPU, build_index_gpu)->name()) { - // find_gpu_res = true; - // Path task_path(paths[i], paths[i].size() - 1); - // task->path() = task_path; - // break; - // } - // } - // } - // if (not find_gpu_res) { - // task->path() = Path(paths[0], paths[0].size() - 1); - // } - // } - // } if (resource->name() == task->path().Last()) { resource->WakeupExecutor(); diff --git a/core/src/scheduler/interface/interfaces.h b/core/src/scheduler/interface/interfaces.h index 9920e4f80a..311560adc6 100644 --- a/core/src/scheduler/interface/interfaces.h +++ b/core/src/scheduler/interface/interfaces.h @@ -33,8 +33,7 @@ namespace milvus { namespace interface { struct dumpable { - virtual ~dumpable() { - } + virtual ~dumpable() = default; virtual json Dump() const = 0; diff --git a/core/src/scheduler/job/BuildIndexJob.cpp b/core/src/scheduler/job/BuildIndexJob.cpp index 3247383db3..4c4c3b5054 100644 --- a/core/src/scheduler/job/BuildIndexJob.cpp +++ b/core/src/scheduler/job/BuildIndexJob.cpp @@ -50,10 +50,7 @@ void BuildIndexJob::BuildIndexDone(size_t to_index_id) { std::unique_lock lock(mutex_); to_index_files_.erase(to_index_id); - if (to_index_files_.empty()) { - cv_.notify_all(); - } - + cv_.notify_all(); SERVER_LOG_DEBUG << "BuildIndexJob " << id() << " finish index file: " << to_index_id; } diff --git a/core/src/scheduler/optimizer/OnlyCPUPass.cpp b/core/src/scheduler/optimizer/BuildIndexPass.cpp similarity index 65% rename from core/src/scheduler/optimizer/OnlyCPUPass.cpp rename to core/src/scheduler/optimizer/BuildIndexPass.cpp index 238a91a82c..d535b9675f 100644 --- a/core/src/scheduler/optimizer/OnlyCPUPass.cpp +++ b/core/src/scheduler/optimizer/BuildIndexPass.cpp @@ -15,32 +15,38 @@ // specific language governing permissions and limitations // under the License. -#include "scheduler/optimizer/OnlyCPUPass.h" +#include "scheduler/optimizer/BuildIndexPass.h" #include "scheduler/SchedInst.h" #include "scheduler/Utils.h" -#include "scheduler/task/SearchTask.h" #include "scheduler/tasklabel/SpecResLabel.h" namespace milvus { namespace scheduler { -bool -OnlyCPUPass::Run(const TaskPtr& task) { - if (task->Type() != TaskType::SearchTask) - return false; - auto search_task = std::static_pointer_cast(task); - if (search_task->file_->engine_type_ != (int)engine::EngineType::FAISS_IVFSQ8 && - search_task->file_->engine_type_ != (int)engine::EngineType::FAISS_IVFFLAT) { - return false; +void +BuildIndexPass::Init() { + server::Config& config = server::Config::GetInstance(); + std::vector build_resources; + Status s = config.GetGpuResourceConfigBuildIndexResources(build_resources); + if (!s.ok()) { + throw; } +} - auto gpu_id = get_gpu_pool(); - if (not gpu_id.empty()) +bool +BuildIndexPass::Run(const TaskPtr& task) { + if (task->Type() != TaskType::BuildIndexTask) return false; - ResourcePtr res_ptr = ResMgrInst::GetInstance()->GetResource("cpu"); + if (build_gpu_ids_.empty()) + return false; + + ResourcePtr res_ptr; + res_ptr = ResMgrInst::GetInstance()->GetResource(ResourceType::GPU, build_gpu_ids_[specified_gpu_id_]); auto label = std::make_shared(std::weak_ptr(res_ptr)); task->label() = label; + + specified_gpu_id_ = (specified_gpu_id_ + 1) % build_gpu_ids_.size(); return true; } diff --git a/core/src/scheduler/optimizer/OnlyGPUPass.h b/core/src/scheduler/optimizer/BuildIndexPass.h similarity index 86% rename from core/src/scheduler/optimizer/OnlyGPUPass.h rename to core/src/scheduler/optimizer/BuildIndexPass.h index 10d909d30e..4f7117fc4e 100644 --- a/core/src/scheduler/optimizer/OnlyGPUPass.h +++ b/core/src/scheduler/optimizer/BuildIndexPass.h @@ -32,20 +32,23 @@ namespace milvus { namespace scheduler { -class OnlyGPUPass : public Pass { +class BuildIndexPass : public Pass { public: - explicit OnlyGPUPass(bool has_cpu); + BuildIndexPass() = default; public: + void + Init() override; + bool Run(const TaskPtr& task) override; private: uint64_t specified_gpu_id_ = 0; - bool has_cpu_ = false; + std::vector build_gpu_ids_; }; -using OnlyGPUPassPtr = std::shared_ptr; +using BuildIndexPassPtr = std::shared_ptr; } // namespace scheduler } // namespace milvus diff --git a/core/src/scheduler/optimizer/LargeSQ8HPass.cpp b/core/src/scheduler/optimizer/FaissFlatPass.cpp similarity index 59% rename from core/src/scheduler/optimizer/LargeSQ8HPass.cpp rename to core/src/scheduler/optimizer/FaissFlatPass.cpp index b9784e3c0a..61ca1b9ec9 100644 --- a/core/src/scheduler/optimizer/LargeSQ8HPass.cpp +++ b/core/src/scheduler/optimizer/FaissFlatPass.cpp @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -#include "scheduler/optimizer/LargeSQ8HPass.h" +#include "scheduler/optimizer/FaissFlatPass.h" #include "cache/GpuCacheMgr.h" #include "scheduler/SchedInst.h" #include "scheduler/Utils.h" @@ -27,57 +27,41 @@ namespace milvus { namespace scheduler { -LargeSQ8HPass::LargeSQ8HPass() { +void +FaissFlatPass::Init() { server::Config& config = server::Config::GetInstance(); Status s = config.GetEngineConfigGpuSearchThreshold(threshold_); if (!s.ok()) { threshold_ = std::numeric_limits::max(); } + s = config.GetGpuResourceConfigSearchResources(gpus); + if (!s.ok()) { + throw; + } } bool -LargeSQ8HPass::Run(const TaskPtr& task) { +FaissFlatPass::Run(const TaskPtr& task) { if (task->Type() != TaskType::SearchTask) { return false; } auto search_task = std::static_pointer_cast(task); - if (search_task->file_->engine_type_ != (int)engine::EngineType::FAISS_IVFSQ8H) { + if (search_task->file_->engine_type_ != (int)engine::EngineType::FAISS_IDMAP) { return false; } auto search_job = std::static_pointer_cast(search_task->job_.lock()); - - // TODO: future, Index::IVFSQ8H, if nq < threshold set cpu, else set gpu - + ResourcePtr res_ptr; if (search_job->nq() < threshold_) { - return false; + res_ptr = ResMgrInst::GetInstance()->GetResource("cpu"); + } else { + auto best_device_id = count_ % gpus.size(); + count_++; + res_ptr = ResMgrInst::GetInstance()->GetResource(ResourceType::GPU, best_device_id); } - - std::vector gpus = scheduler::get_gpu_pool(); - // std::vector all_free_mem; - // for (auto& gpu : gpus) { - // auto cache = cache::GpuCacheMgr::GetInstance(gpu); - // auto free_mem = cache->CacheCapacity() - cache->CacheUsage(); - // all_free_mem.push_back(free_mem); - // } - // - // auto max_e = std::max_element(all_free_mem.begin(), all_free_mem.end()); - // auto best_index = std::distance(all_free_mem.begin(), max_e); - // auto best_device_id = gpus[best_index]; - auto best_device_id = count_ % gpus.size(); - count_++; - - ResourcePtr res_ptr = ResMgrInst::GetInstance()->GetResource(ResourceType::GPU, best_device_id); - if (not res_ptr) { - SERVER_LOG_ERROR << "GpuResource " << best_device_id << " invalid."; - // TODO: throw critical error and exit - return false; - } - - auto label = std::make_shared(std::weak_ptr(res_ptr)); + auto label = std::make_shared(res_ptr); task->label() = label; - return true; } diff --git a/core/src/scheduler/optimizer/LargeSQ8HPass.h b/core/src/scheduler/optimizer/FaissFlatPass.h similarity index 83% rename from core/src/scheduler/optimizer/LargeSQ8HPass.h rename to core/src/scheduler/optimizer/FaissFlatPass.h index 9d135d413a..f219bebdf3 100644 --- a/core/src/scheduler/optimizer/LargeSQ8HPass.h +++ b/core/src/scheduler/optimizer/FaissFlatPass.h @@ -33,20 +33,24 @@ namespace milvus { namespace scheduler { -class LargeSQ8HPass : public Pass { +class FaissFlatPass : public Pass { public: - LargeSQ8HPass(); + FaissFlatPass() = default; public: + void + Init() override; + bool Run(const TaskPtr& task) override; private: - int32_t threshold_ = std::numeric_limits::max(); + int64_t threshold_ = std::numeric_limits::max(); int64_t count_ = 0; + std::vector gpus; }; -using LargeSQ8HPassPtr = std::shared_ptr; +using FaissFlatPassPtr = std::shared_ptr; } // namespace scheduler } // namespace milvus diff --git a/core/src/scheduler/optimizer/FaissIVFFlatPass.cpp b/core/src/scheduler/optimizer/FaissIVFFlatPass.cpp new file mode 100644 index 0000000000..1f1efb374b --- /dev/null +++ b/core/src/scheduler/optimizer/FaissIVFFlatPass.cpp @@ -0,0 +1,69 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "scheduler/optimizer/FaissIVFFlatPass.h" +#include "cache/GpuCacheMgr.h" +#include "scheduler/SchedInst.h" +#include "scheduler/Utils.h" +#include "scheduler/task/SearchTask.h" +#include "scheduler/tasklabel/SpecResLabel.h" +#include "server/Config.h" +#include "utils/Log.h" + +namespace milvus { +namespace scheduler { + +void +FaissIVFFlatPass::Init() { + server::Config& config = server::Config::GetInstance(); + Status s = config.GetEngineConfigGpuSearchThreshold(threshold_); + if (!s.ok()) { + threshold_ = std::numeric_limits::max(); + } + s = config.GetGpuResourceConfigSearchResources(gpus); + if (!s.ok()) { + throw; + } +} + +bool +FaissIVFFlatPass::Run(const TaskPtr& task) { + if (task->Type() != TaskType::SearchTask) { + return false; + } + + auto search_task = std::static_pointer_cast(task); + if (search_task->file_->engine_type_ != (int)engine::EngineType::FAISS_IVFFLAT) { + return false; + } + + auto search_job = std::static_pointer_cast(search_task->job_.lock()); + ResourcePtr res_ptr; + if (search_job->nq() < threshold_) { + res_ptr = ResMgrInst::GetInstance()->GetResource("cpu"); + } else { + auto best_device_id = count_ % gpus.size(); + count_++; + res_ptr = ResMgrInst::GetInstance()->GetResource(ResourceType::GPU, best_device_id); + } + auto label = std::make_shared(res_ptr); + task->label() = label; + return true; +} + +} // namespace scheduler +} // namespace milvus diff --git a/core/src/scheduler/optimizer/HybridPass.h b/core/src/scheduler/optimizer/FaissIVFFlatPass.h similarity index 79% rename from core/src/scheduler/optimizer/HybridPass.h rename to core/src/scheduler/optimizer/FaissIVFFlatPass.h index 0d02a8bda9..2d15539014 100644 --- a/core/src/scheduler/optimizer/HybridPass.h +++ b/core/src/scheduler/optimizer/FaissIVFFlatPass.h @@ -18,6 +18,7 @@ #include #include +#include #include #include #include @@ -32,16 +33,24 @@ namespace milvus { namespace scheduler { -class HybridPass : public Pass { +class FaissIVFFlatPass : public Pass { public: - HybridPass() = default; + FaissIVFFlatPass() = default; public: + void + Init() override; + bool Run(const TaskPtr& task) override; + + private: + int64_t threshold_ = std::numeric_limits::max(); + int64_t count_ = 0; + std::vector gpus; }; -using HybridPassPtr = std::shared_ptr; +using FaissIVFFlatPassPtr = std::shared_ptr; } // namespace scheduler } // namespace milvus diff --git a/core/src/scheduler/optimizer/OnlyGPUPass.cpp b/core/src/scheduler/optimizer/FaissIVFSQ8HPass.cpp similarity index 55% rename from core/src/scheduler/optimizer/OnlyGPUPass.cpp rename to core/src/scheduler/optimizer/FaissIVFSQ8HPass.cpp index e5d3c71fd3..a99e861e03 100644 --- a/core/src/scheduler/optimizer/OnlyGPUPass.cpp +++ b/core/src/scheduler/optimizer/FaissIVFSQ8HPass.cpp @@ -15,38 +15,50 @@ // specific language governing permissions and limitations // under the License. -#include "scheduler/optimizer/OnlyGPUPass.h" +#include "scheduler/optimizer/FaissIVFSQ8HPass.h" +#include "cache/GpuCacheMgr.h" #include "scheduler/SchedInst.h" #include "scheduler/Utils.h" #include "scheduler/task/SearchTask.h" #include "scheduler/tasklabel/SpecResLabel.h" +#include "server/Config.h" +#include "utils/Log.h" namespace milvus { namespace scheduler { -OnlyGPUPass::OnlyGPUPass(bool has_cpu) : has_cpu_(has_cpu) { +void +FaissIVFSQ8HPass::Init() { + server::Config& config = server::Config::GetInstance(); + Status s = config.GetEngineConfigGpuSearchThreshold(threshold_); + if (!s.ok()) { + threshold_ = std::numeric_limits::max(); + } + s = config.GetGpuResourceConfigSearchResources(gpus); } bool -OnlyGPUPass::Run(const TaskPtr& task) { - if (task->Type() != TaskType::SearchTask || has_cpu_) - return false; - - auto search_task = std::static_pointer_cast(task); - if (search_task->file_->engine_type_ != (int)engine::EngineType::FAISS_IVFSQ8 && - search_task->file_->engine_type_ != (int)engine::EngineType::FAISS_IVFFLAT) { +FaissIVFSQ8HPass::Run(const TaskPtr& task) { + if (task->Type() != TaskType::SearchTask) { return false; } - auto gpu_id = get_gpu_pool(); - if (gpu_id.empty()) + auto search_task = std::static_pointer_cast(task); + if (search_task->file_->engine_type_ != (int)engine::EngineType::FAISS_IVFSQ8H) { return false; + } - ResourcePtr res_ptr = ResMgrInst::GetInstance()->GetResource(ResourceType::GPU, gpu_id[specified_gpu_id_]); - auto label = std::make_shared(std::weak_ptr(res_ptr)); + auto search_job = std::static_pointer_cast(search_task->job_.lock()); + ResourcePtr res_ptr; + if (search_job->nq() < threshold_) { + res_ptr = ResMgrInst::GetInstance()->GetResource("cpu"); + } else { + auto best_device_id = count_ % gpus.size(); + count_++; + res_ptr = ResMgrInst::GetInstance()->GetResource(ResourceType::GPU, best_device_id); + } + auto label = std::make_shared(res_ptr); task->label() = label; - - specified_gpu_id_ = (specified_gpu_id_ + 1) % gpu_id.size(); return true; } diff --git a/core/src/scheduler/optimizer/FaissIVFSQ8HPass.h b/core/src/scheduler/optimizer/FaissIVFSQ8HPass.h new file mode 100644 index 0000000000..0d2892809f --- /dev/null +++ b/core/src/scheduler/optimizer/FaissIVFSQ8HPass.h @@ -0,0 +1,56 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "Pass.h" + +namespace milvus { +namespace scheduler { + +class FaissIVFSQ8HPass : public Pass { + public: + FaissIVFSQ8HPass() = default; + + public: + void + Init() override; + + bool + Run(const TaskPtr& task) override; + + private: + int64_t threshold_ = std::numeric_limits::max(); + int64_t count_ = 0; + std::vector gpus; +}; + +using FaissIVFSQ8HPassPtr = std::shared_ptr; + +} // namespace scheduler +} // namespace milvus diff --git a/core/src/scheduler/optimizer/FaissIVFSQ8Pass.cpp b/core/src/scheduler/optimizer/FaissIVFSQ8Pass.cpp new file mode 100644 index 0000000000..30dd306b3b --- /dev/null +++ b/core/src/scheduler/optimizer/FaissIVFSQ8Pass.cpp @@ -0,0 +1,69 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "scheduler/optimizer/FaissIVFSQ8Pass.h" +#include "cache/GpuCacheMgr.h" +#include "scheduler/SchedInst.h" +#include "scheduler/Utils.h" +#include "scheduler/task/SearchTask.h" +#include "scheduler/tasklabel/SpecResLabel.h" +#include "server/Config.h" +#include "utils/Log.h" + +namespace milvus { +namespace scheduler { + +void +FaissIVFSQ8Pass::Init() { + server::Config& config = server::Config::GetInstance(); + Status s = config.GetEngineConfigGpuSearchThreshold(threshold_); + if (!s.ok()) { + threshold_ = std::numeric_limits::max(); + } + s = config.GetGpuResourceConfigSearchResources(gpus); + if (!s.ok()) { + throw; + } +} + +bool +FaissIVFSQ8Pass::Run(const TaskPtr& task) { + if (task->Type() != TaskType::SearchTask) { + return false; + } + + auto search_task = std::static_pointer_cast(task); + if (search_task->file_->engine_type_ != (int)engine::EngineType::FAISS_IVFSQ8) { + return false; + } + + auto search_job = std::static_pointer_cast(search_task->job_.lock()); + ResourcePtr res_ptr; + if (search_job->nq() < threshold_) { + res_ptr = ResMgrInst::GetInstance()->GetResource("cpu"); + } else { + auto best_device_id = count_ % gpus.size(); + count_++; + res_ptr = ResMgrInst::GetInstance()->GetResource(ResourceType::GPU, best_device_id); + } + auto label = std::make_shared(res_ptr); + task->label() = label; + return true; +} + +} // namespace scheduler +} // namespace milvus diff --git a/core/src/scheduler/optimizer/OnlyCPUPass.h b/core/src/scheduler/optimizer/FaissIVFSQ8Pass.h similarity index 79% rename from core/src/scheduler/optimizer/OnlyCPUPass.h rename to core/src/scheduler/optimizer/FaissIVFSQ8Pass.h index 76b42e3766..e92ea2fe4e 100644 --- a/core/src/scheduler/optimizer/OnlyCPUPass.h +++ b/core/src/scheduler/optimizer/FaissIVFSQ8Pass.h @@ -18,6 +18,7 @@ #include #include +#include #include #include #include @@ -32,16 +33,24 @@ namespace milvus { namespace scheduler { -class OnlyCPUPass : public Pass { +class FaissIVFSQ8Pass : public Pass { public: - OnlyCPUPass() = default; + FaissIVFSQ8Pass() = default; public: + void + Init() override; + bool Run(const TaskPtr& task) override; + + private: + int64_t threshold_ = std::numeric_limits::max(); + int64_t count_ = 0; + std::vector gpus; }; -using OnlyCPUPassPtr = std::shared_ptr; +using FaissIVFSQ8PassPtr = std::shared_ptr; } // namespace scheduler } // namespace milvus diff --git a/core/src/scheduler/optimizer/FallbackPass.cpp b/core/src/scheduler/optimizer/FallbackPass.cpp new file mode 100644 index 0000000000..2e275ede4b --- /dev/null +++ b/core/src/scheduler/optimizer/FallbackPass.cpp @@ -0,0 +1,43 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "scheduler/optimizer/FallbackPass.h" +#include "scheduler/SchedInst.h" +#include "scheduler/tasklabel/SpecResLabel.h" + +namespace milvus { +namespace scheduler { + +void +FallbackPass::Init() { +} + +bool +FallbackPass::Run(const TaskPtr& task) { + auto task_type = task->Type(); + if (task_type != TaskType::SearchTask && task_type != TaskType::BuildIndexTask) { + return false; + } + // NEVER be empty + auto cpu = ResMgrInst::GetInstance()->GetCpuResources()[0]; + auto label = std::make_shared(cpu); + task->label() = label; + return true; +} + +} // namespace scheduler +} // namespace milvus diff --git a/core/src/scheduler/tasklabel/DefaultLabel.h b/core/src/scheduler/optimizer/FallbackPass.h similarity index 82% rename from core/src/scheduler/tasklabel/DefaultLabel.h rename to core/src/scheduler/optimizer/FallbackPass.h index c215743575..728740d53a 100644 --- a/core/src/scheduler/tasklabel/DefaultLabel.h +++ b/core/src/scheduler/optimizer/FallbackPass.h @@ -14,23 +14,27 @@ // KIND, either express or implied. See the License for the // specific language governing permissions and limitations // under the License. - #pragma once -#include "TaskLabel.h" - +#include #include +#include "Pass.h" + namespace milvus { namespace scheduler { -class DefaultLabel : public TaskLabel { +class FallbackPass : public Pass { public: - DefaultLabel() : TaskLabel(TaskLabelType::DEFAULT) { - } -}; + FallbackPass() = default; -using DefaultLabelPtr = std::shared_ptr; + public: + void + Init() override; + + bool + Run(const TaskPtr& task) override; +}; } // namespace scheduler } // namespace milvus diff --git a/core/src/scheduler/optimizer/HybridPass.cpp b/core/src/scheduler/optimizer/HybridPass.cpp deleted file mode 100644 index d63fc2e819..0000000000 --- a/core/src/scheduler/optimizer/HybridPass.cpp +++ /dev/null @@ -1,43 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include "scheduler/optimizer/HybridPass.h" -#include "scheduler/SchedInst.h" -#include "scheduler/task/SearchTask.h" -#include "scheduler/tasklabel/SpecResLabel.h" - -namespace milvus { -namespace scheduler { - -bool -HybridPass::Run(const TaskPtr& task) { - // TODO: future, Index::IVFSQ8H, if nq < threshold set cpu, else set gpu - if (task->Type() != TaskType::SearchTask) - return false; - auto search_task = std::static_pointer_cast(task); - if (search_task->file_->engine_type_ == (int)engine::EngineType::FAISS_IVFSQ8H) { - // TODO: remove "cpu" hardcode - ResourcePtr res_ptr = ResMgrInst::GetInstance()->GetResource("cpu"); - auto label = std::make_shared(std::weak_ptr(res_ptr)); - task->label() = label; - return true; - } - return false; -} - -} // namespace scheduler -} // namespace milvus diff --git a/core/src/scheduler/optimizer/Optimizer.cpp b/core/src/scheduler/optimizer/Optimizer.cpp index 46f24ea712..c5fa311a27 100644 --- a/core/src/scheduler/optimizer/Optimizer.cpp +++ b/core/src/scheduler/optimizer/Optimizer.cpp @@ -20,12 +20,12 @@ namespace milvus { namespace scheduler { -// void -// Optimizer::Init() { -// for (auto& pass : pass_list_) { -// pass->Init(); -// } -// } +void +Optimizer::Init() { + for (auto& pass : pass_list_) { + pass->Init(); + } +} bool Optimizer::Run(const TaskPtr& task) { diff --git a/core/src/scheduler/optimizer/Optimizer.h b/core/src/scheduler/optimizer/Optimizer.h index bfabbf7de3..68b519e115 100644 --- a/core/src/scheduler/optimizer/Optimizer.h +++ b/core/src/scheduler/optimizer/Optimizer.h @@ -38,8 +38,8 @@ class Optimizer { explicit Optimizer(std::vector pass_list) : pass_list_(std::move(pass_list)) { } - // void - // Init(); + void + Init(); bool Run(const TaskPtr& task); diff --git a/core/src/scheduler/optimizer/Pass.h b/core/src/scheduler/optimizer/Pass.h index 016b05e457..36a36a1df5 100644 --- a/core/src/scheduler/optimizer/Pass.h +++ b/core/src/scheduler/optimizer/Pass.h @@ -34,9 +34,8 @@ namespace scheduler { class Pass { public: - // virtual void - // Init() { - // } + virtual void + Init() = 0; virtual bool Run(const TaskPtr& task) = 0; diff --git a/core/src/scheduler/tasklabel/TaskLabel.h b/core/src/scheduler/tasklabel/TaskLabel.h index d35ce409ff..33e6eb6e57 100644 --- a/core/src/scheduler/tasklabel/TaskLabel.h +++ b/core/src/scheduler/tasklabel/TaskLabel.h @@ -23,7 +23,6 @@ namespace milvus { namespace scheduler { enum class TaskLabelType { - DEFAULT, // means can be executed in any resource SPECIFIED_RESOURCE, // means must executing in special resource BROADCAST, // means all enable-executor resource must execute task }; diff --git a/core/src/sdk/examples/CMakeLists.txt b/core/src/sdk/examples/CMakeLists.txt index aa15190178..a394f1ce9b 100644 --- a/core/src/sdk/examples/CMakeLists.txt +++ b/core/src/sdk/examples/CMakeLists.txt @@ -17,5 +17,7 @@ # under the License. #------------------------------------------------------------------------------- +aux_source_directory(${MILVUS_SOURCE_DIR}/src/sdk/examples/utils util_files) -add_subdirectory(grpcsimple) +add_subdirectory(simple) +add_subdirectory(partition) diff --git a/core/src/sdk/examples/grpcsimple/src/ClientTest.cpp b/core/src/sdk/examples/grpcsimple/src/ClientTest.cpp deleted file mode 100644 index 09021e33b9..0000000000 --- a/core/src/sdk/examples/grpcsimple/src/ClientTest.cpp +++ /dev/null @@ -1,372 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include "sdk/examples/grpcsimple/src/ClientTest.h" -#include "MilvusApi.h" - -#include -#include -#include -#include -#include -#include -#include -#include - -//#define SET_VECTOR_IDS; - -namespace { -const std::string& -GetTableName(); - -const char* TABLE_NAME = GetTableName().c_str(); -constexpr int64_t TABLE_DIMENSION = 512; -constexpr int64_t TABLE_INDEX_FILE_SIZE = 1024; -constexpr int64_t BATCH_ROW_COUNT = 100000; -constexpr int64_t NQ = 5; -constexpr int64_t TOP_K = 10; -constexpr int64_t SEARCH_TARGET = 5000; // change this value, result is different -constexpr int64_t ADD_VECTOR_LOOP = 5; -constexpr int64_t SECONDS_EACH_HOUR = 3600; -constexpr milvus::IndexType INDEX_TYPE = milvus::IndexType::gpu_ivfsq8; -constexpr int32_t N_LIST = 15000; - -#define BLOCK_SPLITER std::cout << "===========================================" << std::endl; - -void -PrintTableSchema(const milvus::TableSchema& tb_schema) { - BLOCK_SPLITER - std::cout << "Table name: " << tb_schema.table_name << std::endl; - std::cout << "Table dimension: " << tb_schema.dimension << std::endl; - BLOCK_SPLITER -} - -void -PrintSearchResult(const std::vector>& search_record_array, - const milvus::TopKQueryResult& topk_query_result) { - BLOCK_SPLITER - size_t nq = topk_query_result.row_num; - size_t topk = topk_query_result.ids.size() / nq; - std::cout << "Returned result count: " << nq * topk << std::endl; - - int32_t index = 0; - for (size_t i = 0; i < nq; i++) { - auto search_id = search_record_array[index].first; - index++; - std::cout << "No." << index << " vector " << search_id << " top " << topk << " search result:" << std::endl; - for (size_t j = 0; j < topk; j++) { - size_t idx = i * nq + j; - std::cout << "\t" << topk_query_result.ids[idx] << "\t" << topk_query_result.distances[idx] << std::endl; - } - } - BLOCK_SPLITER -} - -std::string -CurrentTime() { - time_t tt; - time(&tt); - tt = tt + 8 * SECONDS_EACH_HOUR; - tm t; - gmtime_r(&tt, &t); - - std::string str = std::to_string(t.tm_year + 1900) + "_" + std::to_string(t.tm_mon + 1) + "_" + - std::to_string(t.tm_mday) + "_" + std::to_string(t.tm_hour) + "_" + std::to_string(t.tm_min) + - "_" + std::to_string(t.tm_sec); - - return str; -} - -std::string -CurrentTmDate(int64_t offset_day = 0) { - time_t tt; - time(&tt); - tt = tt + 8 * SECONDS_EACH_HOUR; - tt = tt + 24 * SECONDS_EACH_HOUR * offset_day; - tm t; - gmtime_r(&tt, &t); - - std::string str = - std::to_string(t.tm_year + 1900) + "-" + std::to_string(t.tm_mon + 1) + "-" + std::to_string(t.tm_mday); - - return str; -} - -const std::string& -GetTableName() { - static std::string s_id("tbl_" + CurrentTime()); - return s_id; -} - -milvus::TableSchema -BuildTableSchema() { - milvus::TableSchema tb_schema; - tb_schema.table_name = TABLE_NAME; - tb_schema.dimension = TABLE_DIMENSION; - tb_schema.index_file_size = TABLE_INDEX_FILE_SIZE; - tb_schema.metric_type = milvus::MetricType::L2; - - return tb_schema; -} - -void -BuildVectors(int64_t from, int64_t to, std::vector& vector_record_array) { - if (to <= from) { - return; - } - - vector_record_array.clear(); - for (int64_t k = from; k < to; k++) { - milvus::RowRecord record; - record.data.resize(TABLE_DIMENSION); - for (int64_t i = 0; i < TABLE_DIMENSION; i++) { - record.data[i] = (float)(k % (i + 1)); - } - - vector_record_array.emplace_back(record); - } -} - -void -Sleep(int seconds) { - std::cout << "Waiting " << seconds << " seconds ..." << std::endl; - sleep(seconds); -} - -class TimeRecorder { - public: - explicit TimeRecorder(const std::string& title) : title_(title) { - start_ = std::chrono::system_clock::now(); - } - - ~TimeRecorder() { - std::chrono::system_clock::time_point end = std::chrono::system_clock::now(); - int64_t span = (std::chrono::duration_cast(end - start_)).count(); - std::cout << title_ << " totally cost: " << span << " ms" << std::endl; - } - - private: - std::string title_; - std::chrono::system_clock::time_point start_; -}; - -void -CheckResult(const std::vector>& search_record_array, - const milvus::TopKQueryResult& topk_query_result) { - BLOCK_SPLITER - size_t nq = topk_query_result.row_num; - size_t result_k = topk_query_result.ids.size() / nq; - int64_t index = 0; - for (size_t i = 0; i < nq; i++) { - auto result_id = topk_query_result.ids[i * result_k]; - auto search_id = search_record_array[index++].first; - if (result_id != search_id) { - std::cout << "The top 1 result is wrong: " << result_id << " vs. " << search_id << std::endl; - } else { - std::cout << "Check result sucessfully" << std::endl; - } - } - BLOCK_SPLITER -} - -void -DoSearch(std::shared_ptr conn, - const std::vector>& search_record_array, const std::string& phase_name) { - std::vector query_range_array; - milvus::Range rg; - rg.start_value = CurrentTmDate(); - rg.end_value = CurrentTmDate(1); - query_range_array.emplace_back(rg); - - std::vector record_array; - for (auto& pair : search_record_array) { - record_array.push_back(pair.second); - } - - auto start = std::chrono::high_resolution_clock::now(); - milvus::TopKQueryResult topk_query_result; - { - TimeRecorder rc(phase_name); - milvus::Status stat = conn->Search(TABLE_NAME, record_array, query_range_array, TOP_K, 32, topk_query_result); - std::cout << "SearchVector function call status: " << stat.message() << std::endl; - } - auto finish = std::chrono::high_resolution_clock::now(); - std::cout << "SEARCHVECTOR COST: " - << std::chrono::duration_cast>(finish - start).count() << "s\n"; - - PrintSearchResult(search_record_array, topk_query_result); - CheckResult(search_record_array, topk_query_result); -} -} // namespace - -void -ClientTest::Test(const std::string& address, const std::string& port) { - std::shared_ptr conn = milvus::Connection::Create(); - - { // connect server - milvus::ConnectParam param = {address, port}; - milvus::Status stat = conn->Connect(param); - std::cout << "Connect function call status: " << stat.message() << std::endl; - } - - { // server version - std::string version = conn->ServerVersion(); - std::cout << "Server version: " << version << std::endl; - } - - { // sdk version - std::string version = conn->ClientVersion(); - std::cout << "SDK version: " << version << std::endl; - } - - { - std::vector tables; - milvus::Status stat = conn->ShowTables(tables); - std::cout << "ShowTables function call status: " << stat.message() << std::endl; - std::cout << "All tables: " << std::endl; - for (auto& table : tables) { - int64_t row_count = 0; - // conn->DropTable(table); - stat = conn->CountTable(table, row_count); - std::cout << "\t" << table << "(" << row_count << " rows)" << std::endl; - } - } - - { // create table - milvus::TableSchema tb_schema = BuildTableSchema(); - milvus::Status stat = conn->CreateTable(tb_schema); - std::cout << "CreateTable function call status: " << stat.message() << std::endl; - PrintTableSchema(tb_schema); - - bool has_table = conn->HasTable(tb_schema.table_name); - if (has_table) { - std::cout << "Table is created" << std::endl; - } - } - - { // describe table - milvus::TableSchema tb_schema; - milvus::Status stat = conn->DescribeTable(TABLE_NAME, tb_schema); - std::cout << "DescribeTable function call status: " << stat.message() << std::endl; - PrintTableSchema(tb_schema); - } - - std::vector> search_record_array; - { // insert vectors - for (int i = 0; i < ADD_VECTOR_LOOP; i++) { // add vectors - std::vector record_array; - int64_t begin_index = i * BATCH_ROW_COUNT; - BuildVectors(begin_index, begin_index + BATCH_ROW_COUNT, record_array); - -#ifdef SET_VECTOR_IDS - record_ids.resize(ADD_VECTOR_LOOP * BATCH_ROW_COUNT); - for (auto j = begin_index; j < begin_index + BATCH_ROW_COUNT; j++) { - record_ids[i * BATCH_ROW_COUNT + j] = i * BATCH_ROW_COUNT + j; - } -#endif - - std::vector record_ids; - // generate user defined ids - for (int k = 0; k < BATCH_ROW_COUNT; k++) { - record_ids.push_back(i * BATCH_ROW_COUNT + k); - } - - auto start = std::chrono::high_resolution_clock::now(); - - milvus::Status stat = conn->Insert(TABLE_NAME, record_array, record_ids); - auto finish = std::chrono::high_resolution_clock::now(); - std::cout << "InsertVector cost: " - << std::chrono::duration_cast>(finish - start).count() << "s\n"; - - std::cout << "InsertVector function call status: " << stat.message() << std::endl; - std::cout << "Returned id array count: " << record_ids.size() << std::endl; - - if (search_record_array.size() < NQ) { - search_record_array.push_back(std::make_pair(record_ids[SEARCH_TARGET], record_array[SEARCH_TARGET])); - } - } - } - - { // search vectors without index - Sleep(2); - - int64_t row_count = 0; - milvus::Status stat = conn->CountTable(TABLE_NAME, row_count); - std::cout << TABLE_NAME << "(" << row_count << " rows)" << std::endl; - // DoSearch(conn, search_record_array, "Search without index"); - } - - { // wait unit build index finish - std::cout << "Wait until create all index done" << std::endl; - milvus::IndexParam index; - index.table_name = TABLE_NAME; - index.index_type = INDEX_TYPE; - index.nlist = N_LIST; - milvus::Status stat = conn->CreateIndex(index); - std::cout << "CreateIndex function call status: " << stat.message() << std::endl; - - milvus::IndexParam index2; - stat = conn->DescribeIndex(TABLE_NAME, index2); - std::cout << "DescribeIndex function call status: " << stat.message() << std::endl; - } - - { // preload table - milvus::Status stat = conn->PreloadTable(TABLE_NAME); - std::cout << "PreloadTable function call status: " << stat.message() << std::endl; - } - - { // search vectors after build index finish - for (uint64_t i = 0; i < 5; ++i) { - DoSearch(conn, search_record_array, "Search after build index finish"); - } - // std::cout << conn->DumpTaskTables() << std::endl; - } - - { // delete index - milvus::Status stat = conn->DropIndex(TABLE_NAME); - std::cout << "DropIndex function call status: " << stat.message() << std::endl; - - int64_t row_count = 0; - stat = conn->CountTable(TABLE_NAME, row_count); - std::cout << TABLE_NAME << "(" << row_count << " rows)" << std::endl; - } - - { // delete by range - milvus::Range rg; - rg.start_value = CurrentTmDate(-3); - rg.end_value = CurrentTmDate(-2); - - milvus::Status stat = conn->DeleteByRange(rg, TABLE_NAME); - std::cout << "DeleteByRange function call status: " << stat.message() << std::endl; - } - - { - // delete table - // Status stat = conn->DropTable(TABLE_NAME); - // std::cout << "DeleteTable function call status: " << stat.message() << std::endl; - } - - { // server status - std::string status = conn->ServerStatus(); - std::cout << "Server status before disconnect: " << status << std::endl; - } - milvus::Connection::Destroy(conn); - { // server status - std::string status = conn->ServerStatus(); - std::cout << "Server status after disconnect: " << status << std::endl; - } -} diff --git a/core/src/sdk/examples/partition/CMakeLists.txt b/core/src/sdk/examples/partition/CMakeLists.txt new file mode 100644 index 0000000000..dc5ea46a62 --- /dev/null +++ b/core/src/sdk/examples/partition/CMakeLists.txt @@ -0,0 +1,34 @@ +#------------------------------------------------------------------------------- +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http:#www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +#------------------------------------------------------------------------------- + + +aux_source_directory(src src_files) + +add_executable(sdk_partition + main.cpp + ${src_files} + ${util_files} + ) + +target_link_libraries(sdk_partition + milvus_sdk + pthread + ) + +install(TARGETS sdk_partition DESTINATION bin) diff --git a/core/src/sdk/examples/partition/main.cpp b/core/src/sdk/examples/partition/main.cpp new file mode 100644 index 0000000000..f0de9b1fc4 --- /dev/null +++ b/core/src/sdk/examples/partition/main.cpp @@ -0,0 +1,79 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include +#include +#include +#include + +#include "sdk/examples/partition/src/ClientTest.h" + +void +print_help(const std::string& app_name); + +int +main(int argc, char* argv[]) { + printf("Client start...\n"); + + std::string app_name = basename(argv[0]); + static struct option long_options[] = {{"server", optional_argument, nullptr, 's'}, + {"port", optional_argument, nullptr, 'p'}, + {"help", no_argument, nullptr, 'h'}, + {nullptr, 0, nullptr, 0}}; + + int option_index = 0; + std::string address = "127.0.0.1", port = "19530"; + app_name = argv[0]; + + int value; + while ((value = getopt_long(argc, argv, "s:p:h", long_options, &option_index)) != -1) { + switch (value) { + case 's': { + char* address_ptr = strdup(optarg); + address = address_ptr; + free(address_ptr); + break; + } + case 'p': { + char* port_ptr = strdup(optarg); + port = port_ptr; + free(port_ptr); + break; + } + case 'h': + default: + print_help(app_name); + return EXIT_SUCCESS; + } + } + + ClientTest test; + test.Test(address, port); + + printf("Client stop...\n"); + return 0; +} + +void +print_help(const std::string& app_name) { + printf("\n Usage: %s [OPTIONS]\n\n", app_name.c_str()); + printf(" Options:\n"); + printf(" -s --server Server address, default 127.0.0.1\n"); + printf(" -p --port Server port, default 19530\n"); + printf(" -h --help Print help information\n"); + printf("\n"); +} diff --git a/core/src/sdk/examples/partition/src/ClientTest.cpp b/core/src/sdk/examples/partition/src/ClientTest.cpp new file mode 100644 index 0000000000..ecbf0a80e7 --- /dev/null +++ b/core/src/sdk/examples/partition/src/ClientTest.cpp @@ -0,0 +1,215 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "sdk/examples/partition/src/ClientTest.h" +#include "MilvusApi.h" +#include "sdk/examples/utils/Utils.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +const char* TABLE_NAME = milvus_sdk::Utils::GenTableName().c_str(); + +constexpr int64_t TABLE_DIMENSION = 512; +constexpr int64_t TABLE_INDEX_FILE_SIZE = 1024; +constexpr milvus::MetricType TABLE_METRIC_TYPE = milvus::MetricType::L2; +constexpr int64_t BATCH_ROW_COUNT = 10000; +constexpr int64_t NQ = 5; +constexpr int64_t TOP_K = 10; +constexpr int64_t NPROBE = 32; +constexpr int64_t SEARCH_TARGET = 5000; // change this value, result is different +constexpr milvus::IndexType INDEX_TYPE = milvus::IndexType::IVFSQ8; +constexpr int32_t N_LIST = 15000; +constexpr int32_t PARTITION_COUNT = 5; +constexpr int32_t TARGET_PARTITION = 3; + +milvus::TableSchema +BuildTableSchema() { + milvus::TableSchema tb_schema = {TABLE_NAME, TABLE_DIMENSION, TABLE_INDEX_FILE_SIZE, TABLE_METRIC_TYPE}; + return tb_schema; +} + +milvus::PartitionParam +BuildPartitionParam(int32_t index) { + std::string tag = std::to_string(index); + std::string partition_name = std::string(TABLE_NAME) + "_" + tag; + milvus::PartitionParam partition_param = {TABLE_NAME, partition_name, tag}; + return partition_param; +} + +milvus::IndexParam +BuildIndexParam() { + milvus::IndexParam index_param = {TABLE_NAME, INDEX_TYPE, N_LIST}; + return index_param; +} + +} // namespace + +void +ClientTest::Test(const std::string& address, const std::string& port) { + std::shared_ptr conn = milvus::Connection::Create(); + + milvus::Status stat; + { // connect server + milvus::ConnectParam param = {address, port}; + stat = conn->Connect(param); + std::cout << "Connect function call status: " << stat.message() << std::endl; + } + + { // create table + milvus::TableSchema tb_schema = BuildTableSchema(); + stat = conn->CreateTable(tb_schema); + std::cout << "CreateTable function call status: " << stat.message() << std::endl; + milvus_sdk::Utils::PrintTableSchema(tb_schema); + } + + { // create partition + for (int32_t i = 0; i < PARTITION_COUNT; i++) { + milvus::PartitionParam partition_param = BuildPartitionParam(i); + stat = conn->CreatePartition(partition_param); + std::cout << "CreatePartition function call status: " << stat.message() << std::endl; + milvus_sdk::Utils::PrintPartitionParam(partition_param); + } + + // show partitions + milvus::PartitionList partition_array; + stat = conn->ShowPartitions(TABLE_NAME, partition_array); + + std::cout << partition_array.size() << " partitions created:" << std::endl; + for (auto& partition : partition_array) { + std::cout << "\t" << partition.partition_name << "\t tag = " << partition.partition_tag << std::endl; + } + } + + { // insert vectors + milvus_sdk::TimeRecorder rc("All vectors"); + for (int i = 0; i < PARTITION_COUNT * 5; i++) { + std::vector record_array; + std::vector record_ids; + int64_t begin_index = i * BATCH_ROW_COUNT; + { // generate vectors + milvus_sdk::TimeRecorder rc("Build vectors No." + std::to_string(i)); + milvus_sdk::Utils::BuildVectors(begin_index, begin_index + BATCH_ROW_COUNT, record_array, record_ids, + TABLE_DIMENSION); + } + + std::string title = "Insert " + std::to_string(record_array.size()) + " vectors No." + std::to_string(i); + milvus_sdk::TimeRecorder rc(title); + stat = conn->Insert(TABLE_NAME, std::to_string(i % PARTITION_COUNT), record_array, record_ids); + } + } + + std::vector> search_record_array; + { // build search vectors + std::vector record_array; + std::vector record_ids; + int64_t index = TARGET_PARTITION * BATCH_ROW_COUNT + SEARCH_TARGET; + milvus_sdk::Utils::BuildVectors(index, index + 1, record_array, record_ids, TABLE_DIMENSION); + search_record_array.push_back(std::make_pair(record_ids[0], record_array[0])); + } + + milvus_sdk::Utils::Sleep(3); + + { // table row count + int64_t row_count = 0; + stat = conn->CountTable(TABLE_NAME, row_count); + std::cout << TABLE_NAME << "(" << row_count << " rows)" << std::endl; + } + + { // search vectors + std::cout << "Search in correct partition" << std::endl; + std::vector partiton_tags = {std::to_string(TARGET_PARTITION)}; + milvus::TopKQueryResult topk_query_result; + milvus_sdk::Utils::DoSearch(conn, TABLE_NAME, partiton_tags, TOP_K, NPROBE, search_record_array, + topk_query_result); + std::cout << "Search in wrong partition" << std::endl; + partiton_tags = {"0"}; + milvus_sdk::Utils::DoSearch(conn, TABLE_NAME, partiton_tags, TOP_K, NPROBE, search_record_array, + topk_query_result); + + std::cout << "Search by regex matched partition tag" << std::endl; + partiton_tags = {"\\d"}; + milvus_sdk::Utils::DoSearch(conn, TABLE_NAME, partiton_tags, TOP_K, NPROBE, search_record_array, + topk_query_result); + } + + { // wait unit build index finish + milvus_sdk::TimeRecorder rc("Create index"); + std::cout << "Wait until create all index done" << std::endl; + milvus::IndexParam index1 = BuildIndexParam(); + milvus_sdk::Utils::PrintIndexParam(index1); + stat = conn->CreateIndex(index1); + std::cout << "CreateIndex function call status: " << stat.message() << std::endl; + + milvus::IndexParam index2; + stat = conn->DescribeIndex(TABLE_NAME, index2); + std::cout << "DescribeIndex function call status: " << stat.message() << std::endl; + milvus_sdk::Utils::PrintIndexParam(index2); + } + + { // table row count + int64_t row_count = 0; + stat = conn->CountTable(TABLE_NAME, row_count); + std::cout << TABLE_NAME << "(" << row_count << " rows)" << std::endl; + } + + { // drop partition + milvus::PartitionParam param1 = {TABLE_NAME, "", std::to_string(TARGET_PARTITION)}; + milvus_sdk::Utils::PrintPartitionParam(param1); + stat = conn->DropPartition(param1); + std::cout << "DropPartition function call status: " << stat.message() << std::endl; + } + + { // table row count + int64_t row_count = 0; + stat = conn->CountTable(TABLE_NAME, row_count); + std::cout << TABLE_NAME << "(" << row_count << " rows)" << std::endl; + } + + { // search vectors + std::cout << "Search in whole table" << std::endl; + std::vector partiton_tags; + milvus::TopKQueryResult topk_query_result; + milvus_sdk::Utils::DoSearch(conn, TABLE_NAME, partiton_tags, TOP_K, NPROBE, search_record_array, + topk_query_result); + } + + { // drop index + stat = conn->DropIndex(TABLE_NAME); + std::cout << "DropIndex function call status: " << stat.message() << std::endl; + + int64_t row_count = 0; + stat = conn->CountTable(TABLE_NAME, row_count); + std::cout << TABLE_NAME << "(" << row_count << " rows)" << std::endl; + } + + { // drop table + stat = conn->DropTable(TABLE_NAME); + std::cout << "DropTable function call status: " << stat.message() << std::endl; + } + + milvus::Connection::Destroy(conn); +} diff --git a/core/src/sdk/examples/grpcsimple/src/ClientTest.h b/core/src/sdk/examples/partition/src/ClientTest.h similarity index 100% rename from core/src/sdk/examples/grpcsimple/src/ClientTest.h rename to core/src/sdk/examples/partition/src/ClientTest.h diff --git a/core/src/sdk/examples/grpcsimple/CMakeLists.txt b/core/src/sdk/examples/simple/CMakeLists.txt similarity index 98% rename from core/src/sdk/examples/grpcsimple/CMakeLists.txt rename to core/src/sdk/examples/simple/CMakeLists.txt index 77542ed2a7..82680e31be 100644 --- a/core/src/sdk/examples/grpcsimple/CMakeLists.txt +++ b/core/src/sdk/examples/simple/CMakeLists.txt @@ -17,12 +17,12 @@ # under the License. #------------------------------------------------------------------------------- - aux_source_directory(src src_files) add_executable(sdk_simple main.cpp ${src_files} + ${util_files} ) target_link_libraries(sdk_simple diff --git a/core/src/sdk/examples/grpcsimple/main.cpp b/core/src/sdk/examples/simple/main.cpp similarity index 98% rename from core/src/sdk/examples/grpcsimple/main.cpp rename to core/src/sdk/examples/simple/main.cpp index c31f491afb..c08741606c 100644 --- a/core/src/sdk/examples/grpcsimple/main.cpp +++ b/core/src/sdk/examples/simple/main.cpp @@ -20,7 +20,7 @@ #include #include -#include "src/ClientTest.h" +#include "sdk/examples/simple/src/ClientTest.h" void print_help(const std::string& app_name); diff --git a/core/src/sdk/examples/simple/src/ClientTest.cpp b/core/src/sdk/examples/simple/src/ClientTest.cpp new file mode 100644 index 0000000000..016c9eceac --- /dev/null +++ b/core/src/sdk/examples/simple/src/ClientTest.cpp @@ -0,0 +1,210 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "sdk/examples/simple/src/ClientTest.h" +#include "MilvusApi.h" +#include "sdk/examples/utils/TimeRecorder.h" +#include "sdk/examples/utils/Utils.h" + +#include +#include +#include +#include +#include +#include +#include + +namespace { + +const char* TABLE_NAME = milvus_sdk::Utils::GenTableName().c_str(); + +constexpr int64_t TABLE_DIMENSION = 512; +constexpr int64_t TABLE_INDEX_FILE_SIZE = 1024; +constexpr milvus::MetricType TABLE_METRIC_TYPE = milvus::MetricType::L2; +constexpr int64_t BATCH_ROW_COUNT = 100000; +constexpr int64_t NQ = 5; +constexpr int64_t TOP_K = 10; +constexpr int64_t NPROBE = 32; +constexpr int64_t SEARCH_TARGET = 5000; // change this value, result is different +constexpr int64_t ADD_VECTOR_LOOP = 5; +constexpr milvus::IndexType INDEX_TYPE = milvus::IndexType::IVFSQ8; +constexpr int32_t N_LIST = 15000; + +milvus::TableSchema +BuildTableSchema() { + milvus::TableSchema tb_schema = {TABLE_NAME, TABLE_DIMENSION, TABLE_INDEX_FILE_SIZE, TABLE_METRIC_TYPE}; + return tb_schema; +} + +milvus::IndexParam +BuildIndexParam() { + milvus::IndexParam index_param = {TABLE_NAME, INDEX_TYPE, N_LIST}; + return index_param; +} + +} // namespace + +void +ClientTest::Test(const std::string& address, const std::string& port) { + std::shared_ptr conn = milvus::Connection::Create(); + + milvus::Status stat; + { // connect server + milvus::ConnectParam param = {address, port}; + stat = conn->Connect(param); + std::cout << "Connect function call status: " << stat.message() << std::endl; + } + + { // server version + std::string version = conn->ServerVersion(); + std::cout << "Server version: " << version << std::endl; + } + + { // sdk version + std::string version = conn->ClientVersion(); + std::cout << "SDK version: " << version << std::endl; + } + + { // show tables + std::vector tables; + stat = conn->ShowTables(tables); + std::cout << "ShowTables function call status: " << stat.message() << std::endl; + std::cout << "All tables: " << std::endl; + for (auto& table : tables) { + int64_t row_count = 0; + // conn->DropTable(table); + stat = conn->CountTable(table, row_count); + std::cout << "\t" << table << "(" << row_count << " rows)" << std::endl; + } + } + + { // create table + milvus::TableSchema tb_schema = BuildTableSchema(); + stat = conn->CreateTable(tb_schema); + std::cout << "CreateTable function call status: " << stat.message() << std::endl; + milvus_sdk::Utils::PrintTableSchema(tb_schema); + + bool has_table = conn->HasTable(tb_schema.table_name); + if (has_table) { + std::cout << "Table is created" << std::endl; + } + } + + { // describe table + milvus::TableSchema tb_schema; + stat = conn->DescribeTable(TABLE_NAME, tb_schema); + std::cout << "DescribeTable function call status: " << stat.message() << std::endl; + milvus_sdk::Utils::PrintTableSchema(tb_schema); + } + + { // insert vectors + for (int i = 0; i < ADD_VECTOR_LOOP; i++) { + std::vector record_array; + std::vector record_ids; + int64_t begin_index = i * BATCH_ROW_COUNT; + { // generate vectors + milvus_sdk::TimeRecorder rc("Build vectors No." + std::to_string(i)); + milvus_sdk::Utils::BuildVectors(begin_index, begin_index + BATCH_ROW_COUNT, record_array, record_ids, + TABLE_DIMENSION); + } + + std::string title = "Insert " + std::to_string(record_array.size()) + " vectors No." + std::to_string(i); + milvus_sdk::TimeRecorder rc(title); + stat = conn->Insert(TABLE_NAME, "", record_array, record_ids); + std::cout << "InsertVector function call status: " << stat.message() << std::endl; + std::cout << "Returned id array count: " << record_ids.size() << std::endl; + } + } + + std::vector> search_record_array; + { // build search vectors + for (int64_t i = 0; i < NQ; i++) { + std::vector record_array; + std::vector record_ids; + int64_t index = i * BATCH_ROW_COUNT + SEARCH_TARGET; + milvus_sdk::Utils::BuildVectors(index, index + 1, record_array, record_ids, TABLE_DIMENSION); + search_record_array.push_back(std::make_pair(record_ids[0], record_array[0])); + } + } + + milvus_sdk::Utils::Sleep(3); + { // search vectors + std::vector partiton_tags; + milvus::TopKQueryResult topk_query_result; + milvus_sdk::Utils::DoSearch(conn, TABLE_NAME, partiton_tags, TOP_K, NPROBE, search_record_array, + topk_query_result); + } + + { // wait unit build index finish + milvus_sdk::TimeRecorder rc("Create index"); + std::cout << "Wait until create all index done" << std::endl; + milvus::IndexParam index1 = BuildIndexParam(); + milvus_sdk::Utils::PrintIndexParam(index1); + stat = conn->CreateIndex(index1); + std::cout << "CreateIndex function call status: " << stat.message() << std::endl; + + milvus::IndexParam index2; + stat = conn->DescribeIndex(TABLE_NAME, index2); + std::cout << "DescribeIndex function call status: " << stat.message() << std::endl; + milvus_sdk::Utils::PrintIndexParam(index2); + } + + { // preload table + stat = conn->PreloadTable(TABLE_NAME); + std::cout << "PreloadTable function call status: " << stat.message() << std::endl; + } + + { // search vectors + std::vector partiton_tags; + milvus::TopKQueryResult topk_query_result; + milvus_sdk::Utils::DoSearch(conn, TABLE_NAME, partiton_tags, TOP_K, NPROBE, search_record_array, + topk_query_result); + } + + { // drop index + stat = conn->DropIndex(TABLE_NAME); + std::cout << "DropIndex function call status: " << stat.message() << std::endl; + + int64_t row_count = 0; + stat = conn->CountTable(TABLE_NAME, row_count); + std::cout << TABLE_NAME << "(" << row_count << " rows)" << std::endl; + } + + { // delete by range + milvus::Range rg; + rg.start_value = milvus_sdk::Utils::CurrentTmDate(-3); + rg.end_value = milvus_sdk::Utils::CurrentTmDate(-2); + + stat = conn->DeleteByDate(TABLE_NAME, rg); + std::cout << "DeleteByDate function call status: " << stat.message() << std::endl; + } + + { // drop table + stat = conn->DropTable(TABLE_NAME); + std::cout << "DropTable function call status: " << stat.message() << std::endl; + } + + { // server status + std::string status = conn->ServerStatus(); + std::cout << "Server status before disconnect: " << status << std::endl; + } + milvus::Connection::Destroy(conn); + { // server status + std::string status = conn->ServerStatus(); + std::cout << "Server status after disconnect: " << status << std::endl; + } +} diff --git a/core/src/version.h.macro b/core/src/sdk/examples/simple/src/ClientTest.h similarity index 86% rename from core/src/version.h.macro rename to core/src/sdk/examples/simple/src/ClientTest.h index 454d8a990a..b028b63f44 100644 --- a/core/src/version.h.macro +++ b/core/src/sdk/examples/simple/src/ClientTest.h @@ -17,6 +17,10 @@ #pragma once -#define MILVUS_VERSION "@MILVUS_VERSION@" -#define BUILD_TYPE "@BUILD_TYPE@" -#define BUILD_TIME @BUILD_TIME@ \ No newline at end of file +#include + +class ClientTest { + public: + void + Test(const std::string& address, const std::string& port); +}; diff --git a/core/src/sdk/examples/utils/TimeRecorder.cpp b/core/src/sdk/examples/utils/TimeRecorder.cpp new file mode 100644 index 0000000000..cdf9eda5ec --- /dev/null +++ b/core/src/sdk/examples/utils/TimeRecorder.cpp @@ -0,0 +1,35 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "sdk/examples/utils/TimeRecorder.h" + +#include + +namespace milvus_sdk { + +TimeRecorder::TimeRecorder(const std::string& title) : title_(title) { + start_ = std::chrono::system_clock::now(); + std::cout << title_ << " begin..." << std::endl; +} + +TimeRecorder::~TimeRecorder() { + std::chrono::system_clock::time_point end = std::chrono::system_clock::now(); + int64_t span = (std::chrono::duration_cast(end - start_)).count(); + std::cout << title_ << " totally cost: " << span << " ms" << std::endl; +} + +} // namespace milvus_sdk diff --git a/core/src/sdk/examples/utils/TimeRecorder.h b/core/src/sdk/examples/utils/TimeRecorder.h new file mode 100644 index 0000000000..edfb9d2679 --- /dev/null +++ b/core/src/sdk/examples/utils/TimeRecorder.h @@ -0,0 +1,36 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include + +namespace milvus_sdk { + +class TimeRecorder { + public: + explicit TimeRecorder(const std::string& title); + + ~TimeRecorder(); + + private: + std::string title_; + std::chrono::system_clock::time_point start_; +}; + +} // namespace milvus_sdk diff --git a/core/src/sdk/examples/utils/Utils.cpp b/core/src/sdk/examples/utils/Utils.cpp new file mode 100644 index 0000000000..fa373cd498 --- /dev/null +++ b/core/src/sdk/examples/utils/Utils.cpp @@ -0,0 +1,228 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "sdk/examples/utils/Utils.h" +#include "sdk/examples/utils/TimeRecorder.h" + +#include +#include +#include +#include +#include +#include +#include + +namespace milvus_sdk { + +constexpr int64_t SECONDS_EACH_HOUR = 3600; + +#define BLOCK_SPLITER std::cout << "===========================================" << std::endl; + +std::string +Utils::CurrentTime() { + time_t tt; + time(&tt); + tt = tt + 8 * SECONDS_EACH_HOUR; + tm t; + gmtime_r(&tt, &t); + + std::string str = std::to_string(t.tm_year + 1900) + "_" + std::to_string(t.tm_mon + 1) + "_" + + std::to_string(t.tm_mday) + "_" + std::to_string(t.tm_hour) + "_" + std::to_string(t.tm_min) + + "_" + std::to_string(t.tm_sec); + + return str; +} + +std::string +Utils::CurrentTmDate(int64_t offset_day) { + time_t tt; + time(&tt); + tt = tt + 8 * SECONDS_EACH_HOUR; + tt = tt + 24 * SECONDS_EACH_HOUR * offset_day; + tm t; + gmtime_r(&tt, &t); + + std::string str = + std::to_string(t.tm_year + 1900) + "-" + std::to_string(t.tm_mon + 1) + "-" + std::to_string(t.tm_mday); + + return str; +} + +void +Utils::Sleep(int seconds) { + std::cout << "Waiting " << seconds << " seconds ..." << std::endl; + sleep(seconds); +} + +const std::string& +Utils::GenTableName() { + static std::string s_id("tbl_" + CurrentTime()); + return s_id; +} + +std::string +Utils::MetricTypeName(const milvus::MetricType& metric_type) { + switch (metric_type) { + case milvus::MetricType::L2: + return "L2 distance"; + case milvus::MetricType::IP: + return "Inner product"; + default: + return "Unknown metric type"; + } +} + +std::string +Utils::IndexTypeName(const milvus::IndexType& index_type) { + switch (index_type) { + case milvus::IndexType::FLAT: + return "FLAT"; + case milvus::IndexType::IVFFLAT: + return "IVFFLAT"; + case milvus::IndexType::IVFSQ8: + return "IVFSQ8"; + case milvus::IndexType::NSG: + return "NSG"; + case milvus::IndexType::IVFSQ8H: + return "IVFSQ8H"; + default: + return "Unknown index type"; + } +} + +void +Utils::PrintTableSchema(const milvus::TableSchema& tb_schema) { + BLOCK_SPLITER + std::cout << "Table name: " << tb_schema.table_name << std::endl; + std::cout << "Table dimension: " << tb_schema.dimension << std::endl; + std::cout << "Table index file size: " << tb_schema.index_file_size << std::endl; + std::cout << "Table metric type: " << MetricTypeName(tb_schema.metric_type) << std::endl; + BLOCK_SPLITER +} + +void +Utils::PrintPartitionParam(const milvus::PartitionParam& partition_param) { + BLOCK_SPLITER + std::cout << "Table name: " << partition_param.table_name << std::endl; + std::cout << "Partition name: " << partition_param.partition_name << std::endl; + std::cout << "Partition tag: " << partition_param.partition_tag << std::endl; + BLOCK_SPLITER +} + +void +Utils::PrintIndexParam(const milvus::IndexParam& index_param) { + BLOCK_SPLITER + std::cout << "Index table name: " << index_param.table_name << std::endl; + std::cout << "Index type: " << IndexTypeName(index_param.index_type) << std::endl; + std::cout << "Index nlist: " << index_param.nlist << std::endl; + BLOCK_SPLITER +} + +void +Utils::BuildVectors(int64_t from, int64_t to, std::vector& vector_record_array, + std::vector& record_ids, int64_t dimension) { + if (to <= from) { + return; + } + + vector_record_array.clear(); + record_ids.clear(); + for (int64_t k = from; k < to; k++) { + milvus::RowRecord record; + record.data.resize(dimension); + for (int64_t i = 0; i < dimension; i++) { + record.data[i] = (float)(k % (i + 1)); + } + + vector_record_array.emplace_back(record); + record_ids.push_back(k); + } +} + +void +Utils::PrintSearchResult(const std::vector>& search_record_array, + const milvus::TopKQueryResult& topk_query_result) { + BLOCK_SPLITER + std::cout << "Returned result count: " << topk_query_result.size() << std::endl; + + if (topk_query_result.size() != search_record_array.size()) { + std::cout << "ERROR: Returned result count dones equal nq" << std::endl; + return; + } + + for (size_t i = 0; i < topk_query_result.size(); i++) { + const milvus::QueryResult& one_result = topk_query_result[i]; + size_t topk = one_result.ids.size(); + auto search_id = search_record_array[i].first; + std::cout << "No." << i << " vector " << search_id << " top " << topk << " search result:" << std::endl; + for (size_t j = 0; j < topk; j++) { + std::cout << "\t" << one_result.ids[j] << "\t" << one_result.distances[j] << std::endl; + } + } + BLOCK_SPLITER +} + +void +Utils::CheckSearchResult(const std::vector>& search_record_array, + const milvus::TopKQueryResult& topk_query_result) { + BLOCK_SPLITER + size_t nq = topk_query_result.size(); + for (size_t i = 0; i < nq; i++) { + const milvus::QueryResult& one_result = topk_query_result[i]; + auto search_id = search_record_array[i].first; + int64_t result_id = one_result.ids[0]; + if (result_id != search_id) { + std::cout << "The top 1 result is wrong: " << result_id << " vs. " << search_id << std::endl; + } else { + std::cout << "Check result sucessfully" << std::endl; + } + } + BLOCK_SPLITER +} + +void +Utils::DoSearch(std::shared_ptr conn, const std::string& table_name, + const std::vector& partiton_tags, int64_t top_k, int64_t nprobe, + const std::vector>& search_record_array, + milvus::TopKQueryResult& topk_query_result) { + topk_query_result.clear(); + + std::vector query_range_array; + milvus::Range rg; + rg.start_value = CurrentTmDate(); + rg.end_value = CurrentTmDate(1); + query_range_array.emplace_back(rg); + + std::vector record_array; + for (auto& pair : search_record_array) { + record_array.push_back(pair.second); + } + + { + BLOCK_SPLITER + milvus_sdk::TimeRecorder rc("search"); + milvus::Status stat = + conn->Search(table_name, partiton_tags, record_array, query_range_array, top_k, nprobe, topk_query_result); + std::cout << "SearchVector function call status: " << stat.message() << std::endl; + BLOCK_SPLITER + } + + PrintSearchResult(search_record_array, topk_query_result); + CheckSearchResult(search_record_array, topk_query_result); +} + +} // namespace milvus_sdk diff --git a/core/src/sdk/examples/utils/Utils.h b/core/src/sdk/examples/utils/Utils.h new file mode 100644 index 0000000000..8066a24ff4 --- /dev/null +++ b/core/src/sdk/examples/utils/Utils.h @@ -0,0 +1,77 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include "MilvusApi.h" + +#include +#include +#include +#include + +namespace milvus_sdk { + +class Utils { + public: + static std::string + CurrentTime(); + + static std::string + CurrentTmDate(int64_t offset_day = 0); + + static const std::string& + GenTableName(); + + static void + Sleep(int seconds); + + static std::string + MetricTypeName(const milvus::MetricType& metric_type); + + static std::string + IndexTypeName(const milvus::IndexType& index_type); + + static void + PrintTableSchema(const milvus::TableSchema& tb_schema); + + static void + PrintPartitionParam(const milvus::PartitionParam& partition_param); + + static void + PrintIndexParam(const milvus::IndexParam& index_param); + + static void + BuildVectors(int64_t from, int64_t to, std::vector& vector_record_array, + std::vector& record_ids, int64_t dimension); + + static void + PrintSearchResult(const std::vector>& search_record_array, + const milvus::TopKQueryResult& topk_query_result); + + static void + CheckSearchResult(const std::vector>& search_record_array, + const milvus::TopKQueryResult& topk_query_result); + + static void + DoSearch(std::shared_ptr conn, const std::string& table_name, + const std::vector& partiton_tags, int64_t top_k, int64_t nprobe, + const std::vector>& search_record_array, + milvus::TopKQueryResult& topk_query_result); +}; + +} // namespace milvus_sdk diff --git a/core/src/sdk/grpc/ClientProxy.cpp b/core/src/sdk/grpc/ClientProxy.cpp index 65b5f52a94..fd19281343 100644 --- a/core/src/sdk/grpc/ClientProxy.cpp +++ b/core/src/sdk/grpc/ClientProxy.cpp @@ -32,6 +32,13 @@ UriCheck(const std::string& uri) { return (index != std::string::npos); } +void +CopyRowRecord(::milvus::grpc::RowRecord* target, const RowRecord& src) { + auto vector_data = target->mutable_vector_data(); + vector_data->Resize(static_cast(src.data.size()), 0.0); + memcpy(vector_data->mutable_data(), src.data.data(), src.data.size() * sizeof(float)); +} + Status ClientProxy::Connect(const ConnectParam& param) { std::string uri = param.ip_address + ":" + param.port; @@ -138,8 +145,8 @@ ClientProxy::CreateIndex(const IndexParam& index_param) { } Status -ClientProxy::Insert(const std::string& table_name, const std::vector& record_array, - std::vector& id_array) { +ClientProxy::Insert(const std::string& table_name, const std::string& partition_tag, + const std::vector& record_array, std::vector& id_array) { Status status = Status::OK(); try { //////////////////////////////////////////////////////////////////////////// @@ -185,17 +192,20 @@ ClientProxy::Insert(const std::string& table_name, const std::vector& #else ::milvus::grpc::InsertParam insert_param; insert_param.set_table_name(table_name); + insert_param.set_partition_tag(partition_tag); for (auto& record : record_array) { ::milvus::grpc::RowRecord* grpc_record = insert_param.add_row_record_array(); - grpc_record->add_vector_data(record.data.begin(), record.data.end()); + CopyRowRecord(grpc_record, record); } // Single thread ::milvus::grpc::VectorIds vector_ids; if (!id_array.empty()) { /* set user's ids */ - insert_param.add_row_id_array(id_array.begin(), id_array.end()); + auto row_ids = insert_param.mutable_row_id_array(); + row_ids->Resize(static_cast(id_array.size()), -1); + memcpy(row_ids->mutable_data(), id_array.data(), id_array.size() * sizeof(int64_t)); client_ptr_->Insert(vector_ids, insert_param, status); } else { client_ptr_->Insert(vector_ids, insert_param, status); @@ -211,18 +221,21 @@ ClientProxy::Insert(const std::string& table_name, const std::vector& } Status -ClientProxy::Search(const std::string& table_name, const std::vector& query_record_array, - const std::vector& query_range_array, int64_t topk, int64_t nprobe, - TopKQueryResult& topk_query_result) { +ClientProxy::Search(const std::string& table_name, const std::vector& partiton_tags, + const std::vector& query_record_array, const std::vector& query_range_array, + int64_t topk, int64_t nprobe, TopKQueryResult& topk_query_result) { try { // step 1: convert vectors data ::milvus::grpc::SearchParam search_param; search_param.set_table_name(table_name); search_param.set_topk(topk); search_param.set_nprobe(nprobe); + for (auto& tag : partiton_tags) { + search_param.add_partition_tag_array(tag); + } for (auto& record : query_record_array) { ::milvus::grpc::RowRecord* row_record = search_param.add_query_record_array(); - row_record->add_vector_data(record.data.begin(), record.data.end()); + CopyRowRecord(row_record, record); } // step 2: convert range array @@ -237,12 +250,17 @@ ClientProxy::Search(const std::string& table_name, const std::vector& Status status = client_ptr_->Search(result, search_param); // step 4: convert result array - topk_query_result.row_num = result.row_num(); - topk_query_result.ids.resize(result.ids().size()); - memcpy(topk_query_result.ids.data(), result.ids().data(), result.ids().size() * sizeof(int64_t)); - topk_query_result.distances.resize(result.distances().size()); - memcpy(topk_query_result.distances.data(), result.distances().data(), - result.distances().size() * sizeof(float)); + topk_query_result.reserve(result.row_num()); + int64_t nq = result.row_num(); + int64_t topk = result.ids().size() / nq; + for (int64_t i = 0; i < result.row_num(); i++) { + milvus::QueryResult one_result; + one_result.ids.resize(topk); + one_result.distances.resize(topk); + memcpy(one_result.ids.data(), result.ids().data() + topk * i, topk * sizeof(int64_t)); + memcpy(one_result.distances.data(), result.distances().data() + topk * i, topk * sizeof(float)); + topk_query_result.emplace_back(one_result); + } return status; } catch (std::exception& ex) { @@ -339,13 +357,13 @@ ClientProxy::DumpTaskTables() const { } Status -ClientProxy::DeleteByRange(milvus::Range& range, const std::string& table_name) { +ClientProxy::DeleteByDate(const std::string& table_name, const milvus::Range& range) { try { - ::milvus::grpc::DeleteByRangeParam delete_by_range_param; + ::milvus::grpc::DeleteByDateParam delete_by_range_param; delete_by_range_param.set_table_name(table_name); delete_by_range_param.mutable_range()->set_start_value(range.start_value); delete_by_range_param.mutable_range()->set_end_value(range.end_value); - return client_ptr_->DeleteByRange(delete_by_range_param); + return client_ptr_->DeleteByDate(delete_by_range_param); } catch (std::exception& ex) { return Status(StatusCode::UnknownError, "fail to delete by range: " + std::string(ex.what())); } @@ -391,4 +409,51 @@ ClientProxy::DropIndex(const std::string& table_name) const { } } +Status +ClientProxy::CreatePartition(const PartitionParam& partition_param) { + try { + ::milvus::grpc::PartitionParam grpc_partition_param; + grpc_partition_param.set_table_name(partition_param.table_name); + grpc_partition_param.set_partition_name(partition_param.partition_name); + grpc_partition_param.set_tag(partition_param.partition_tag); + Status status = client_ptr_->CreatePartition(grpc_partition_param); + return status; + } catch (std::exception& ex) { + return Status(StatusCode::UnknownError, "fail to create partition: " + std::string(ex.what())); + } +} + +Status +ClientProxy::ShowPartitions(const std::string& table_name, PartitionList& partition_array) const { + try { + ::milvus::grpc::TableName grpc_table_name; + grpc_table_name.set_table_name(table_name); + ::milvus::grpc::PartitionList grpc_partition_list; + Status status = client_ptr_->ShowPartitions(grpc_table_name, grpc_partition_list); + partition_array.resize(grpc_partition_list.partition_array_size()); + for (uint64_t i = 0; i < grpc_partition_list.partition_array_size(); ++i) { + partition_array[i].table_name = grpc_partition_list.partition_array(i).table_name(); + partition_array[i].partition_name = grpc_partition_list.partition_array(i).partition_name(); + partition_array[i].partition_tag = grpc_partition_list.partition_array(i).tag(); + } + return status; + } catch (std::exception& ex) { + return Status(StatusCode::UnknownError, "fail to show partitions: " + std::string(ex.what())); + } +} + +Status +ClientProxy::DropPartition(const PartitionParam& partition_param) { + try { + ::milvus::grpc::PartitionParam grpc_partition_param; + grpc_partition_param.set_table_name(partition_param.table_name); + grpc_partition_param.set_partition_name(partition_param.partition_name); + grpc_partition_param.set_tag(partition_param.partition_tag); + Status status = client_ptr_->DropPartition(grpc_partition_param); + return status; + } catch (std::exception& ex) { + return Status(StatusCode::UnknownError, "fail to drop partition: " + std::string(ex.what())); + } +} + } // namespace milvus diff --git a/core/src/sdk/grpc/ClientProxy.h b/core/src/sdk/grpc/ClientProxy.h index 8bcece4f20..e332266acf 100644 --- a/core/src/sdk/grpc/ClientProxy.h +++ b/core/src/sdk/grpc/ClientProxy.h @@ -54,13 +54,13 @@ class ClientProxy : public Connection { CreateIndex(const IndexParam& index_param) override; Status - Insert(const std::string& table_name, const std::vector& record_array, + Insert(const std::string& table_name, const std::string& partition_tag, const std::vector& record_array, std::vector& id_array) override; Status - Search(const std::string& table_name, const std::vector& query_record_array, - const std::vector& query_range_array, int64_t topk, int64_t nprobe, - TopKQueryResult& topk_query_result) override; + Search(const std::string& table_name, const std::vector& partiton_tags, + const std::vector& query_record_array, const std::vector& query_range_array, int64_t topk, + int64_t nprobe, TopKQueryResult& topk_query_result) override; Status DescribeTable(const std::string& table_name, TableSchema& table_schema) override; @@ -84,7 +84,7 @@ class ClientProxy : public Connection { DumpTaskTables() const override; Status - DeleteByRange(Range& range, const std::string& table_name) override; + DeleteByDate(const std::string& table_name, const Range& range) override; Status PreloadTable(const std::string& table_name) const override; @@ -95,6 +95,15 @@ class ClientProxy : public Connection { Status DropIndex(const std::string& table_name) const override; + Status + CreatePartition(const PartitionParam& partition_param) override; + + Status + ShowPartitions(const std::string& table_name, PartitionList& partition_array) const override; + + Status + DropPartition(const PartitionParam& partition_param) override; + private: std::shared_ptr<::grpc::Channel> channel_; diff --git a/core/src/sdk/grpc/GrpcClient.cpp b/core/src/sdk/grpc/GrpcClient.cpp index 5f278eb289..44c6565ceb 100644 --- a/core/src/sdk/grpc/GrpcClient.cpp +++ b/core/src/sdk/grpc/GrpcClient.cpp @@ -259,13 +259,13 @@ GrpcClient::PreloadTable(milvus::grpc::TableName& table_name) { } Status -GrpcClient::DeleteByRange(grpc::DeleteByRangeParam& delete_by_range_param) { +GrpcClient::DeleteByDate(grpc::DeleteByDateParam& delete_by_range_param) { ClientContext context; ::milvus::grpc::Status response; - ::grpc::Status grpc_status = stub_->DeleteByRange(&context, delete_by_range_param, &response); + ::grpc::Status grpc_status = stub_->DeleteByDate(&context, delete_by_range_param, &response); if (!grpc_status.ok()) { - std::cerr << "DeleteByRange gRPC failed!" << std::endl; + std::cerr << "DeleteByDate gRPC failed!" << std::endl; return Status(StatusCode::RPCFailed, grpc_status.error_message()); } @@ -317,4 +317,57 @@ GrpcClient::DropIndex(grpc::TableName& table_name) { return Status::OK(); } +Status +GrpcClient::CreatePartition(const grpc::PartitionParam& partition_param) { + ClientContext context; + ::milvus::grpc::Status response; + ::grpc::Status grpc_status = stub_->CreatePartition(&context, partition_param, &response); + + if (!grpc_status.ok()) { + std::cerr << "CreatePartition gRPC failed!" << std::endl; + return Status(StatusCode::RPCFailed, grpc_status.error_message()); + } + + if (response.error_code() != grpc::SUCCESS) { + std::cerr << response.reason() << std::endl; + return Status(StatusCode::ServerFailed, response.reason()); + } + return Status::OK(); +} + +Status +GrpcClient::ShowPartitions(const grpc::TableName& table_name, grpc::PartitionList& partition_array) const { + ClientContext context; + ::grpc::Status grpc_status = stub_->ShowPartitions(&context, table_name, &partition_array); + + if (!grpc_status.ok()) { + std::cerr << "ShowPartitions gRPC failed!" << std::endl; + return Status(StatusCode::RPCFailed, grpc_status.error_message()); + } + + if (partition_array.status().error_code() != grpc::SUCCESS) { + std::cerr << partition_array.status().reason() << std::endl; + return Status(StatusCode::ServerFailed, partition_array.status().reason()); + } + return Status::OK(); +} + +Status +GrpcClient::DropPartition(const ::milvus::grpc::PartitionParam& partition_param) { + ClientContext context; + ::milvus::grpc::Status response; + ::grpc::Status grpc_status = stub_->DropPartition(&context, partition_param, &response); + + if (!grpc_status.ok()) { + std::cerr << "DropPartition gRPC failed!" << std::endl; + return Status(StatusCode::RPCFailed, grpc_status.error_message()); + } + + if (response.error_code() != grpc::SUCCESS) { + std::cerr << response.reason() << std::endl; + return Status(StatusCode::ServerFailed, response.reason()); + } + return Status::OK(); +} + } // namespace milvus diff --git a/core/src/sdk/grpc/GrpcClient.h b/core/src/sdk/grpc/GrpcClient.h index a5e8b44968..329622dc59 100644 --- a/core/src/sdk/grpc/GrpcClient.h +++ b/core/src/sdk/grpc/GrpcClient.h @@ -72,7 +72,7 @@ class GrpcClient { Cmd(std::string& result, const std::string& cmd); Status - DeleteByRange(grpc::DeleteByRangeParam& delete_by_range_param); + DeleteByDate(grpc::DeleteByDateParam& delete_by_range_param); Status PreloadTable(grpc::TableName& table_name); @@ -83,6 +83,15 @@ class GrpcClient { Status DropIndex(grpc::TableName& table_name); + Status + CreatePartition(const grpc::PartitionParam& partition_param); + + Status + ShowPartitions(const grpc::TableName& table_name, grpc::PartitionList& partition_array) const; + + Status + DropPartition(const ::milvus::grpc::PartitionParam& partition_param); + Status Disconnect(); diff --git a/core/src/sdk/include/MilvusApi.h b/core/src/sdk/include/MilvusApi.h index 87ce88e460..9fa98deb40 100644 --- a/core/src/sdk/include/MilvusApi.h +++ b/core/src/sdk/include/MilvusApi.h @@ -31,12 +31,12 @@ namespace milvus { * @brief Index Type */ enum class IndexType { - invalid = 0, - cpu_idmap, - gpu_ivfflat, - gpu_ivfsq8, - mix_nsg, - ivfsq8h, + INVALID = 0, + FLAT = 1, + IVFFLAT = 2, + IVFSQ8 = 3, + NSG = 4, + IVFSQ8H = 5, }; enum class MetricType { @@ -64,7 +64,7 @@ struct TableSchema { /** * @brief Range information - * for DATE partition, the format is like: 'year-month-day' + * for DATE range, the format is like: 'year-month-day' */ struct Range { std::string start_value; ///< Range start @@ -81,11 +81,11 @@ struct RowRecord { /** * @brief TopK query result */ -struct TopKQueryResult { - int64_t row_num; +struct QueryResult { std::vector ids; std::vector distances; }; +using TopKQueryResult = std::vector; /** * @brief index parameters @@ -96,6 +96,17 @@ struct IndexParam { int32_t nlist; }; +/** + * @brief partition parameters + */ +struct PartitionParam { + std::string table_name; + std::string partition_name; + std::string partition_tag; +}; + +using PartitionList = std::vector; + /** * @brief SDK main class */ @@ -189,7 +200,7 @@ class Connection { * * This method is used to create table * - * @param table_name, table name is going to be tested. + * @param table_name, target table's name. * * @return Indicate if table is cexist */ @@ -199,9 +210,9 @@ class Connection { /** * @brief Delete table method * - * This method is used to delete table. + * This method is used to delete table(and its partitions). * - * @param table_name, table name is going to be deleted. + * @param table_name, target table's name. * * @return Indicate if table is delete successfully. */ @@ -211,7 +222,7 @@ class Connection { /** * @brief Create index method * - * This method is used to create index for whole table + * This method is used to create index for whole table(and its partitions). * * @param IndexParam * table_name, table name is going to be create index. @@ -229,14 +240,15 @@ class Connection { * * This method is used to add vector array to table. * - * @param table_name, table_name is inserted. + * @param table_name, target table's name. + * @param partition_tag, target partition's tag, keep empty if no partition. * @param record_array, vector array is inserted. * @param id_array, after inserted every vector is given a id. * * @return Indicate if vector array are inserted successfully */ virtual Status - Insert(const std::string& table_name, const std::vector& record_array, + Insert(const std::string& table_name, const std::string& partition_tag, const std::vector& record_array, std::vector& id_array) = 0; /** @@ -244,7 +256,8 @@ class Connection { * * This method is used to query vector in table. * - * @param table_name, table_name is queried. + * @param table_name, target table's name, keep empty if no partition. + * @param partition_tags, target partitions. * @param query_record_array, all vector are going to be queried. * @param query_range_array, time ranges, if not specified, will search in whole table * @param topk, how many similarity vectors will be searched. @@ -253,16 +266,16 @@ class Connection { * @return Indicate if query is successful. */ virtual Status - Search(const std::string& table_name, const std::vector& query_record_array, - const std::vector& query_range_array, int64_t topk, int64_t nprobe, - TopKQueryResult& topk_query_result) = 0; + Search(const std::string& table_name, const std::vector& partiton_tags, + const std::vector& query_record_array, const std::vector& query_range_array, int64_t topk, + int64_t nprobe, TopKQueryResult& topk_query_result) = 0; /** * @brief Show table description * * This method is used to show table information. * - * @param table_name, which table is show. + * @param table_name, target table's name. * @param table_schema, table_schema is given when operation is successful. * * @return Indicate if this operation is successful. @@ -275,8 +288,8 @@ class Connection { * * This method is used to get table row count. * - * @param table_name, table's name. - * @param row_count, table total row count. + * @param table_name, target table's name. + * @param row_count, table total row count(including partitions). * * @return Indicate if this operation is successful. */ @@ -325,21 +338,28 @@ class Connection { virtual std::string ServerStatus() const = 0; + /** + * @brief dump server tasks information + * + * This method is internal used. + * + * @return Server status. + */ virtual std::string DumpTaskTables() const = 0; /** - * @brief delete tables by range + * @brief delete tables by date range * - * This method is used to delete tables by range. + * This method is used to delete table data by date range. * + * @param table_name, target table's name. * @param Range, table range to delete. - * @param table_name * * @return Indicate if this operation is successful. */ virtual Status - DeleteByRange(Range& range, const std::string& table_name) = 0; + DeleteByDate(const std::string& table_name, const Range& range) = 0; /** * @brief preload table @@ -358,9 +378,10 @@ class Connection { * * This method is used to describe index * - * @param table_name + * @param table_name, target table's name. + * @param index_param, returned index information. * - * @return index informations and indicate if this operation is successful. + * @return Indicate if this operation is successful. */ virtual Status DescribeIndex(const std::string& table_name, IndexParam& index_param) const = 0; @@ -368,14 +389,53 @@ class Connection { /** * @brief drop index * - * This method is used to drop index + * This method is used to drop index of table(and its partitions) * - * @param table_name + * @param table_name, target table's name. * * @return Indicate if this operation is successful. */ virtual Status DropIndex(const std::string& table_name) const = 0; + + /** + * @brief Create partition method + * + * This method is used to create table partition + * + * @param param, use to provide partition information to be created. + * + * @return Indicate if partition is created successfully + */ + virtual Status + CreatePartition(const PartitionParam& param) = 0; + + /** + * @brief Test table existence method + * + * This method is used to create table + * + * @param table_name, table name is going to be tested. + * @param partition_array, partition array of the table. + * + * @return Indicate if this operation is successful + */ + virtual Status + ShowPartitions(const std::string& table_name, PartitionList& partition_array) const = 0; + + /** + * @brief Delete partition method + * + * This method is used to delete table partition. + * + * @param param, target partition to be deleted. + * NOTE: if param.table_name is empty, you must specify param.partition_name, + * else you can specify param.table_name and param.tag and let the param.partition_name be empty + * + * @return Indicate if partition is delete successfully. + */ + virtual Status + DropPartition(const PartitionParam& param) = 0; }; } // namespace milvus diff --git a/core/src/sdk/interface/ConnectionImpl.cpp b/core/src/sdk/interface/ConnectionImpl.cpp index 05d3bf6c36..448c011c1f 100644 --- a/core/src/sdk/interface/ConnectionImpl.cpp +++ b/core/src/sdk/interface/ConnectionImpl.cpp @@ -83,16 +83,17 @@ ConnectionImpl::CreateIndex(const IndexParam& index_param) { } Status -ConnectionImpl::Insert(const std::string& table_name, const std::vector& record_array, - std::vector& id_array) { - return client_proxy_->Insert(table_name, record_array, id_array); +ConnectionImpl::Insert(const std::string& table_name, const std::string& partition_tag, + const std::vector& record_array, std::vector& id_array) { + return client_proxy_->Insert(table_name, partition_tag, record_array, id_array); } Status -ConnectionImpl::Search(const std::string& table_name, const std::vector& query_record_array, - const std::vector& query_range_array, int64_t topk, int64_t nprobe, - TopKQueryResult& topk_query_result) { - return client_proxy_->Search(table_name, query_record_array, query_range_array, topk, nprobe, topk_query_result); +ConnectionImpl::Search(const std::string& table_name, const std::vector& partiton_tags, + const std::vector& query_record_array, const std::vector& query_range_array, + int64_t topk, int64_t nprobe, TopKQueryResult& topk_query_result) { + return client_proxy_->Search(table_name, partiton_tags, query_record_array, query_range_array, topk, nprobe, + topk_query_result); } Status @@ -126,8 +127,8 @@ ConnectionImpl::DumpTaskTables() const { } Status -ConnectionImpl::DeleteByRange(Range& range, const std::string& table_name) { - return client_proxy_->DeleteByRange(range, table_name); +ConnectionImpl::DeleteByDate(const std::string& table_name, const Range& range) { + return client_proxy_->DeleteByDate(table_name, range); } Status @@ -145,4 +146,19 @@ ConnectionImpl::DropIndex(const std::string& table_name) const { return client_proxy_->DropIndex(table_name); } +Status +ConnectionImpl::CreatePartition(const PartitionParam& param) { + return client_proxy_->CreatePartition(param); +} + +Status +ConnectionImpl::ShowPartitions(const std::string& table_name, PartitionList& partition_array) const { + return client_proxy_->ShowPartitions(table_name, partition_array); +} + +Status +ConnectionImpl::DropPartition(const PartitionParam& param) { + return client_proxy_->DropPartition(param); +} + } // namespace milvus diff --git a/core/src/sdk/interface/ConnectionImpl.h b/core/src/sdk/interface/ConnectionImpl.h index b7dd5b6402..126e59c375 100644 --- a/core/src/sdk/interface/ConnectionImpl.h +++ b/core/src/sdk/interface/ConnectionImpl.h @@ -56,13 +56,13 @@ class ConnectionImpl : public Connection { CreateIndex(const IndexParam& index_param) override; Status - Insert(const std::string& table_name, const std::vector& record_array, + Insert(const std::string& table_name, const std::string& partition_tag, const std::vector& record_array, std::vector& id_array) override; Status - Search(const std::string& table_name, const std::vector& query_record_array, - const std::vector& query_range_array, int64_t topk, int64_t nprobe, - TopKQueryResult& topk_query_result) override; + Search(const std::string& table_name, const std::vector& partiton_tags, + const std::vector& query_record_array, const std::vector& query_range_array, int64_t topk, + int64_t nprobe, TopKQueryResult& topk_query_result) override; Status DescribeTable(const std::string& table_name, TableSchema& table_schema) override; @@ -86,7 +86,7 @@ class ConnectionImpl : public Connection { DumpTaskTables() const override; Status - DeleteByRange(Range& range, const std::string& table_name) override; + DeleteByDate(const std::string& table_name, const Range& range) override; Status PreloadTable(const std::string& table_name) const override; @@ -97,6 +97,15 @@ class ConnectionImpl : public Connection { Status DropIndex(const std::string& table_name) const override; + Status + CreatePartition(const PartitionParam& param) override; + + Status + ShowPartitions(const std::string& table_name, PartitionList& partition_array) const override; + + Status + DropPartition(const PartitionParam& param) override; + private: std::shared_ptr client_proxy_; }; diff --git a/core/src/server/Config.cpp b/core/src/server/Config.cpp index ccc518cac3..f3efcff0cc 100644 --- a/core/src/server/Config.cpp +++ b/core/src/server/Config.cpp @@ -113,19 +113,19 @@ Config::ValidateConfig() { return s; } - int32_t db_archive_disk_threshold; + int64_t db_archive_disk_threshold; s = GetDBConfigArchiveDiskThreshold(db_archive_disk_threshold); if (!s.ok()) { return s; } - int32_t db_archive_days_threshold; + int64_t db_archive_days_threshold; s = GetDBConfigArchiveDaysThreshold(db_archive_days_threshold); if (!s.ok()) { return s; } - int32_t db_insert_buffer_size; + int64_t db_insert_buffer_size; s = GetDBConfigInsertBufferSize(db_insert_buffer_size); if (!s.ok()) { return s; @@ -163,18 +163,6 @@ Config::ValidateConfig() { return s; } - int64_t cache_gpu_cache_capacity; - s = GetCacheConfigGpuCacheCapacity(cache_gpu_cache_capacity); - if (!s.ok()) { - return s; - } - - float cache_gpu_cache_threshold; - s = GetCacheConfigGpuCacheThreshold(cache_gpu_cache_threshold); - if (!s.ok()) { - return s; - } - bool cache_insert_data; s = GetCacheConfigCacheInsertData(cache_insert_data); if (!s.ok()) { @@ -182,43 +170,57 @@ Config::ValidateConfig() { } /* engine config */ - int32_t engine_use_blas_threshold; + int64_t engine_use_blas_threshold; s = GetEngineConfigUseBlasThreshold(engine_use_blas_threshold); if (!s.ok()) { return s; } - int32_t engine_omp_thread_num; + int64_t engine_omp_thread_num; s = GetEngineConfigOmpThreadNum(engine_omp_thread_num); if (!s.ok()) { return s; } - int32_t engine_gpu_search_threshold; + int64_t engine_gpu_search_threshold; s = GetEngineConfigGpuSearchThreshold(engine_gpu_search_threshold); if (!s.ok()) { return s; } - /* resource config */ - std::string resource_mode; - s = GetResourceConfigMode(resource_mode); + /* gpu resource config */ +#ifdef MILVUS_GPU_VERSION + bool gpu_resource_enable; + s = GetGpuResourceConfigEnable(gpu_resource_enable); if (!s.ok()) { return s; } - std::vector search_resources; - s = GetResourceConfigSearchResources(search_resources); + int64_t resource_cache_capacity; + s = GetGpuResourceConfigCacheCapacity(resource_cache_capacity); if (!s.ok()) { return s; } - int32_t resource_index_build_device; - s = GetResourceConfigIndexBuildDevice(resource_index_build_device); + float resource_cache_threshold; + s = GetGpuResourceConfigCacheThreshold(resource_cache_threshold); if (!s.ok()) { return s; } + std::vector search_resources; + s = GetGpuResourceConfigSearchResources(search_resources); + if (!s.ok()) { + return s; + } + + std::vector index_build_resources; + s = GetGpuResourceConfigBuildIndexResources(index_build_resources); + if (!s.ok()) { + return s; + } +#endif + return Status::OK(); } @@ -305,16 +307,6 @@ Config::ResetDefaultConfig() { return s; } - s = SetCacheConfigGpuCacheCapacity(CONFIG_CACHE_GPU_CACHE_CAPACITY_DEFAULT); - if (!s.ok()) { - return s; - } - - s = SetCacheConfigGpuCacheThreshold(CONFIG_CACHE_GPU_CACHE_THRESHOLD_DEFAULT); - if (!s.ok()) { - return s; - } - s = SetCacheConfigCacheInsertData(CONFIG_CACHE_CACHE_INSERT_DATA_DEFAULT); if (!s.ok()) { return s; @@ -336,22 +328,34 @@ Config::ResetDefaultConfig() { return s; } - /* resource config */ - s = SetResourceConfigMode(CONFIG_RESOURCE_MODE_DEFAULT); + /* gpu resource config */ +#ifdef MILVUS_GPU_VERSION + s = SetGpuResourceConfigEnable(CONFIG_GPU_RESOURCE_ENABLE_DEFAULT); if (!s.ok()) { return s; } - s = SetResourceConfigSearchResources(CONFIG_RESOURCE_SEARCH_RESOURCES_DEFAULT); + s = SetGpuResourceConfigCacheCapacity(CONFIG_GPU_RESOURCE_CACHE_CAPACITY_DEFAULT); if (!s.ok()) { return s; } - s = SetResourceConfigIndexBuildDevice(CONFIG_RESOURCE_INDEX_BUILD_DEVICE_DEFAULT); + s = SetGpuResourceConfigCacheThreshold(CONFIG_GPU_RESOURCE_CACHE_THRESHOLD_DEFAULT); if (!s.ok()) { return s; } + s = SetGpuResourceConfigSearchResources(CONFIG_GPU_RESOURCE_SEARCH_RESOURCES_DEFAULT); + if (!s.ok()) { + return s; + } + + s = SetGpuResourceConfigBuildIndexResources(CONFIG_GPU_RESOURCE_BUILD_INDEX_RESOURCES_DEFAULT); + if (!s.ok()) { + return s; + } +#endif + return Status::OK(); } @@ -373,7 +377,7 @@ Config::PrintAll() { PrintConfigSection(CONFIG_CACHE); PrintConfigSection(CONFIG_METRIC); PrintConfigSection(CONFIG_ENGINE); - PrintConfigSection(CONFIG_RESOURCE); + PrintConfigSection(CONFIG_GPU_RESOURCE); } //////////////////////////////////////////////////////////////////////////////// @@ -481,7 +485,7 @@ Config::CheckDBConfigInsertBufferSize(const std::string& value) { ". Possible reason: db_config.insert_buffer_size is not a positive integer."; return Status(SERVER_INVALID_ARGUMENT, msg); } else { - int64_t buffer_size = std::stoi(value) * GB; + int64_t buffer_size = std::stoll(value) * GB; if (buffer_size <= 0) { std::string msg = "Invalid insert buffer size: " + value + ". Possible reason: db_config.insert_buffer_size is not a positive integer."; @@ -536,7 +540,7 @@ Config::CheckCacheConfigCpuCacheCapacity(const std::string& value) { ". Possible reason: cache_config.cpu_cache_capacity is not a positive integer."; return Status(SERVER_INVALID_ARGUMENT, msg); } else { - int64_t cpu_cache_capacity = std::stoi(value) * GB; + int64_t cpu_cache_capacity = std::stoll(value) * GB; if (cpu_cache_capacity <= 0) { std::string msg = "Invalid cpu cache capacity: " + value + ". Possible reason: cache_config.cpu_cache_capacity is not a positive integer."; @@ -553,7 +557,7 @@ Config::CheckCacheConfigCpuCacheCapacity(const std::string& value) { std::cerr << "WARNING: cpu cache capacity value is too big" << std::endl; } - int32_t buffer_value; + int64_t buffer_value; Status s = GetDBConfigInsertBufferSize(buffer_value); if (!s.ok()) { return s; @@ -587,55 +591,6 @@ Config::CheckCacheConfigCpuCacheThreshold(const std::string& value) { return Status::OK(); } -Status -Config::CheckCacheConfigGpuCacheCapacity(const std::string& value) { - if (!ValidationUtil::ValidateStringIsNumber(value).ok()) { - std::string msg = "Invalid gpu cache capacity: " + value + - ". Possible reason: cache_config.gpu_cache_capacity is not a positive integer."; - return Status(SERVER_INVALID_ARGUMENT, msg); - } else { - uint64_t gpu_cache_capacity = std::stoi(value) * GB; - int device_id; - Status s = GetResourceConfigIndexBuildDevice(device_id); - if (!s.ok()) { - return s; - } - - if (device_id == server::CPU_DEVICE_ID) - return Status::OK(); - - size_t gpu_memory; - if (!ValidationUtil::GetGpuMemory(device_id, gpu_memory).ok()) { - std::string msg = "Fail to get GPU memory for GPU device: " + std::to_string(device_id); - return Status(SERVER_UNEXPECTED_ERROR, msg); - } else if (gpu_cache_capacity >= gpu_memory) { - std::string msg = "Invalid gpu cache capacity: " + value + - ". Possible reason: cache_config.gpu_cache_capacity exceeds GPU memory."; - return Status(SERVER_INVALID_ARGUMENT, msg); - } else if (gpu_cache_capacity > (double)gpu_memory * 0.9) { - std::cerr << "Warning: gpu cache capacity value is too big" << std::endl; - } - } - return Status::OK(); -} - -Status -Config::CheckCacheConfigGpuCacheThreshold(const std::string& value) { - if (!ValidationUtil::ValidateStringIsFloat(value).ok()) { - std::string msg = "Invalid gpu cache threshold: " + value + - ". Possible reason: cache_config.gpu_cache_threshold is not in range (0.0, 1.0]."; - return Status(SERVER_INVALID_ARGUMENT, msg); - } else { - float gpu_cache_threshold = std::stof(value); - if (gpu_cache_threshold <= 0.0 || gpu_cache_threshold >= 1.0) { - std::string msg = "Invalid gpu cache threshold: " + value + - ". Possible reason: cache_config.gpu_cache_threshold is not in range (0.0, 1.0]."; - return Status(SERVER_INVALID_ARGUMENT, msg); - } - } - return Status::OK(); -} - Status Config::CheckCacheConfigCacheInsertData(const std::string& value) { if (!ValidationUtil::ValidateStringIsBool(value).ok()) { @@ -664,10 +619,10 @@ Config::CheckEngineConfigOmpThreadNum(const std::string& value) { return Status(SERVER_INVALID_ARGUMENT, msg); } - int32_t omp_thread = std::stoi(value); - uint32_t sys_thread_cnt = 8; + int64_t omp_thread = std::stoll(value); + int64_t sys_thread_cnt = 8; CommonUtil::GetSystemAvailableThreads(sys_thread_cnt); - if (omp_thread > static_cast(sys_thread_cnt)) { + if (omp_thread > sys_thread_cnt) { std::string msg = "Invalid omp thread num: " + value + ". Possible reason: engine_config.omp_thread_num exceeds system cpu cores."; return Status(SERVER_INVALID_ARGUMENT, msg); @@ -686,75 +641,122 @@ Config::CheckEngineConfigGpuSearchThreshold(const std::string& value) { } Status -Config::CheckResourceConfigMode(const std::string& value) { - if (value != "simple") { - std::string msg = "Invalid resource mode: " + value + ". Possible reason: resource_config.mode is invalid."; - return Status(SERVER_INVALID_ARGUMENT, msg); - } - return Status::OK(); -} - -Status -CheckGpuDevice(const std::string& value) { - const std::regex pat("gpu(\\d+)"); - std::cmatch m; - if (!std::regex_match(value.c_str(), m, pat)) { - std::string msg = "Invalid gpu device: " + value + - ". Possible reason: resource_config.search_resources does not match your hardware."; - return Status(SERVER_INVALID_ARGUMENT, msg); - } - - int32_t gpu_index = std::stoi(value.substr(3)); - if (!ValidationUtil::ValidateGpuIndex(gpu_index).ok()) { - std::string msg = "Invalid gpu device: " + value + - ". Possible reason: resource_config.search_resources does not match your hardware."; - return Status(SERVER_INVALID_ARGUMENT, msg); - } - return Status::OK(); -} - -Status -Config::CheckResourceConfigSearchResources(const std::vector& value) { - if (value.empty()) { +Config::CheckGpuResourceConfigEnable(const std::string& value) { + if (!ValidationUtil::ValidateStringIsBool(value).ok()) { std::string msg = - "Invalid search resource. " - "Possible reason: resource_config.search_resources is empty."; + "Invalid gpu resource config: " + value + ". Possible reason: gpu_resource_config.enable is not a boolean."; + return Status(SERVER_INVALID_ARGUMENT, msg); + } + return Status::OK(); +} + +Status +Config::CheckGpuResourceConfigCacheCapacity(const std::string& value) { + if (!ValidationUtil::ValidateStringIsNumber(value).ok()) { + std::string msg = "Invalid gpu cache capacity: " + value + + ". Possible reason: gpu_resource_config.cache_capacity is not a positive integer."; + return Status(SERVER_INVALID_ARGUMENT, msg); + } else { + int64_t gpu_cache_capacity = std::stoll(value) * GB; + std::vector gpu_ids; + Status s = GetGpuResourceConfigBuildIndexResources(gpu_ids); + if (!s.ok()) { + return s; + } + + for (int64_t gpu_id : gpu_ids) { + size_t gpu_memory; + if (!ValidationUtil::GetGpuMemory(gpu_id, gpu_memory).ok()) { + std::string msg = "Fail to get GPU memory for GPU device: " + std::to_string(gpu_id); + return Status(SERVER_UNEXPECTED_ERROR, msg); + } else if (gpu_cache_capacity >= gpu_memory) { + std::string msg = "Invalid gpu cache capacity: " + value + + ". Possible reason: gpu_resource_config.cache_capacity exceeds GPU memory."; + return Status(SERVER_INVALID_ARGUMENT, msg); + } else if (gpu_cache_capacity > (double)gpu_memory * 0.9) { + std::cerr << "Warning: gpu cache capacity value is too big" << std::endl; + } + } + } + return Status::OK(); +} + +Status +Config::CheckGpuResourceConfigCacheThreshold(const std::string& value) { + if (!ValidationUtil::ValidateStringIsFloat(value).ok()) { + std::string msg = "Invalid gpu cache threshold: " + value + + ". Possible reason: gpu_resource_config.cache_threshold is not in range (0.0, 1.0]."; + return Status(SERVER_INVALID_ARGUMENT, msg); + } else { + float gpu_cache_threshold = std::stof(value); + if (gpu_cache_threshold <= 0.0 || gpu_cache_threshold >= 1.0) { + std::string msg = "Invalid gpu cache threshold: " + value + + ". Possible reason: gpu_resource_config.cache_threshold is not in range (0.0, 1.0]."; + return Status(SERVER_INVALID_ARGUMENT, msg); + } + } + return Status::OK(); +} + +Status +CheckGpuResource(const std::string& value) { + std::string s = value; + std::transform(s.begin(), s.end(), s.begin(), ::tolower); + + const std::regex pat("gpu(\\d+)"); + std::smatch m; + if (!std::regex_match(s, m, pat)) { + std::string msg = "Invalid gpu resource: " + value + + ". Possible reason: gpu_resource_config is not in the format of cpux or gpux"; return Status(SERVER_INVALID_ARGUMENT, msg); } - bool cpu_found = false, gpu_found = false; - for (auto& device : value) { - if (device == "cpu") { - cpu_found = true; - continue; - } - if (CheckGpuDevice(device).ok()) { - gpu_found = true; - } else { - std::string msg = "Invalid search resource: " + device + - ". Possible reason: resource_config.search_resources does not match your hardware."; + if (s.compare(0, 3, "gpu") == 0) { + int32_t gpu_index = std::stoi(s.substr(3)); + if (!ValidationUtil::ValidateGpuIndex(gpu_index).ok()) { + std::string msg = "Invalid gpu resource: " + value + + ". Possible reason: gpu_resource_config does not match with the hardware."; return Status(SERVER_INVALID_ARGUMENT, msg); } } - if (cpu_found && !gpu_found) { + return Status::OK(); +} + +Status +Config::CheckGpuResourceConfigSearchResources(const std::vector& value) { + if (value.empty()) { std::string msg = - "Invalid search resource. Possible reason: resource_config.search_resources has only CPU resource."; + "Invalid gpu search resource. " + "Possible reason: gpu_resource_config.search_resources is empty."; return Status(SERVER_INVALID_ARGUMENT, msg); } + + for (auto& resource : value) { + auto status = CheckGpuResource(resource); + if (!status.ok()) { + return Status(SERVER_INVALID_ARGUMENT, status.message()); + } + } return Status::OK(); } Status -Config::CheckResourceConfigIndexBuildDevice(const std::string& value) { - // if (value == "cpu") { - // return Status::OK(); - // } - if (!CheckGpuDevice(value).ok()) { - std::string msg = "Invalid index build device: " + value + - ". Possible reason: resource_config.index_build_device does not match your hardware."; +Config::CheckGpuResourceConfigBuildIndexResources(const std::vector& value) { + if (value.empty()) { + std::string msg = + "Invalid gpu build index resource. " + "Possible reason: gpu_resource_config.build_index_resources is empty."; return Status(SERVER_INVALID_ARGUMENT, msg); } + + for (auto& resource : value) { + auto status = CheckGpuResource(resource); + if (!status.ok()) { + return Status(SERVER_INVALID_ARGUMENT, status.message()); + } + } + return Status::OK(); } @@ -853,40 +855,37 @@ Config::GetDBConfigBackendUrl(std::string& value) { } Status -Config::GetDBConfigArchiveDiskThreshold(int32_t& value) { +Config::GetDBConfigArchiveDiskThreshold(int64_t& value) { std::string str = GetConfigStr(CONFIG_DB, CONFIG_DB_ARCHIVE_DISK_THRESHOLD, CONFIG_DB_ARCHIVE_DISK_THRESHOLD_DEFAULT); Status s = CheckDBConfigArchiveDiskThreshold(str); if (!s.ok()) { return s; } - - value = std::stoi(str); + value = std::stoll(str); return Status::OK(); } Status -Config::GetDBConfigArchiveDaysThreshold(int32_t& value) { +Config::GetDBConfigArchiveDaysThreshold(int64_t& value) { std::string str = GetConfigStr(CONFIG_DB, CONFIG_DB_ARCHIVE_DAYS_THRESHOLD, CONFIG_DB_ARCHIVE_DAYS_THRESHOLD_DEFAULT); Status s = CheckDBConfigArchiveDaysThreshold(str); if (!s.ok()) { return s; } - - value = std::stoi(str); + value = std::stoll(str); return Status::OK(); } Status -Config::GetDBConfigInsertBufferSize(int32_t& value) { +Config::GetDBConfigInsertBufferSize(int64_t& value) { std::string str = GetConfigStr(CONFIG_DB, CONFIG_DB_INSERT_BUFFER_SIZE, CONFIG_DB_INSERT_BUFFER_SIZE_DEFAULT); Status s = CheckDBConfigInsertBufferSize(str); if (!s.ok()) { return s; } - - value = std::stoi(str); + value = std::stoll(str); return Status::OK(); } @@ -903,7 +902,6 @@ Config::GetMetricConfigEnableMonitor(bool& value) { if (!s.ok()) { return s; } - std::transform(str.begin(), str.end(), str.begin(), ::tolower); value = (str == "true" || str == "on" || str == "yes" || str == "1"); return Status::OK(); @@ -929,8 +927,7 @@ Config::GetCacheConfigCpuCacheCapacity(int64_t& value) { if (!s.ok()) { return s; } - - value = std::stoi(str); + value = std::stoll(str); return Status::OK(); } @@ -942,33 +939,6 @@ Config::GetCacheConfigCpuCacheThreshold(float& value) { if (!s.ok()) { return s; } - - value = std::stof(str); - return Status::OK(); -} - -Status -Config::GetCacheConfigGpuCacheCapacity(int64_t& value) { - std::string str = - GetConfigStr(CONFIG_CACHE, CONFIG_CACHE_GPU_CACHE_CAPACITY, CONFIG_CACHE_GPU_CACHE_CAPACITY_DEFAULT); - Status s = CheckCacheConfigGpuCacheCapacity(str); - if (!s.ok()) { - return s; - } - - value = std::stoi(str); - return Status::OK(); -} - -Status -Config::GetCacheConfigGpuCacheThreshold(float& value) { - std::string str = - GetConfigStr(CONFIG_CACHE, CONFIG_CACHE_GPU_CACHE_THRESHOLD, CONFIG_CACHE_GPU_CACHE_THRESHOLD_DEFAULT); - Status s = CheckCacheConfigGpuCacheThreshold(str); - if (!s.ok()) { - return s; - } - value = std::stof(str); return Status::OK(); } @@ -981,80 +951,148 @@ Config::GetCacheConfigCacheInsertData(bool& value) { if (!s.ok()) { return s; } - std::transform(str.begin(), str.end(), str.begin(), ::tolower); value = (str == "true" || str == "on" || str == "yes" || str == "1"); return Status::OK(); } Status -Config::GetEngineConfigUseBlasThreshold(int32_t& value) { +Config::GetEngineConfigUseBlasThreshold(int64_t& value) { std::string str = GetConfigStr(CONFIG_ENGINE, CONFIG_ENGINE_USE_BLAS_THRESHOLD, CONFIG_ENGINE_USE_BLAS_THRESHOLD_DEFAULT); Status s = CheckEngineConfigUseBlasThreshold(str); if (!s.ok()) { return s; } - - value = std::stoi(str); + value = std::stoll(str); return Status::OK(); } Status -Config::GetEngineConfigOmpThreadNum(int32_t& value) { +Config::GetEngineConfigOmpThreadNum(int64_t& value) { std::string str = GetConfigStr(CONFIG_ENGINE, CONFIG_ENGINE_OMP_THREAD_NUM, CONFIG_ENGINE_OMP_THREAD_NUM_DEFAULT); Status s = CheckEngineConfigOmpThreadNum(str); if (!s.ok()) { return s; } - - value = std::stoi(str); + value = std::stoll(str); return Status::OK(); } Status -Config::GetEngineConfigGpuSearchThreshold(int32_t& value) { +Config::GetEngineConfigGpuSearchThreshold(int64_t& value) { std::string str = GetConfigStr(CONFIG_ENGINE, CONFIG_ENGINE_GPU_SEARCH_THRESHOLD, CONFIG_ENGINE_GPU_SEARCH_THRESHOLD_DEFAULT); Status s = CheckEngineConfigGpuSearchThreshold(str); if (!s.ok()) { return s; } - - value = std::stoi(str); + value = std::stoll(str); return Status::OK(); } Status -Config::GetResourceConfigMode(std::string& value) { - value = GetConfigStr(CONFIG_RESOURCE, CONFIG_RESOURCE_MODE, CONFIG_RESOURCE_MODE_DEFAULT); - return CheckResourceConfigMode(value); -} - -Status -Config::GetResourceConfigSearchResources(std::vector& value) { - std::string str = - GetConfigSequenceStr(CONFIG_RESOURCE, CONFIG_RESOURCE_SEARCH_RESOURCES, - CONFIG_RESOURCE_SEARCH_RESOURCES_DELIMITER, CONFIG_RESOURCE_SEARCH_RESOURCES_DEFAULT); - server::StringHelpFunctions::SplitStringByDelimeter(str, CONFIG_RESOURCE_SEARCH_RESOURCES_DELIMITER, value); - return CheckResourceConfigSearchResources(value); -} - -Status -Config::GetResourceConfigIndexBuildDevice(int32_t& value) { - std::string str = - GetConfigStr(CONFIG_RESOURCE, CONFIG_RESOURCE_INDEX_BUILD_DEVICE, CONFIG_RESOURCE_INDEX_BUILD_DEVICE_DEFAULT); - Status s = CheckResourceConfigIndexBuildDevice(str); +Config::GetGpuResourceConfigEnable(bool& value) { + std::string str = GetConfigStr(CONFIG_GPU_RESOURCE, CONFIG_GPU_RESOURCE_ENABLE, CONFIG_GPU_RESOURCE_ENABLE_DEFAULT); + Status s = CheckGpuResourceConfigEnable(str); if (!s.ok()) { return s; } + std::transform(str.begin(), str.end(), str.begin(), ::tolower); + value = (str == "true" || str == "on" || str == "yes" || str == "1"); + return Status::OK(); +} - if (str == "cpu") { - value = CPU_DEVICE_ID; - } else { - value = std::stoi(str.substr(3)); +Status +Config::GetGpuResourceConfigCacheCapacity(int64_t& value) { + bool gpu_resource_enable = false; + Status s = GetGpuResourceConfigEnable(gpu_resource_enable); + if (!s.ok()) { + return s; } + if (!gpu_resource_enable) { + std::string msg = "GPU not supported. Possible reason: gpu_resource_config.enable is set to false."; + return Status(SERVER_UNSUPPORTED_ERROR, msg); + } + std::string str = GetConfigStr(CONFIG_GPU_RESOURCE, CONFIG_GPU_RESOURCE_CACHE_CAPACITY, + CONFIG_GPU_RESOURCE_CACHE_CAPACITY_DEFAULT); + s = CheckGpuResourceConfigCacheCapacity(str); + if (!s.ok()) { + return s; + } + value = std::stoll(str); + return Status::OK(); +} +Status +Config::GetGpuResourceConfigCacheThreshold(float& value) { + bool gpu_resource_enable = false; + Status s = GetGpuResourceConfigEnable(gpu_resource_enable); + if (!s.ok()) { + return s; + } + if (!gpu_resource_enable) { + std::string msg = "GPU not supported. Possible reason: gpu_resource_config.enable is set to false."; + return Status(SERVER_UNSUPPORTED_ERROR, msg); + } + std::string str = GetConfigStr(CONFIG_GPU_RESOURCE, CONFIG_GPU_RESOURCE_CACHE_THRESHOLD, + CONFIG_GPU_RESOURCE_CACHE_THRESHOLD_DEFAULT); + s = CheckGpuResourceConfigCacheThreshold(str); + if (!s.ok()) { + return s; + } + value = std::stof(str); + return Status::OK(); +} + +Status +Config::GetGpuResourceConfigSearchResources(std::vector& value) { + bool gpu_resource_enable = false; + Status s = GetGpuResourceConfigEnable(gpu_resource_enable); + if (!s.ok()) { + return s; + } + if (!gpu_resource_enable) { + std::string msg = "GPU not supported. Possible reason: gpu_resource_config.enable is set to false."; + return Status(SERVER_UNSUPPORTED_ERROR, msg); + } + std::string str = GetConfigSequenceStr(CONFIG_GPU_RESOURCE, CONFIG_GPU_RESOURCE_SEARCH_RESOURCES, + CONFIG_GPU_RESOURCE_DELIMITER, CONFIG_GPU_RESOURCE_SEARCH_RESOURCES_DEFAULT); + std::vector res_vec; + server::StringHelpFunctions::SplitStringByDelimeter(str, CONFIG_GPU_RESOURCE_DELIMITER, res_vec); + s = CheckGpuResourceConfigSearchResources(res_vec); + if (!s.ok()) { + return s; + } + for (std::string& res : res_vec) { + value.push_back(std::stoll(res.substr(3))); + } + return Status::OK(); +} + +Status +Config::GetGpuResourceConfigBuildIndexResources(std::vector& value) { + bool gpu_resource_enable = false; + Status s = GetGpuResourceConfigEnable(gpu_resource_enable); + if (!s.ok()) { + return s; + } + if (!gpu_resource_enable) { + std::string msg = "GPU not supported. Possible reason: gpu_resource_config.enable is set to false."; + return Status(SERVER_UNSUPPORTED_ERROR, msg); + } + std::string str = + GetConfigSequenceStr(CONFIG_GPU_RESOURCE, CONFIG_GPU_RESOURCE_BUILD_INDEX_RESOURCES, + CONFIG_GPU_RESOURCE_DELIMITER, CONFIG_GPU_RESOURCE_BUILD_INDEX_RESOURCES_DEFAULT); + std::vector res_vec; + server::StringHelpFunctions::SplitStringByDelimeter(str, CONFIG_GPU_RESOURCE_DELIMITER, res_vec); + s = CheckGpuResourceConfigBuildIndexResources(res_vec); + if (!s.ok()) { + return s; + } + for (std::string& res : res_vec) { + value.push_back(std::stoll(res.substr(3))); + } return Status::OK(); } @@ -1066,7 +1104,6 @@ Config::SetServerConfigAddress(const std::string& value) { if (!s.ok()) { return s; } - SetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_ADDRESS, value); return Status::OK(); } @@ -1077,7 +1114,6 @@ Config::SetServerConfigPort(const std::string& value) { if (!s.ok()) { return s; } - SetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_PORT, value); return Status::OK(); } @@ -1088,7 +1124,6 @@ Config::SetServerConfigDeployMode(const std::string& value) { if (!s.ok()) { return s; } - SetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_DEPLOY_MODE, value); return Status::OK(); } @@ -1099,7 +1134,6 @@ Config::SetServerConfigTimeZone(const std::string& value) { if (!s.ok()) { return s; } - SetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_TIME_ZONE, value); return Status::OK(); } @@ -1111,7 +1145,6 @@ Config::SetDBConfigPrimaryPath(const std::string& value) { if (!s.ok()) { return s; } - SetConfigValueInMem(CONFIG_DB, CONFIG_DB_PRIMARY_PATH, value); return Status::OK(); } @@ -1122,7 +1155,6 @@ Config::SetDBConfigSecondaryPath(const std::string& value) { if (!s.ok()) { return s; } - SetConfigValueInMem(CONFIG_DB, CONFIG_DB_SECONDARY_PATH, value); return Status::OK(); } @@ -1133,7 +1165,6 @@ Config::SetDBConfigBackendUrl(const std::string& value) { if (!s.ok()) { return s; } - SetConfigValueInMem(CONFIG_DB, CONFIG_DB_BACKEND_URL, value); return Status::OK(); } @@ -1144,7 +1175,6 @@ Config::SetDBConfigArchiveDiskThreshold(const std::string& value) { if (!s.ok()) { return s; } - SetConfigValueInMem(CONFIG_DB, CONFIG_DB_ARCHIVE_DISK_THRESHOLD, value); return Status::OK(); } @@ -1155,7 +1185,6 @@ Config::SetDBConfigArchiveDaysThreshold(const std::string& value) { if (!s.ok()) { return s; } - SetConfigValueInMem(CONFIG_DB, CONFIG_DB_ARCHIVE_DAYS_THRESHOLD, value); return Status::OK(); } @@ -1166,7 +1195,6 @@ Config::SetDBConfigInsertBufferSize(const std::string& value) { if (!s.ok()) { return s; } - SetConfigValueInMem(CONFIG_DB, CONFIG_DB_INSERT_BUFFER_SIZE, value); return Status::OK(); } @@ -1178,7 +1206,6 @@ Config::SetMetricConfigEnableMonitor(const std::string& value) { if (!s.ok()) { return s; } - SetConfigValueInMem(CONFIG_METRIC, CONFIG_METRIC_ENABLE_MONITOR, value); return Status::OK(); } @@ -1189,7 +1216,6 @@ Config::SetMetricConfigCollector(const std::string& value) { if (!s.ok()) { return s; } - SetConfigValueInMem(CONFIG_METRIC, CONFIG_METRIC_COLLECTOR, value); return Status::OK(); } @@ -1200,7 +1226,6 @@ Config::SetMetricConfigPrometheusPort(const std::string& value) { if (!s.ok()) { return s; } - SetConfigValueInMem(CONFIG_METRIC, CONFIG_METRIC_PROMETHEUS_PORT, value); return Status::OK(); } @@ -1212,7 +1237,6 @@ Config::SetCacheConfigCpuCacheCapacity(const std::string& value) { if (!s.ok()) { return s; } - SetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_CPU_CACHE_CAPACITY, value); return Status::OK(); } @@ -1223,40 +1247,16 @@ Config::SetCacheConfigCpuCacheThreshold(const std::string& value) { if (!s.ok()) { return s; } - SetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_CPU_CACHE_THRESHOLD, value); return Status::OK(); } -Status -Config::SetCacheConfigGpuCacheCapacity(const std::string& value) { - Status s = CheckCacheConfigGpuCacheCapacity(value); - if (!s.ok()) { - return s; - } - - SetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_GPU_CACHE_CAPACITY, value); - return Status::OK(); -} - -Status -Config::SetCacheConfigGpuCacheThreshold(const std::string& value) { - Status s = CheckCacheConfigGpuCacheThreshold(value); - if (!s.ok()) { - return s; - } - - SetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_GPU_CACHE_THRESHOLD, value); - return Status::OK(); -} - Status Config::SetCacheConfigCacheInsertData(const std::string& value) { Status s = CheckCacheConfigCacheInsertData(value); if (!s.ok()) { return s; } - SetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_CACHE_INSERT_DATA, value); return Status::OK(); } @@ -1268,7 +1268,6 @@ Config::SetEngineConfigUseBlasThreshold(const std::string& value) { if (!s.ok()) { return s; } - SetConfigValueInMem(CONFIG_ENGINE, CONFIG_ENGINE_USE_BLAS_THRESHOLD, value); return Status::OK(); } @@ -1279,7 +1278,6 @@ Config::SetEngineConfigOmpThreadNum(const std::string& value) { if (!s.ok()) { return s; } - SetConfigValueInMem(CONFIG_ENGINE, CONFIG_ENGINE_OMP_THREAD_NUM, value); return Status::OK(); } @@ -1290,47 +1288,64 @@ Config::SetEngineConfigGpuSearchThreshold(const std::string& value) { if (!s.ok()) { return s; } - SetConfigValueInMem(CONFIG_ENGINE, CONFIG_ENGINE_GPU_SEARCH_THRESHOLD, value); return Status::OK(); } -/* resource config */ +/* gpu resource config */ Status -Config::SetResourceConfigMode(const std::string& value) { - Status s = CheckResourceConfigMode(value); +Config::SetGpuResourceConfigEnable(const std::string& value) { + Status s = CheckGpuResourceConfigEnable(value); if (!s.ok()) { return s; } - - SetConfigValueInMem(CONFIG_RESOURCE, CONFIG_RESOURCE_MODE, value); + SetConfigValueInMem(CONFIG_GPU_RESOURCE, CONFIG_GPU_RESOURCE_ENABLE, value); return Status::OK(); } Status -Config::SetResourceConfigSearchResources(const std::string& value) { +Config::SetGpuResourceConfigCacheCapacity(const std::string& value) { + Status s = CheckGpuResourceConfigCacheCapacity(value); + if (!s.ok()) { + return s; + } + SetConfigValueInMem(CONFIG_GPU_RESOURCE, CONFIG_GPU_RESOURCE_CACHE_CAPACITY, value); + return Status::OK(); +} + +Status +Config::SetGpuResourceConfigCacheThreshold(const std::string& value) { + Status s = CheckGpuResourceConfigCacheThreshold(value); + if (!s.ok()) { + return s; + } + SetConfigValueInMem(CONFIG_GPU_RESOURCE, CONFIG_GPU_RESOURCE_CACHE_THRESHOLD, value); + return Status::OK(); +} + +Status +Config::SetGpuResourceConfigSearchResources(const std::string& value) { std::vector res_vec; - server::StringHelpFunctions::SplitStringByDelimeter(value, CONFIG_RESOURCE_SEARCH_RESOURCES_DELIMITER, res_vec); - - Status s = CheckResourceConfigSearchResources(res_vec); + server::StringHelpFunctions::SplitStringByDelimeter(value, CONFIG_GPU_RESOURCE_DELIMITER, res_vec); + Status s = CheckGpuResourceConfigSearchResources(res_vec); if (!s.ok()) { return s; } - - SetConfigValueInMem(CONFIG_RESOURCE, CONFIG_RESOURCE_SEARCH_RESOURCES, value); + SetConfigValueInMem(CONFIG_GPU_RESOURCE, CONFIG_GPU_RESOURCE_SEARCH_RESOURCES, value); return Status::OK(); } Status -Config::SetResourceConfigIndexBuildDevice(const std::string& value) { - Status s = CheckResourceConfigIndexBuildDevice(value); +Config::SetGpuResourceConfigBuildIndexResources(const std::string& value) { + std::vector res_vec; + server::StringHelpFunctions::SplitStringByDelimeter(value, CONFIG_GPU_RESOURCE_DELIMITER, res_vec); + Status s = CheckGpuResourceConfigBuildIndexResources(res_vec); if (!s.ok()) { return s; } - - SetConfigValueInMem(CONFIG_RESOURCE, CONFIG_RESOURCE_INDEX_BUILD_DEVICE, value); + SetConfigValueInMem(CONFIG_GPU_RESOURCE, CONFIG_GPU_RESOURCE_BUILD_INDEX_RESOURCES, value); return Status::OK(); -} +} // namespace server } // namespace server } // namespace milvus diff --git a/core/src/server/Config.h b/core/src/server/Config.h index e466b70a3b..0907080a6f 100644 --- a/core/src/server/Config.h +++ b/core/src/server/Config.h @@ -59,12 +59,8 @@ static const char* CONFIG_DB_PRELOAD_TABLE = "preload_table"; static const char* CONFIG_CACHE = "cache_config"; static const char* CONFIG_CACHE_CPU_CACHE_CAPACITY = "cpu_cache_capacity"; static const char* CONFIG_CACHE_CPU_CACHE_CAPACITY_DEFAULT = "16"; -static const char* CONFIG_CACHE_GPU_CACHE_CAPACITY = "gpu_cache_capacity"; -static const char* CONFIG_CACHE_GPU_CACHE_CAPACITY_DEFAULT = "4"; -static const char* CONFIG_CACHE_CPU_CACHE_THRESHOLD = "cpu_mem_threshold"; +static const char* CONFIG_CACHE_CPU_CACHE_THRESHOLD = "cpu_cache_threshold"; static const char* CONFIG_CACHE_CPU_CACHE_THRESHOLD_DEFAULT = "0.85"; -static const char* CONFIG_CACHE_GPU_CACHE_THRESHOLD = "gpu_mem_threshold"; -static const char* CONFIG_CACHE_GPU_CACHE_THRESHOLD_DEFAULT = "0.85"; static const char* CONFIG_CACHE_CACHE_INSERT_DATA = "cache_insert_data"; static const char* CONFIG_CACHE_CACHE_INSERT_DATA_DEFAULT = "false"; @@ -87,17 +83,23 @@ static const char* CONFIG_ENGINE_OMP_THREAD_NUM_DEFAULT = "0"; static const char* CONFIG_ENGINE_GPU_SEARCH_THRESHOLD = "gpu_search_threshold"; static const char* CONFIG_ENGINE_GPU_SEARCH_THRESHOLD_DEFAULT = "1000"; -/* resource config */ -static const char* CONFIG_RESOURCE = "resource_config"; -static const char* CONFIG_RESOURCE_MODE = "mode"; -static const char* CONFIG_RESOURCE_MODE_DEFAULT = "simple"; -static const char* CONFIG_RESOURCE_SEARCH_RESOURCES = "search_resources"; -static const char* CONFIG_RESOURCE_SEARCH_RESOURCES_DELIMITER = ","; -static const char* CONFIG_RESOURCE_SEARCH_RESOURCES_DEFAULT = "cpu,gpu0"; -static const char* CONFIG_RESOURCE_INDEX_BUILD_DEVICE = "index_build_device"; -static const char* CONFIG_RESOURCE_INDEX_BUILD_DEVICE_DEFAULT = "gpu0"; - -const int32_t CPU_DEVICE_ID = -1; +/* gpu resource config */ +static const char* CONFIG_GPU_RESOURCE = "gpu_resource_config"; +static const char* CONFIG_GPU_RESOURCE_ENABLE = "enable"; +#ifdef MILVUS_GPU_VERSION +static const char* CONFIG_GPU_RESOURCE_ENABLE_DEFAULT = "true"; +#else +static const char* CONFIG_GPU_RESOURCE_ENABLE_DEFAULT = "false"; +#endif +static const char* CONFIG_GPU_RESOURCE_CACHE_CAPACITY = "cache_capacity"; +static const char* CONFIG_GPU_RESOURCE_CACHE_CAPACITY_DEFAULT = "4"; +static const char* CONFIG_GPU_RESOURCE_CACHE_THRESHOLD = "cache_threshold"; +static const char* CONFIG_GPU_RESOURCE_CACHE_THRESHOLD_DEFAULT = "0.85"; +static const char* CONFIG_GPU_RESOURCE_DELIMITER = ","; +static const char* CONFIG_GPU_RESOURCE_SEARCH_RESOURCES = "search_resources"; +static const char* CONFIG_GPU_RESOURCE_SEARCH_RESOURCES_DEFAULT = "gpu0"; +static const char* CONFIG_GPU_RESOURCE_BUILD_INDEX_RESOURCES = "build_index_resources"; +static const char* CONFIG_GPU_RESOURCE_BUILD_INDEX_RESOURCES_DEFAULT = "gpu0"; class Config { public: @@ -161,10 +163,6 @@ class Config { Status CheckCacheConfigCpuCacheThreshold(const std::string& value); Status - CheckCacheConfigGpuCacheCapacity(const std::string& value); - Status - CheckCacheConfigGpuCacheThreshold(const std::string& value); - Status CheckCacheConfigCacheInsertData(const std::string& value); /* engine config */ @@ -175,13 +173,17 @@ class Config { Status CheckEngineConfigGpuSearchThreshold(const std::string& value); - /* resource config */ + /* gpu resource config */ Status - CheckResourceConfigMode(const std::string& value); + CheckGpuResourceConfigEnable(const std::string& value); Status - CheckResourceConfigSearchResources(const std::vector& value); + CheckGpuResourceConfigCacheCapacity(const std::string& value); Status - CheckResourceConfigIndexBuildDevice(const std::string& value); + CheckGpuResourceConfigCacheThreshold(const std::string& value); + Status + CheckGpuResourceConfigSearchResources(const std::vector& value); + Status + CheckGpuResourceConfigBuildIndexResources(const std::vector& value); std::string GetConfigStr(const std::string& parent_key, const std::string& child_key, const std::string& default_value = ""); @@ -208,11 +210,11 @@ class Config { Status GetDBConfigBackendUrl(std::string& value); Status - GetDBConfigArchiveDiskThreshold(int32_t& value); + GetDBConfigArchiveDiskThreshold(int64_t& value); Status - GetDBConfigArchiveDaysThreshold(int32_t& value); + GetDBConfigArchiveDaysThreshold(int64_t& value); Status - GetDBConfigInsertBufferSize(int32_t& value); + GetDBConfigInsertBufferSize(int64_t& value); Status GetDBConfigPreloadTable(std::string& value); @@ -230,27 +232,27 @@ class Config { Status GetCacheConfigCpuCacheThreshold(float& value); Status - GetCacheConfigGpuCacheCapacity(int64_t& value); - Status - GetCacheConfigGpuCacheThreshold(float& value); - Status GetCacheConfigCacheInsertData(bool& value); /* engine config */ Status - GetEngineConfigUseBlasThreshold(int32_t& value); + GetEngineConfigUseBlasThreshold(int64_t& value); Status - GetEngineConfigOmpThreadNum(int32_t& value); + GetEngineConfigOmpThreadNum(int64_t& value); Status - GetEngineConfigGpuSearchThreshold(int32_t& value); + GetEngineConfigGpuSearchThreshold(int64_t& value); - /* resource config */ + /* gpu resource config */ Status - GetResourceConfigMode(std::string& value); + GetGpuResourceConfigEnable(bool& value); Status - GetResourceConfigSearchResources(std::vector& value); + GetGpuResourceConfigCacheCapacity(int64_t& value); Status - GetResourceConfigIndexBuildDevice(int32_t& value); + GetGpuResourceConfigCacheThreshold(float& value); + Status + GetGpuResourceConfigSearchResources(std::vector& value); + Status + GetGpuResourceConfigBuildIndexResources(std::vector& value); public: /* server config */ @@ -291,10 +293,6 @@ class Config { Status SetCacheConfigCpuCacheThreshold(const std::string& value); Status - SetCacheConfigGpuCacheCapacity(const std::string& value); - Status - SetCacheConfigGpuCacheThreshold(const std::string& value); - Status SetCacheConfigCacheInsertData(const std::string& value); /* engine config */ @@ -305,13 +303,17 @@ class Config { Status SetEngineConfigGpuSearchThreshold(const std::string& value); - /* resource config */ + /* gpu resource config */ Status - SetResourceConfigMode(const std::string& value); + SetGpuResourceConfigEnable(const std::string& value); Status - SetResourceConfigSearchResources(const std::string& value); + SetGpuResourceConfigCacheCapacity(const std::string& value); Status - SetResourceConfigIndexBuildDevice(const std::string& value); + SetGpuResourceConfigCacheThreshold(const std::string& value); + Status + SetGpuResourceConfigSearchResources(const std::string& value); + Status + SetGpuResourceConfigBuildIndexResources(const std::string& value); private: std::unordered_map> config_map_; diff --git a/core/src/server/DBWrapper.cpp b/core/src/server/DBWrapper.cpp index 1ff914dc69..f7a43d671c 100644 --- a/core/src/server/DBWrapper.cpp +++ b/core/src/server/DBWrapper.cpp @@ -89,7 +89,7 @@ DBWrapper::StartService() { } // engine config - int32_t omp_thread; + int64_t omp_thread; s = config.GetEngineConfigOmpThreadNum(omp_thread); if (!s.ok()) { std::cerr << s.ToString() << std::endl; @@ -100,7 +100,7 @@ DBWrapper::StartService() { omp_set_num_threads(omp_thread); SERVER_LOG_DEBUG << "Specify openmp thread number: " << omp_thread; } else { - uint32_t sys_thread_cnt = 8; + int64_t sys_thread_cnt = 8; if (CommonUtil::GetSystemAvailableThreads(sys_thread_cnt)) { omp_thread = static_cast(ceil(sys_thread_cnt * 0.5)); omp_set_num_threads(omp_thread); @@ -108,7 +108,7 @@ DBWrapper::StartService() { } // init faiss global variable - int32_t use_blas_threshold; + int64_t use_blas_threshold; s = config.GetEngineConfigUseBlasThreshold(use_blas_threshold); if (!s.ok()) { std::cerr << s.ToString() << std::endl; @@ -119,7 +119,7 @@ DBWrapper::StartService() { // set archive config engine::ArchiveConf::CriteriaT criterial; - int32_t disk, days; + int64_t disk, days; s = config.GetDBConfigArchiveDiskThreshold(disk); if (!s.ok()) { std::cerr << s.ToString() << std::endl; diff --git a/core/src/server/grpc_impl/GrpcRequestHandler.cpp b/core/src/server/grpc_impl/GrpcRequestHandler.cpp index f78aea204c..014a0e2180 100644 --- a/core/src/server/grpc_impl/GrpcRequestHandler.cpp +++ b/core/src/server/grpc_impl/GrpcRequestHandler.cpp @@ -16,7 +16,24 @@ // under the License. #include "server/grpc_impl/GrpcRequestHandler.h" -#include "server/grpc_impl/GrpcRequestTask.h" +#include "server/grpc_impl/GrpcRequestScheduler.h" +#include "server/grpc_impl/request/CmdRequest.h" +#include "server/grpc_impl/request/CountTableRequest.h" +#include "server/grpc_impl/request/CreateIndexRequest.h" +#include "server/grpc_impl/request/CreatePartitionRequest.h" +#include "server/grpc_impl/request/CreateTableRequest.h" +#include "server/grpc_impl/request/DeleteByDateRequest.h" +#include "server/grpc_impl/request/DescribeIndexRequest.h" +#include "server/grpc_impl/request/DescribeTableRequest.h" +#include "server/grpc_impl/request/DropIndexRequest.h" +#include "server/grpc_impl/request/DropPartitionRequest.h" +#include "server/grpc_impl/request/DropTableRequest.h" +#include "server/grpc_impl/request/HasTableRequest.h" +#include "server/grpc_impl/request/InsertRequest.h" +#include "server/grpc_impl/request/PreloadTableRequest.h" +#include "server/grpc_impl/request/SearchRequest.h" +#include "server/grpc_impl/request/ShowPartitionsRequest.h" +#include "server/grpc_impl/request/ShowTablesRequest.h" #include "utils/TimeRecorder.h" #include @@ -28,8 +45,8 @@ namespace grpc { ::grpc::Status GrpcRequestHandler::CreateTable(::grpc::ServerContext* context, const ::milvus::grpc::TableSchema* request, ::milvus::grpc::Status* response) { - BaseTaskPtr task_ptr = CreateTableTask::Create(request); - GrpcRequestScheduler::ExecTask(task_ptr, response); + BaseRequestPtr request_ptr = CreateTableRequest::Create(request); + GrpcRequestScheduler::ExecRequest(request_ptr, response); return ::grpc::Status::OK; } @@ -37,9 +54,9 @@ GrpcRequestHandler::CreateTable(::grpc::ServerContext* context, const ::milvus:: GrpcRequestHandler::HasTable(::grpc::ServerContext* context, const ::milvus::grpc::TableName* request, ::milvus::grpc::BoolReply* response) { bool has_table = false; - BaseTaskPtr task_ptr = HasTableTask::Create(request->table_name(), has_table); + BaseRequestPtr request_ptr = HasTableRequest::Create(request->table_name(), has_table); ::milvus::grpc::Status grpc_status; - GrpcRequestScheduler::ExecTask(task_ptr, &grpc_status); + GrpcRequestScheduler::ExecRequest(request_ptr, &grpc_status); response->set_bool_reply(has_table); response->mutable_status()->set_reason(grpc_status.reason()); response->mutable_status()->set_error_code(grpc_status.error_code()); @@ -49,25 +66,25 @@ GrpcRequestHandler::HasTable(::grpc::ServerContext* context, const ::milvus::grp ::grpc::Status GrpcRequestHandler::DropTable(::grpc::ServerContext* context, const ::milvus::grpc::TableName* request, ::milvus::grpc::Status* response) { - BaseTaskPtr task_ptr = DropTableTask::Create(request->table_name()); - GrpcRequestScheduler::ExecTask(task_ptr, response); + BaseRequestPtr request_ptr = DropTableRequest::Create(request->table_name()); + GrpcRequestScheduler::ExecRequest(request_ptr, response); return ::grpc::Status::OK; } ::grpc::Status GrpcRequestHandler::CreateIndex(::grpc::ServerContext* context, const ::milvus::grpc::IndexParam* request, ::milvus::grpc::Status* response) { - BaseTaskPtr task_ptr = CreateIndexTask::Create(request); - GrpcRequestScheduler::ExecTask(task_ptr, response); + BaseRequestPtr request_ptr = CreateIndexRequest::Create(request); + GrpcRequestScheduler::ExecRequest(request_ptr, response); return ::grpc::Status::OK; } ::grpc::Status GrpcRequestHandler::Insert(::grpc::ServerContext* context, const ::milvus::grpc::InsertParam* request, ::milvus::grpc::VectorIds* response) { - BaseTaskPtr task_ptr = InsertTask::Create(request, response); + BaseRequestPtr request_ptr = InsertRequest::Create(request, response); ::milvus::grpc::Status grpc_status; - GrpcRequestScheduler::ExecTask(task_ptr, &grpc_status); + GrpcRequestScheduler::ExecRequest(request_ptr, &grpc_status); response->mutable_status()->set_reason(grpc_status.reason()); response->mutable_status()->set_error_code(grpc_status.error_code()); return ::grpc::Status::OK; @@ -77,9 +94,9 @@ GrpcRequestHandler::Insert(::grpc::ServerContext* context, const ::milvus::grpc: GrpcRequestHandler::Search(::grpc::ServerContext* context, const ::milvus::grpc::SearchParam* request, ::milvus::grpc::TopKQueryResult* response) { std::vector file_id_array; - BaseTaskPtr task_ptr = SearchTask::Create(request, file_id_array, response); + BaseRequestPtr request_ptr = SearchRequest::Create(request, file_id_array, response); ::milvus::grpc::Status grpc_status; - GrpcRequestScheduler::ExecTask(task_ptr, &grpc_status); + GrpcRequestScheduler::ExecRequest(request_ptr, &grpc_status); response->mutable_status()->set_error_code(grpc_status.error_code()); response->mutable_status()->set_reason(grpc_status.reason()); return ::grpc::Status::OK; @@ -93,9 +110,10 @@ GrpcRequestHandler::SearchInFiles(::grpc::ServerContext* context, const ::milvus file_id_array.push_back(request->file_id_array(i)); } ::milvus::grpc::SearchInFilesParam* request_mutable = const_cast<::milvus::grpc::SearchInFilesParam*>(request); - BaseTaskPtr task_ptr = SearchTask::Create(request_mutable->mutable_search_param(), file_id_array, response); + BaseRequestPtr request_ptr = + SearchRequest::Create(request_mutable->mutable_search_param(), file_id_array, response); ::milvus::grpc::Status grpc_status; - GrpcRequestScheduler::ExecTask(task_ptr, &grpc_status); + GrpcRequestScheduler::ExecRequest(request_ptr, &grpc_status); response->mutable_status()->set_error_code(grpc_status.error_code()); response->mutable_status()->set_reason(grpc_status.reason()); return ::grpc::Status::OK; @@ -104,9 +122,9 @@ GrpcRequestHandler::SearchInFiles(::grpc::ServerContext* context, const ::milvus ::grpc::Status GrpcRequestHandler::DescribeTable(::grpc::ServerContext* context, const ::milvus::grpc::TableName* request, ::milvus::grpc::TableSchema* response) { - BaseTaskPtr task_ptr = DescribeTableTask::Create(request->table_name(), response); + BaseRequestPtr request_ptr = DescribeTableRequest::Create(request->table_name(), response); ::milvus::grpc::Status grpc_status; - GrpcRequestScheduler::ExecTask(task_ptr, &grpc_status); + GrpcRequestScheduler::ExecRequest(request_ptr, &grpc_status); response->mutable_status()->set_error_code(grpc_status.error_code()); response->mutable_status()->set_reason(grpc_status.reason()); return ::grpc::Status::OK; @@ -116,9 +134,9 @@ GrpcRequestHandler::DescribeTable(::grpc::ServerContext* context, const ::milvus GrpcRequestHandler::CountTable(::grpc::ServerContext* context, const ::milvus::grpc::TableName* request, ::milvus::grpc::TableRowCount* response) { int64_t row_count = 0; - BaseTaskPtr task_ptr = CountTableTask::Create(request->table_name(), row_count); + BaseRequestPtr request_ptr = CountTableRequest::Create(request->table_name(), row_count); ::milvus::grpc::Status grpc_status; - GrpcRequestScheduler::ExecTask(task_ptr, &grpc_status); + GrpcRequestScheduler::ExecRequest(request_ptr, &grpc_status); response->set_table_row_count(row_count); response->mutable_status()->set_reason(grpc_status.reason()); response->mutable_status()->set_error_code(grpc_status.error_code()); @@ -128,9 +146,9 @@ GrpcRequestHandler::CountTable(::grpc::ServerContext* context, const ::milvus::g ::grpc::Status GrpcRequestHandler::ShowTables(::grpc::ServerContext* context, const ::milvus::grpc::Command* request, ::milvus::grpc::TableNameList* response) { - BaseTaskPtr task_ptr = ShowTablesTask::Create(response); + BaseRequestPtr request_ptr = ShowTablesRequest::Create(response); ::milvus::grpc::Status grpc_status; - GrpcRequestScheduler::ExecTask(task_ptr, &grpc_status); + GrpcRequestScheduler::ExecRequest(request_ptr, &grpc_status); response->mutable_status()->set_error_code(grpc_status.error_code()); response->mutable_status()->set_reason(grpc_status.reason()); return ::grpc::Status::OK; @@ -140,9 +158,9 @@ GrpcRequestHandler::ShowTables(::grpc::ServerContext* context, const ::milvus::g GrpcRequestHandler::Cmd(::grpc::ServerContext* context, const ::milvus::grpc::Command* request, ::milvus::grpc::StringReply* response) { std::string result; - BaseTaskPtr task_ptr = CmdTask::Create(request->cmd(), result); + BaseRequestPtr request_ptr = CmdRequest::Create(request->cmd(), result); ::milvus::grpc::Status grpc_status; - GrpcRequestScheduler::ExecTask(task_ptr, &grpc_status); + GrpcRequestScheduler::ExecRequest(request_ptr, &grpc_status); response->set_string_reply(result); response->mutable_status()->set_reason(grpc_status.reason()); response->mutable_status()->set_error_code(grpc_status.error_code()); @@ -150,11 +168,11 @@ GrpcRequestHandler::Cmd(::grpc::ServerContext* context, const ::milvus::grpc::Co } ::grpc::Status -GrpcRequestHandler::DeleteByRange(::grpc::ServerContext* context, const ::milvus::grpc::DeleteByRangeParam* request, - ::milvus::grpc::Status* response) { - BaseTaskPtr task_ptr = DeleteByRangeTask::Create(request); +GrpcRequestHandler::DeleteByDate(::grpc::ServerContext* context, const ::milvus::grpc::DeleteByDateParam* request, + ::milvus::grpc::Status* response) { + BaseRequestPtr request_ptr = DeleteByDateRequest::Create(request); ::milvus::grpc::Status grpc_status; - GrpcRequestScheduler::ExecTask(task_ptr, &grpc_status); + GrpcRequestScheduler::ExecRequest(request_ptr, &grpc_status); response->set_error_code(grpc_status.error_code()); response->set_reason(grpc_status.reason()); return ::grpc::Status::OK; @@ -163,9 +181,9 @@ GrpcRequestHandler::DeleteByRange(::grpc::ServerContext* context, const ::milvus ::grpc::Status GrpcRequestHandler::PreloadTable(::grpc::ServerContext* context, const ::milvus::grpc::TableName* request, ::milvus::grpc::Status* response) { - BaseTaskPtr task_ptr = PreloadTableTask::Create(request->table_name()); + BaseRequestPtr request_ptr = PreloadTableRequest::Create(request->table_name()); ::milvus::grpc::Status grpc_status; - GrpcRequestScheduler::ExecTask(task_ptr, &grpc_status); + GrpcRequestScheduler::ExecRequest(request_ptr, &grpc_status); response->set_reason(grpc_status.reason()); response->set_error_code(grpc_status.error_code()); return ::grpc::Status::OK; @@ -174,9 +192,9 @@ GrpcRequestHandler::PreloadTable(::grpc::ServerContext* context, const ::milvus: ::grpc::Status GrpcRequestHandler::DescribeIndex(::grpc::ServerContext* context, const ::milvus::grpc::TableName* request, ::milvus::grpc::IndexParam* response) { - BaseTaskPtr task_ptr = DescribeIndexTask::Create(request->table_name(), response); + BaseRequestPtr request_ptr = DescribeIndexRequest::Create(request->table_name(), response); ::milvus::grpc::Status grpc_status; - GrpcRequestScheduler::ExecTask(task_ptr, &grpc_status); + GrpcRequestScheduler::ExecRequest(request_ptr, &grpc_status); response->mutable_status()->set_reason(grpc_status.reason()); response->mutable_status()->set_error_code(grpc_status.error_code()); return ::grpc::Status::OK; @@ -185,9 +203,39 @@ GrpcRequestHandler::DescribeIndex(::grpc::ServerContext* context, const ::milvus ::grpc::Status GrpcRequestHandler::DropIndex(::grpc::ServerContext* context, const ::milvus::grpc::TableName* request, ::milvus::grpc::Status* response) { - BaseTaskPtr task_ptr = DropIndexTask::Create(request->table_name()); + BaseRequestPtr request_ptr = DropIndexRequest::Create(request->table_name()); ::milvus::grpc::Status grpc_status; - GrpcRequestScheduler::ExecTask(task_ptr, &grpc_status); + GrpcRequestScheduler::ExecRequest(request_ptr, &grpc_status); + response->set_reason(grpc_status.reason()); + response->set_error_code(grpc_status.error_code()); + return ::grpc::Status::OK; +} + +::grpc::Status +GrpcRequestHandler::CreatePartition(::grpc::ServerContext* context, const ::milvus::grpc::PartitionParam* request, + ::milvus::grpc::Status* response) { + BaseRequestPtr request_ptr = CreatePartitionRequest::Create(request); + GrpcRequestScheduler::ExecRequest(request_ptr, response); + return ::grpc::Status::OK; +} + +::grpc::Status +GrpcRequestHandler::ShowPartitions(::grpc::ServerContext* context, const ::milvus::grpc::TableName* request, + ::milvus::grpc::PartitionList* response) { + BaseRequestPtr request_ptr = ShowPartitionsRequest::Create(request->table_name(), response); + ::milvus::grpc::Status grpc_status; + GrpcRequestScheduler::ExecRequest(request_ptr, &grpc_status); + response->mutable_status()->set_reason(grpc_status.reason()); + response->mutable_status()->set_error_code(grpc_status.error_code()); + return ::grpc::Status::OK; +} + +::grpc::Status +GrpcRequestHandler::DropPartition(::grpc::ServerContext* context, const ::milvus::grpc::PartitionParam* request, + ::milvus::grpc::Status* response) { + BaseRequestPtr request_ptr = DropPartitionRequest::Create(request); + ::milvus::grpc::Status grpc_status; + GrpcRequestScheduler::ExecRequest(request_ptr, &grpc_status); response->set_reason(grpc_status.reason()); response->set_error_code(grpc_status.error_code()); return ::grpc::Status::OK; diff --git a/core/src/server/grpc_impl/GrpcRequestHandler.h b/core/src/server/grpc_impl/GrpcRequestHandler.h index 935600616c..7ba40af0de 100644 --- a/core/src/server/grpc_impl/GrpcRequestHandler.h +++ b/core/src/server/grpc_impl/GrpcRequestHandler.h @@ -28,296 +28,170 @@ namespace server { namespace grpc { class GrpcRequestHandler final : public ::milvus::grpc::MilvusService::Service { public: - /** - * @brief Create table method - * - * This method is used to create table - * - * @param context, add context for every RPC - * @param request, used to provide table information to be created. - * @param response, used to get the status - * - * @return status - * - * @param request - * @param response - * @param context - */ + // * + // @brief This method is used to create table + // + // @param TableSchema, use to provide table information to be created. + // + // @return Status ::grpc::Status CreateTable(::grpc::ServerContext* context, const ::milvus::grpc::TableSchema* request, ::milvus::grpc::Status* response) override; - - /** - * @brief Test table existence method - * - * This method is used to test table existence. - * - * @param context, add context for every RPC - * @param request, table name is going to be tested. - * @param response, get the bool reply of hastable - * - * @return status - * - * @param request - * @param response - * @param context - */ + // * + // @brief This method is used to test table existence. + // + // @param TableName, table name is going to be tested. + // + // @return BoolReply ::grpc::Status HasTable(::grpc::ServerContext* context, const ::milvus::grpc::TableName* request, ::milvus::grpc::BoolReply* response) override; - - /** - * @brief Drop table method - * - * This method is used to drop table. - * - * @param context, add context for every RPC - * @param request, table name is going to be deleted. - * @param response, get the status of droptable - * - * @return status - * - * @param request - * @param response - * @param context - */ + // * + // @brief This method is used to get table schema. + // + // @param TableName, target table name. + // + // @return TableSchema + ::grpc::Status + DescribeTable(::grpc::ServerContext* context, const ::milvus::grpc::TableName* request, + ::milvus::grpc::TableSchema* response) override; + // * + // @brief This method is used to get table schema. + // + // @param TableName, target table name. + // + // @return TableRowCount + ::grpc::Status + CountTable(::grpc::ServerContext* context, const ::milvus::grpc::TableName* request, + ::milvus::grpc::TableRowCount* response) override; + // * + // @brief This method is used to list all tables. + // + // @param Command, dummy parameter. + // + // @return TableNameList + ::grpc::Status + ShowTables(::grpc::ServerContext* context, const ::milvus::grpc::Command* request, + ::milvus::grpc::TableNameList* response) override; + // * + // @brief This method is used to delete table. + // + // @param TableName, table name is going to be deleted. + // + // @return TableNameList ::grpc::Status DropTable(::grpc::ServerContext* context, const ::milvus::grpc::TableName* request, ::milvus::grpc::Status* response) override; - - /** - * @brief build index by table method - * - * This method is used to build index by table in sync. - * - * @param context, add context for every RPC - * @param request, table name is going to be built index. - * @param response, get the status of buildindex - * - * @return status - * - * @param request - * @param response - * @param context - */ + // * + // @brief This method is used to build index by table in sync mode. + // + // @param IndexParam, index paramters. + // + // @return Status ::grpc::Status CreateIndex(::grpc::ServerContext* context, const ::milvus::grpc::IndexParam* request, ::milvus::grpc::Status* response) override; - - /** - * @brief Insert vector array to table - * - * This method is used to insert vector array to table. - * - * @param context, add context for every RPC - * @param request, table_name is inserted. - * @param response, vector array is inserted. - * - * @return status - * - * @param context - * @param request - * @param response - */ + // * + // @brief This method is used to describe index + // + // @param TableName, target table name. + // + // @return IndexParam + ::grpc::Status + DescribeIndex(::grpc::ServerContext* context, const ::milvus::grpc::TableName* request, + ::milvus::grpc::IndexParam* response) override; + // * + // @brief This method is used to drop index + // + // @param TableName, target table name. + // + // @return Status + ::grpc::Status + DropIndex(::grpc::ServerContext* context, const ::milvus::grpc::TableName* request, + ::milvus::grpc::Status* response) override; + // * + // @brief This method is used to create partition + // + // @param PartitionParam, partition parameters. + // + // @return Status + ::grpc::Status + CreatePartition(::grpc::ServerContext* context, const ::milvus::grpc::PartitionParam* request, + ::milvus::grpc::Status* response) override; + // * + // @brief This method is used to show partition information + // + // @param TableName, target table name. + // + // @return PartitionList + ::grpc::Status + ShowPartitions(::grpc::ServerContext* context, const ::milvus::grpc::TableName* request, + ::milvus::grpc::PartitionList* response) override; + // * + // @brief This method is used to drop partition + // + // @param PartitionName, target partition name. + // + // @return Status + ::grpc::Status + DropPartition(::grpc::ServerContext* context, const ::milvus::grpc::PartitionParam* request, + ::milvus::grpc::Status* response) override; + // * + // @brief This method is used to add vector array to table. + // + // @param InsertParam, insert parameters. + // + // @return VectorIds ::grpc::Status Insert(::grpc::ServerContext* context, const ::milvus::grpc::InsertParam* request, ::milvus::grpc::VectorIds* response) override; - - /** - * @brief Query vector - * - * This method is used to query vector in table. - * - * @param context, add context for every RPC - * @param request: - * table_name, table_name is queried. - * query_record_array, all vector are going to be queried. - * query_range_array, optional ranges for conditional search. If not specified, search whole table - * topk, how many similarity vectors will be searched. - * - * @param writer, write query result array. - * - * @return status - * - * @param context - * @param request - * @param writer - */ + // * + // @brief This method is used to query vector in table. + // + // @param SearchParam, search parameters. + // + // @return TopKQueryResultList ::grpc::Status Search(::grpc::ServerContext* context, const ::milvus::grpc::SearchParam* request, ::milvus::grpc::TopKQueryResult* response) override; - /** - * @brief Internal use query interface - * - * This method is used to query vector in specified files. - * - * @param context, add context for every RPC - * @param request: - * file_id_array, specified files id array, queried. - * query_record_array, all vector are going to be queried. - * query_range_array, optional ranges for conditional search. If not specified, search whole table - * topk, how many similarity vectors will be searched. - * - * @param writer, write query result array. - * - * @return status - * - * @param context - * @param request - * @param writer - */ + // * + // @brief This method is used to query vector in specified files. + // + // @param SearchInFilesParam, search in files paremeters. + // + // @return TopKQueryResultList ::grpc::Status SearchInFiles(::grpc::ServerContext* context, const ::milvus::grpc::SearchInFilesParam* request, ::milvus::grpc::TopKQueryResult* response) override; - /** - * @brief Get table schema - * - * This method is used to get table schema. - * - * @param context, add context for every RPC - * @param request, target table name. - * @param response, table schema - * - * @return status - * - * @param context - * @param request - * @param response - */ - ::grpc::Status - DescribeTable(::grpc::ServerContext* context, const ::milvus::grpc::TableName* request, - ::milvus::grpc::TableSchema* response) override; - - /** - * @brief Get table row count - * - * This method is used to get table row count. - * - * @param context, add context for every RPC - * @param request, target table name. - * @param response, table row count - * - * @return table row count - * - * @param request - * @param response - * @param context - */ - ::grpc::Status - CountTable(::grpc::ServerContext* context, const ::milvus::grpc::TableName* request, - ::milvus::grpc::TableRowCount* response) override; - - /** - * @brief List all tables in database - * - * This method is used to list all tables. - * - * @param context, add context for every RPC - * @param request, show table command, usually not use - * @param writer, write tables to client - * - * @return status - * - * @param context - * @param request - * @param writer - */ - ::grpc::Status - ShowTables(::grpc::ServerContext* context, const ::milvus::grpc::Command* request, - ::milvus::grpc::TableNameList* response) override; - - /** - * @brief Give the server status - * - * - * This method is used to give the server status. - * @param context, add context for every RPC - * @param request, give server command - * @param response, server status - * - * @return status - * - * @param context - * @param request - * @param response - */ + // * + // @brief This method is used to give the server status. + // + // @param Command, command string + // + // @return StringReply ::grpc::Status Cmd(::grpc::ServerContext* context, const ::milvus::grpc::Command* request, ::milvus::grpc::StringReply* response) override; - - /** - * @brief delete table by range - * - * This method is used to delete table by range. - * @param context, add context for every RPC - * @param request, table name and range - * @param response, status - * - * @return status - * - * @param context - * @param request - * @param response - */ + // * + // @brief This method is used to delete vector by date range + // + // @param DeleteByDateParam, delete parameters. + // + // @return status ::grpc::Status - DeleteByRange(::grpc::ServerContext* context, const ::milvus::grpc::DeleteByRangeParam* request, - ::milvus::grpc::Status* response) override; - - /** - * @brief preload table - * - * This method is used to preload table. - * @param context, add context for every RPC - * @param request, table name - * @param response, status - * - * @return status - * - * @param context - * @param request - * @param response - */ + DeleteByDate(::grpc::ServerContext* context, const ::milvus::grpc::DeleteByDateParam* request, + ::milvus::grpc::Status* response) override; + // * + // @brief This method is used to preload table + // + // @param TableName, target table name. + // + // @return Status ::grpc::Status PreloadTable(::grpc::ServerContext* context, const ::milvus::grpc::TableName* request, ::milvus::grpc::Status* response) override; - - /** - * @brief Describe index - * - * This method is used to describe index. - * @param context, add context for every RPC - * @param request, table name - * @param response, index informations - * - * @return status - * - * @param context - * @param request - * @param response - */ - ::grpc::Status - DescribeIndex(::grpc::ServerContext* context, const ::milvus::grpc::TableName* request, - ::milvus::grpc::IndexParam* response) override; - - /** - * @brief Drop index - * - * This method is used to drop index. - * @param context, add context for every RPC - * @param request, table name - * @param response, status - * - * @return status - * - * @param context - * @param request - * @param response - */ - ::grpc::Status - DropIndex(::grpc::ServerContext* context, const ::milvus::grpc::TableName* request, - ::milvus::grpc::Status* response) override; }; } // namespace grpc diff --git a/core/src/server/grpc_impl/GrpcRequestScheduler.cpp b/core/src/server/grpc_impl/GrpcRequestScheduler.cpp index ac35f82947..9755678ddc 100644 --- a/core/src/server/grpc_impl/GrpcRequestScheduler.cpp +++ b/core/src/server/grpc_impl/GrpcRequestScheduler.cpp @@ -70,43 +70,6 @@ ErrorMap(ErrorCode code) { } } // namespace -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -GrpcBaseTask::GrpcBaseTask(const std::string& task_group, bool async) - : task_group_(task_group), async_(async), done_(false) { -} - -GrpcBaseTask::~GrpcBaseTask() { - WaitToFinish(); -} - -Status -GrpcBaseTask::Execute() { - status_ = OnExecute(); - Done(); - return status_; -} - -void -GrpcBaseTask::Done() { - done_ = true; - finish_cond_.notify_all(); -} - -Status -GrpcBaseTask::SetStatus(ErrorCode error_code, const std::string& error_msg) { - status_ = Status(error_code, error_msg); - SERVER_LOG_ERROR << error_msg; - return status_; -} - -Status -GrpcBaseTask::WaitToFinish() { - std::unique_lock lock(finish_mtx_); - finish_cond_.wait(lock, [this] { return done_; }); - - return status_; -} - //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// GrpcRequestScheduler::GrpcRequestScheduler() : stopped_(false) { Start(); @@ -117,17 +80,17 @@ GrpcRequestScheduler::~GrpcRequestScheduler() { } void -GrpcRequestScheduler::ExecTask(BaseTaskPtr& task_ptr, ::milvus::grpc::Status* grpc_status) { - if (task_ptr == nullptr) { +GrpcRequestScheduler::ExecRequest(BaseRequestPtr& request_ptr, ::milvus::grpc::Status* grpc_status) { + if (request_ptr == nullptr) { return; } GrpcRequestScheduler& scheduler = GrpcRequestScheduler::GetInstance(); - scheduler.ExecuteTask(task_ptr); + scheduler.ExecuteRequest(request_ptr); - if (!task_ptr->IsAsync()) { - task_ptr->WaitToFinish(); - const Status& status = task_ptr->status(); + if (!request_ptr->IsAsync()) { + request_ptr->WaitToFinish(); + const Status& status = request_ptr->status(); if (!status.ok()) { grpc_status->set_reason(status.message()); grpc_status->set_error_code(ErrorMap(status.code())); @@ -153,7 +116,7 @@ GrpcRequestScheduler::Stop() { SERVER_LOG_INFO << "Scheduler gonna stop..."; { std::lock_guard lock(queue_mtx_); - for (auto iter : task_groups_) { + for (auto iter : request_groups_) { if (iter.second != nullptr) { iter.second->Put(nullptr); } @@ -171,64 +134,64 @@ GrpcRequestScheduler::Stop() { } Status -GrpcRequestScheduler::ExecuteTask(const BaseTaskPtr& task_ptr) { - if (task_ptr == nullptr) { +GrpcRequestScheduler::ExecuteRequest(const BaseRequestPtr& request_ptr) { + if (request_ptr == nullptr) { return Status::OK(); } - auto status = PutTaskToQueue(task_ptr); + auto status = PutToQueue(request_ptr); if (!status.ok()) { - SERVER_LOG_ERROR << "Put task to queue failed with code: " << status.ToString(); + SERVER_LOG_ERROR << "Put request to queue failed with code: " << status.ToString(); return status; } - if (task_ptr->IsAsync()) { + if (request_ptr->IsAsync()) { return Status::OK(); // async execution, caller need to call WaitToFinish at somewhere } - return task_ptr->WaitToFinish(); // sync execution + return request_ptr->WaitToFinish(); // sync execution } void -GrpcRequestScheduler::TakeTaskToExecute(TaskQueuePtr task_queue) { - if (task_queue == nullptr) { +GrpcRequestScheduler::TakeToExecute(RequestQueuePtr request_queue) { + if (request_queue == nullptr) { return; } while (true) { - BaseTaskPtr task = task_queue->Take(); - if (task == nullptr) { - SERVER_LOG_ERROR << "Take null from task queue, stop thread"; + BaseRequestPtr request = request_queue->Take(); + if (request == nullptr) { + SERVER_LOG_ERROR << "Take null from request queue, stop thread"; break; // stop the thread } try { - auto status = task->Execute(); + auto status = request->Execute(); if (!status.ok()) { - SERVER_LOG_ERROR << "Task failed with code: " << status.ToString(); + SERVER_LOG_ERROR << "Request failed with code: " << status.ToString(); } } catch (std::exception& ex) { - SERVER_LOG_ERROR << "Task failed to execute: " << ex.what(); + SERVER_LOG_ERROR << "Request failed to execute: " << ex.what(); } } } Status -GrpcRequestScheduler::PutTaskToQueue(const BaseTaskPtr& task_ptr) { +GrpcRequestScheduler::PutToQueue(const BaseRequestPtr& request_ptr) { std::lock_guard lock(queue_mtx_); - std::string group_name = task_ptr->TaskGroup(); - if (task_groups_.count(group_name) > 0) { - task_groups_[group_name]->Put(task_ptr); + std::string group_name = request_ptr->RequestGroup(); + if (request_groups_.count(group_name) > 0) { + request_groups_[group_name]->Put(request_ptr); } else { - TaskQueuePtr queue = std::make_shared(); - queue->Put(task_ptr); - task_groups_.insert(std::make_pair(group_name, queue)); + RequestQueuePtr queue = std::make_shared(); + queue->Put(request_ptr); + request_groups_.insert(std::make_pair(group_name, queue)); // start a thread - ThreadPtr thread = std::make_shared(&GrpcRequestScheduler::TakeTaskToExecute, this, queue); + ThreadPtr thread = std::make_shared(&GrpcRequestScheduler::TakeToExecute, this, queue); execute_threads_.push_back(thread); - SERVER_LOG_INFO << "Create new thread for task group: " << group_name; + SERVER_LOG_INFO << "Create new thread for request group: " << group_name; } return Status::OK(); diff --git a/core/src/server/grpc_impl/GrpcRequestScheduler.h b/core/src/server/grpc_impl/GrpcRequestScheduler.h index 802d247fb5..73f3a6e2c7 100644 --- a/core/src/server/grpc_impl/GrpcRequestScheduler.h +++ b/core/src/server/grpc_impl/GrpcRequestScheduler.h @@ -19,6 +19,7 @@ #include "grpc/gen-status/status.grpc.pb.h" #include "grpc/gen-status/status.pb.h" +#include "server/grpc_impl/request/GrpcBaseRequest.h" #include "utils/BlockingQueue.h" #include "utils/Status.h" @@ -32,57 +33,8 @@ namespace milvus { namespace server { namespace grpc { -class GrpcBaseTask { - protected: - explicit GrpcBaseTask(const std::string& task_group, bool async = false); - - virtual ~GrpcBaseTask(); - - public: - Status - Execute(); - - void - Done(); - - Status - WaitToFinish(); - - std::string - TaskGroup() const { - return task_group_; - } - - const Status& - status() const { - return status_; - } - - bool - IsAsync() const { - return async_; - } - - protected: - virtual Status - OnExecute() = 0; - - Status - SetStatus(ErrorCode error_code, const std::string& error_msg); - - protected: - mutable std::mutex finish_mtx_; - std::condition_variable finish_cond_; - - std::string task_group_; - bool async_; - bool done_; - Status status_; -}; - -using BaseTaskPtr = std::shared_ptr; -using TaskQueue = BlockingQueue; -using TaskQueuePtr = std::shared_ptr; +using RequestQueue = BlockingQueue; +using RequestQueuePtr = std::shared_ptr; using ThreadPtr = std::shared_ptr; class GrpcRequestScheduler { @@ -100,10 +52,10 @@ class GrpcRequestScheduler { Stop(); Status - ExecuteTask(const BaseTaskPtr& task_ptr); + ExecuteRequest(const BaseRequestPtr& request_ptr); static void - ExecTask(BaseTaskPtr& task_ptr, ::milvus::grpc::Status* grpc_status); + ExecRequest(BaseRequestPtr& request_ptr, ::milvus::grpc::Status* grpc_status); protected: GrpcRequestScheduler(); @@ -111,15 +63,15 @@ class GrpcRequestScheduler { virtual ~GrpcRequestScheduler(); void - TakeTaskToExecute(TaskQueuePtr task_queue); + TakeToExecute(RequestQueuePtr request_queue); Status - PutTaskToQueue(const BaseTaskPtr& task_ptr); + PutToQueue(const BaseRequestPtr& request_ptr); private: mutable std::mutex queue_mtx_; - std::map task_groups_; + std::map request_groups_; std::vector execute_threads_; diff --git a/core/src/server/grpc_impl/GrpcRequestTask.cpp b/core/src/server/grpc_impl/GrpcRequestTask.cpp deleted file mode 100644 index b81fab0645..0000000000 --- a/core/src/server/grpc_impl/GrpcRequestTask.cpp +++ /dev/null @@ -1,942 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include "server/grpc_impl/GrpcRequestTask.h" - -#include -#include -#include -#include -#include -//#include - -#include "GrpcServer.h" -#include "db/Utils.h" -#include "scheduler/SchedInst.h" -#include "server/DBWrapper.h" -#include "server/Server.h" -#include "src/version.h" -#include "utils/CommonUtil.h" -#include "utils/Log.h" -#include "utils/TimeRecorder.h" -#include "utils/ValidationUtil.h" - -namespace milvus { -namespace server { -namespace grpc { - -static const char* DQL_TASK_GROUP = "dql"; -static const char* DDL_DML_TASK_GROUP = "ddl_dml"; -static const char* INFO_TASK_GROUP = "info"; - -constexpr int64_t DAY_SECONDS = 24 * 60 * 60; - -using DB_META = milvus::engine::meta::Meta; -using DB_DATE = milvus::engine::meta::DateT; - -namespace { -engine::EngineType -EngineType(int type) { - static std::map map_type = { - {0, engine::EngineType::INVALID}, - {1, engine::EngineType::FAISS_IDMAP}, - {2, engine::EngineType::FAISS_IVFFLAT}, - {3, engine::EngineType::FAISS_IVFSQ8}, - }; - - if (map_type.find(type) == map_type.end()) { - return engine::EngineType::INVALID; - } - - return map_type[type]; -} - -int -IndexType(engine::EngineType type) { - static std::map map_type = { - {engine::EngineType::INVALID, 0}, - {engine::EngineType::FAISS_IDMAP, 1}, - {engine::EngineType::FAISS_IVFFLAT, 2}, - {engine::EngineType::FAISS_IVFSQ8, 3}, - }; - - if (map_type.find(type) == map_type.end()) { - return 0; - } - - return map_type[type]; -} - -Status -ConvertTimeRangeToDBDates(const std::vector<::milvus::grpc::Range>& range_array, std::vector& dates) { - dates.clear(); - for (auto& range : range_array) { - time_t tt_start, tt_end; - tm tm_start, tm_end; - if (!CommonUtil::TimeStrToTime(range.start_value(), tt_start, tm_start)) { - return Status(SERVER_INVALID_TIME_RANGE, "Invalid time range: " + range.start_value()); - } - - if (!CommonUtil::TimeStrToTime(range.end_value(), tt_end, tm_end)) { - return Status(SERVER_INVALID_TIME_RANGE, "Invalid time range: " + range.start_value()); - } - - int64_t days = (tt_end - tt_start) / DAY_SECONDS; - if (days <= 0) { - return Status(SERVER_INVALID_TIME_RANGE, - "Invalid time range: The start-date should be smaller than end-date!"); - } - - // range: [start_day, end_day) - for (int64_t i = 0; i < days; i++) { - time_t tt_day = tt_start + DAY_SECONDS * i; - tm tm_day; - CommonUtil::ConvertTime(tt_day, tm_day); - - int64_t date = tm_day.tm_year * 10000 + tm_day.tm_mon * 100 + tm_day.tm_mday; // according to db logic - dates.push_back(date); - } - } - - return Status::OK(); -} - -std::string -TableNotExistMsg(const std::string& table_name) { - return "Table " + table_name + - " not exist. Use milvus.has_table to verify whether the table exists. You also can check if the table name " - "exists."; -} - -} // namespace - -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -CreateTableTask::CreateTableTask(const ::milvus::grpc::TableSchema* schema) - : GrpcBaseTask(DDL_DML_TASK_GROUP), schema_(schema) { -} - -BaseTaskPtr -CreateTableTask::Create(const ::milvus::grpc::TableSchema* schema) { - if (schema == nullptr) { - SERVER_LOG_ERROR << "grpc input is null!"; - return nullptr; - } - return std::shared_ptr(new CreateTableTask(schema)); -} - -Status -CreateTableTask::OnExecute() { - TimeRecorder rc("CreateTableTask"); - - try { - // step 1: check arguments - auto status = ValidationUtil::ValidateTableName(schema_->table_name()); - if (!status.ok()) { - return status; - } - - status = ValidationUtil::ValidateTableDimension(schema_->dimension()); - if (!status.ok()) { - return status; - } - - status = ValidationUtil::ValidateTableIndexFileSize(schema_->index_file_size()); - if (!status.ok()) { - return status; - } - - status = ValidationUtil::ValidateTableIndexMetricType(schema_->metric_type()); - if (!status.ok()) { - return status; - } - - // step 2: construct table schema - engine::meta::TableSchema table_info; - table_info.table_id_ = schema_->table_name(); - table_info.dimension_ = static_cast(schema_->dimension()); - table_info.index_file_size_ = schema_->index_file_size(); - table_info.metric_type_ = schema_->metric_type(); - - // step 3: create table - status = DBWrapper::DB()->CreateTable(table_info); - if (!status.ok()) { - // table could exist - if (status.code() == DB_ALREADY_EXIST) { - return Status(SERVER_INVALID_TABLE_NAME, status.message()); - } - return status; - } - } catch (std::exception& ex) { - return Status(SERVER_UNEXPECTED_ERROR, ex.what()); - } - - rc.ElapseFromBegin("totally cost"); - - return Status::OK(); -} - -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -DescribeTableTask::DescribeTableTask(const std::string& table_name, ::milvus::grpc::TableSchema* schema) - : GrpcBaseTask(INFO_TASK_GROUP), table_name_(table_name), schema_(schema) { -} - -BaseTaskPtr -DescribeTableTask::Create(const std::string& table_name, ::milvus::grpc::TableSchema* schema) { - return std::shared_ptr(new DescribeTableTask(table_name, schema)); -} - -Status -DescribeTableTask::OnExecute() { - TimeRecorder rc("DescribeTableTask"); - - try { - // step 1: check arguments - auto status = ValidationUtil::ValidateTableName(table_name_); - if (!status.ok()) { - return status; - } - - // step 2: get table info - engine::meta::TableSchema table_info; - table_info.table_id_ = table_name_; - status = DBWrapper::DB()->DescribeTable(table_info); - if (!status.ok()) { - return status; - } - - schema_->set_table_name(table_info.table_id_); - schema_->set_dimension(table_info.dimension_); - schema_->set_index_file_size(table_info.index_file_size_); - schema_->set_metric_type(table_info.metric_type_); - } catch (std::exception& ex) { - return Status(SERVER_UNEXPECTED_ERROR, ex.what()); - } - - rc.ElapseFromBegin("totally cost"); - - return Status::OK(); -} - -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -CreateIndexTask::CreateIndexTask(const ::milvus::grpc::IndexParam* index_param) - : GrpcBaseTask(DDL_DML_TASK_GROUP), index_param_(index_param) { -} - -BaseTaskPtr -CreateIndexTask::Create(const ::milvus::grpc::IndexParam* index_param) { - if (index_param == nullptr) { - SERVER_LOG_ERROR << "grpc input is null!"; - return nullptr; - } - return std::shared_ptr(new CreateIndexTask(index_param)); -} - -Status -CreateIndexTask::OnExecute() { - try { - TimeRecorder rc("CreateIndexTask"); - - // step 1: check arguments - std::string table_name_ = index_param_->table_name(); - auto status = ValidationUtil::ValidateTableName(table_name_); - if (!status.ok()) { - return status; - } - - bool has_table = false; - status = DBWrapper::DB()->HasTable(table_name_, has_table); - if (!status.ok()) { - return status; - } - - if (!has_table) { - return Status(SERVER_TABLE_NOT_EXIST, TableNotExistMsg(table_name_)); - } - - auto& grpc_index = index_param_->index(); - status = ValidationUtil::ValidateTableIndexType(grpc_index.index_type()); - if (!status.ok()) { - return status; - } - - status = ValidationUtil::ValidateTableIndexNlist(grpc_index.nlist()); - if (!status.ok()) { - return status; - } - - // step 2: check table existence - engine::TableIndex index; - index.engine_type_ = grpc_index.index_type(); - index.nlist_ = grpc_index.nlist(); - status = DBWrapper::DB()->CreateIndex(table_name_, index); - if (!status.ok()) { - return status; - } - - rc.ElapseFromBegin("totally cost"); - } catch (std::exception& ex) { - return Status(SERVER_UNEXPECTED_ERROR, ex.what()); - } - - return Status::OK(); -} - -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -HasTableTask::HasTableTask(const std::string& table_name, bool& has_table) - : GrpcBaseTask(INFO_TASK_GROUP), table_name_(table_name), has_table_(has_table) { -} - -BaseTaskPtr -HasTableTask::Create(const std::string& table_name, bool& has_table) { - return std::shared_ptr(new HasTableTask(table_name, has_table)); -} - -Status -HasTableTask::OnExecute() { - try { - TimeRecorder rc("HasTableTask"); - - // step 1: check arguments - auto status = ValidationUtil::ValidateTableName(table_name_); - if (!status.ok()) { - return status; - } - - // step 2: check table existence - status = DBWrapper::DB()->HasTable(table_name_, has_table_); - if (!status.ok()) { - return status; - } - - rc.ElapseFromBegin("totally cost"); - } catch (std::exception& ex) { - return Status(SERVER_UNEXPECTED_ERROR, ex.what()); - } - - return Status::OK(); -} - -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -DropTableTask::DropTableTask(const std::string& table_name) - : GrpcBaseTask(DDL_DML_TASK_GROUP), table_name_(table_name) { -} - -BaseTaskPtr -DropTableTask::Create(const std::string& table_name) { - return std::shared_ptr(new DropTableTask(table_name)); -} - -Status -DropTableTask::OnExecute() { - try { - TimeRecorder rc("DropTableTask"); - - // step 1: check arguments - auto status = ValidationUtil::ValidateTableName(table_name_); - if (!status.ok()) { - return status; - } - - // step 2: check table existence - engine::meta::TableSchema table_info; - table_info.table_id_ = table_name_; - status = DBWrapper::DB()->DescribeTable(table_info); - if (!status.ok()) { - if (status.code() == DB_NOT_FOUND) { - return Status(SERVER_TABLE_NOT_EXIST, TableNotExistMsg(table_name_)); - } else { - return status; - } - } - - rc.ElapseFromBegin("check validation"); - - // step 3: Drop table - std::vector dates; - status = DBWrapper::DB()->DeleteTable(table_name_, dates); - if (!status.ok()) { - return status; - } - - rc.ElapseFromBegin("total cost"); - } catch (std::exception& ex) { - return Status(SERVER_UNEXPECTED_ERROR, ex.what()); - } - - return Status::OK(); -} - -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -ShowTablesTask::ShowTablesTask(::milvus::grpc::TableNameList* table_name_list) - : GrpcBaseTask(INFO_TASK_GROUP), table_name_list_(table_name_list) { -} - -BaseTaskPtr -ShowTablesTask::Create(::milvus::grpc::TableNameList* table_name_list) { - return std::shared_ptr(new ShowTablesTask(table_name_list)); -} - -Status -ShowTablesTask::OnExecute() { - std::vector schema_array; - auto statuts = DBWrapper::DB()->AllTables(schema_array); - if (!statuts.ok()) { - return statuts; - } - - for (auto& schema : schema_array) { - table_name_list_->add_table_names(schema.table_id_); - } - return Status::OK(); -} - -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -InsertTask::InsertTask(const ::milvus::grpc::InsertParam* insert_param, ::milvus::grpc::VectorIds* record_ids) - : GrpcBaseTask(DDL_DML_TASK_GROUP), insert_param_(insert_param), record_ids_(record_ids) { -} - -BaseTaskPtr -InsertTask::Create(const ::milvus::grpc::InsertParam* insert_param, ::milvus::grpc::VectorIds* record_ids) { - if (insert_param == nullptr) { - SERVER_LOG_ERROR << "grpc input is null!"; - return nullptr; - } - return std::shared_ptr(new InsertTask(insert_param, record_ids)); -} - -Status -InsertTask::OnExecute() { - try { - TimeRecorder rc("InsertVectorTask"); - - // step 1: check arguments - auto status = ValidationUtil::ValidateTableName(insert_param_->table_name()); - if (!status.ok()) { - return status; - } - if (insert_param_->row_record_array().empty()) { - return Status(SERVER_INVALID_ROWRECORD_ARRAY, - "The vector array is empty. Make sure you have entered vector records."); - } - - if (!insert_param_->row_id_array().empty()) { - if (insert_param_->row_id_array().size() != insert_param_->row_record_array_size()) { - return Status(SERVER_ILLEGAL_VECTOR_ID, - "The size of vector ID array must be equal to the size of the vector."); - } - } - - // step 2: check table existence - engine::meta::TableSchema table_info; - table_info.table_id_ = insert_param_->table_name(); - status = DBWrapper::DB()->DescribeTable(table_info); - if (!status.ok()) { - if (status.code() == DB_NOT_FOUND) { - return Status(SERVER_TABLE_NOT_EXIST, TableNotExistMsg(insert_param_->table_name())); - } else { - return status; - } - } - - // step 3: check table flag - // all user provide id, or all internal id - bool user_provide_ids = !insert_param_->row_id_array().empty(); - // user already provided id before, all insert action require user id - if ((table_info.flag_ & engine::meta::FLAG_MASK_HAS_USERID) != 0 && !user_provide_ids) { - return Status(SERVER_ILLEGAL_VECTOR_ID, - "Table vector IDs are user-defined. Please provide IDs for all vectors of this table."); - } - - // user didn't provided id before, no need to provide user id - if ((table_info.flag_ & engine::meta::FLAG_MASK_NO_USERID) != 0 && user_provide_ids) { - return Status( - SERVER_ILLEGAL_VECTOR_ID, - "Table vector IDs are auto-generated. All vectors of this table must use auto-generated IDs."); - } - - rc.RecordSection("check validation"); - -#ifdef MILVUS_ENABLE_PROFILING - std::string fname = - "/tmp/insert_" + std::to_string(this->insert_param_->row_record_array_size()) + ".profiling"; - ProfilerStart(fname.c_str()); -#endif - - // step 4: prepare float data - std::vector vec_f(insert_param_->row_record_array_size() * table_info.dimension_, 0); - - // TODO(yk): change to one dimension array or use multiple-thread to copy the data - for (size_t i = 0; i < insert_param_->row_record_array_size(); i++) { - if (insert_param_->row_record_array(i).vector_data().empty()) { - return Status(SERVER_INVALID_ROWRECORD_ARRAY, - "The vector dimension must be equal to the table dimension."); - } - uint64_t vec_dim = insert_param_->row_record_array(i).vector_data().size(); - if (vec_dim != table_info.dimension_) { - ErrorCode error_code = SERVER_INVALID_VECTOR_DIMENSION; - std::string error_msg = "The vector dimension must be equal to the table dimension."; - return Status(error_code, error_msg); - } - memcpy(&vec_f[i * table_info.dimension_], insert_param_->row_record_array(i).vector_data().data(), - table_info.dimension_ * sizeof(float)); - } - - rc.ElapseFromBegin("prepare vectors data"); - - // step 5: insert vectors - auto vec_count = static_cast(insert_param_->row_record_array_size()); - std::vector vec_ids(insert_param_->row_id_array_size(), 0); - if (!insert_param_->row_id_array().empty()) { - const int64_t* src_data = insert_param_->row_id_array().data(); - int64_t* target_data = vec_ids.data(); - memcpy(target_data, src_data, static_cast(sizeof(int64_t) * insert_param_->row_id_array_size())); - } - - status = DBWrapper::DB()->InsertVectors(insert_param_->table_name(), vec_count, vec_f.data(), vec_ids); - rc.ElapseFromBegin("add vectors to engine"); - if (!status.ok()) { - return status; - } - for (int64_t id : vec_ids) { - record_ids_->add_vector_id_array(id); - } - - auto ids_size = record_ids_->vector_id_array_size(); - if (ids_size != vec_count) { - std::string msg = - "Add " + std::to_string(vec_count) + " vectors but only return " + std::to_string(ids_size) + " id"; - return Status(SERVER_ILLEGAL_VECTOR_ID, msg); - } - - // step 6: update table flag - user_provide_ids ? table_info.flag_ |= engine::meta::FLAG_MASK_HAS_USERID - : table_info.flag_ |= engine::meta::FLAG_MASK_NO_USERID; - status = DBWrapper::DB()->UpdateTableFlag(insert_param_->table_name(), table_info.flag_); - -#ifdef MILVUS_ENABLE_PROFILING - ProfilerStop(); -#endif - - rc.RecordSection("add vectors to engine"); - rc.ElapseFromBegin("total cost"); - } catch (std::exception& ex) { - return Status(SERVER_UNEXPECTED_ERROR, ex.what()); - } - - return Status::OK(); -} - -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -SearchTask::SearchTask(const ::milvus::grpc::SearchParam* search_vector_infos, - const std::vector& file_id_array, ::milvus::grpc::TopKQueryResult* response) - : GrpcBaseTask(DQL_TASK_GROUP), - search_param_(search_vector_infos), - file_id_array_(file_id_array), - topk_result_(response) { -} - -BaseTaskPtr -SearchTask::Create(const ::milvus::grpc::SearchParam* search_vector_infos, - const std::vector& file_id_array, ::milvus::grpc::TopKQueryResult* response) { - if (search_vector_infos == nullptr) { - SERVER_LOG_ERROR << "grpc input is null!"; - return nullptr; - } - return std::shared_ptr(new SearchTask(search_vector_infos, file_id_array, response)); -} - -Status -SearchTask::OnExecute() { - try { - int64_t top_k = search_param_->topk(); - int64_t nprobe = search_param_->nprobe(); - - std::string hdr = "SearchTask(k=" + std::to_string(top_k) + ", nprob=" + std::to_string(nprobe) + ")"; - TimeRecorder rc(hdr); - - // step 1: check table name - std::string table_name_ = search_param_->table_name(); - auto status = ValidationUtil::ValidateTableName(table_name_); - if (!status.ok()) { - return status; - } - - // step 2: check table existence - engine::meta::TableSchema table_info; - table_info.table_id_ = table_name_; - status = DBWrapper::DB()->DescribeTable(table_info); - if (!status.ok()) { - if (status.code() == DB_NOT_FOUND) { - return Status(SERVER_TABLE_NOT_EXIST, TableNotExistMsg(table_name_)); - } else { - return status; - } - } - - // step 3: check search parameter - status = ValidationUtil::ValidateSearchTopk(top_k, table_info); - if (!status.ok()) { - return status; - } - - status = ValidationUtil::ValidateSearchNprobe(nprobe, table_info); - if (!status.ok()) { - return status; - } - - if (search_param_->query_record_array().empty()) { - return Status(SERVER_INVALID_ROWRECORD_ARRAY, - "The vector array is empty. Make sure you have entered vector records."); - } - - // step 4: check date range, and convert to db dates - std::vector dates; - std::vector<::milvus::grpc::Range> range_array; - for (size_t i = 0; i < search_param_->query_range_array_size(); i++) { - range_array.emplace_back(search_param_->query_range_array(i)); - } - - status = ConvertTimeRangeToDBDates(range_array, dates); - if (!status.ok()) { - return status; - } - - rc.RecordSection("check validation"); - - // step 5: prepare float data - auto record_array_size = search_param_->query_record_array_size(); - std::vector vec_f(record_array_size * table_info.dimension_, 0); - for (size_t i = 0; i < record_array_size; i++) { - if (search_param_->query_record_array(i).vector_data().empty()) { - return Status(SERVER_INVALID_ROWRECORD_ARRAY, - "The vector dimension must be equal to the table dimension."); - } - uint64_t query_vec_dim = search_param_->query_record_array(i).vector_data().size(); - if (query_vec_dim != table_info.dimension_) { - ErrorCode error_code = SERVER_INVALID_VECTOR_DIMENSION; - std::string error_msg = "The vector dimension must be equal to the table dimension."; - return Status(error_code, error_msg); - } - - memcpy(&vec_f[i * table_info.dimension_], search_param_->query_record_array(i).vector_data().data(), - table_info.dimension_ * sizeof(float)); - } - rc.RecordSection("prepare vector data"); - - // step 6: search vectors - engine::ResultIds result_ids; - engine::ResultDistances result_distances; - auto record_count = (uint64_t)search_param_->query_record_array().size(); - -#ifdef MILVUS_ENABLE_PROFILING - std::string fname = - "/tmp/search_nq_" + std::to_string(this->search_param_->query_record_array_size()) + ".profiling"; - ProfilerStart(fname.c_str()); -#endif - - if (file_id_array_.empty()) { - status = DBWrapper::DB()->Query(table_name_, (size_t)top_k, record_count, nprobe, vec_f.data(), dates, - result_ids, result_distances); - } else { - status = DBWrapper::DB()->Query(table_name_, file_id_array_, (size_t)top_k, record_count, nprobe, - vec_f.data(), dates, result_ids, result_distances); - } - -#ifdef MILVUS_ENABLE_PROFILING - ProfilerStop(); -#endif - - rc.RecordSection("search vectors from engine"); - if (!status.ok()) { - return status; - } - - if (result_ids.empty()) { - return Status::OK(); // empty table - } - - // step 7: construct result array - topk_result_->set_row_num(record_count); - topk_result_->add_ids(result_ids.begin(), result_ids.end()); - topk_result_->add_distances(result_distances.begin(), result_distances.end()); - - // step 8: print time cost percent - rc.RecordSection("construct result and send"); - rc.ElapseFromBegin("totally cost"); - } catch (std::exception& ex) { - return Status(SERVER_UNEXPECTED_ERROR, ex.what()); - } - - return Status::OK(); -} - -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -CountTableTask::CountTableTask(const std::string& table_name, int64_t& row_count) - : GrpcBaseTask(INFO_TASK_GROUP), table_name_(table_name), row_count_(row_count) { -} - -BaseTaskPtr -CountTableTask::Create(const std::string& table_name, int64_t& row_count) { - return std::shared_ptr(new CountTableTask(table_name, row_count)); -} - -Status -CountTableTask::OnExecute() { - try { - TimeRecorder rc("GetTableRowCountTask"); - - // step 1: check arguments - auto status = ValidationUtil::ValidateTableName(table_name_); - if (!status.ok()) { - return status; - } - - // step 2: get row count - uint64_t row_count = 0; - status = DBWrapper::DB()->GetTableRowCount(table_name_, row_count); - if (!status.ok()) { - if (status.code(), DB_NOT_FOUND) { - return Status(SERVER_TABLE_NOT_EXIST, TableNotExistMsg(table_name_)); - } else { - return status; - } - } - - row_count_ = static_cast(row_count); - - rc.ElapseFromBegin("total cost"); - } catch (std::exception& ex) { - return Status(SERVER_UNEXPECTED_ERROR, ex.what()); - } - - return Status::OK(); -} - -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -CmdTask::CmdTask(const std::string& cmd, std::string& result) - : GrpcBaseTask(INFO_TASK_GROUP), cmd_(cmd), result_(result) { -} - -BaseTaskPtr -CmdTask::Create(const std::string& cmd, std::string& result) { - return std::shared_ptr(new CmdTask(cmd, result)); -} - -Status -CmdTask::OnExecute() { - if (cmd_ == "version") { - result_ = MILVUS_VERSION; - } else if (cmd_ == "tasktable") { - result_ = scheduler::ResMgrInst::GetInstance()->DumpTaskTables(); - } else { - result_ = "OK"; - } - - return Status::OK(); -} - -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -DeleteByRangeTask::DeleteByRangeTask(const ::milvus::grpc::DeleteByRangeParam* delete_by_range_param) - : GrpcBaseTask(DDL_DML_TASK_GROUP), delete_by_range_param_(delete_by_range_param) { -} - -BaseTaskPtr -DeleteByRangeTask::Create(const ::milvus::grpc::DeleteByRangeParam* delete_by_range_param) { - if (delete_by_range_param == nullptr) { - SERVER_LOG_ERROR << "grpc input is null!"; - return nullptr; - } - - return std::shared_ptr(new DeleteByRangeTask(delete_by_range_param)); -} - -Status -DeleteByRangeTask::OnExecute() { - try { - TimeRecorder rc("DeleteByRangeTask"); - - // step 1: check arguments - std::string table_name = delete_by_range_param_->table_name(); - auto status = ValidationUtil::ValidateTableName(table_name); - if (!status.ok()) { - return status; - } - - // step 2: check table existence - engine::meta::TableSchema table_info; - table_info.table_id_ = table_name; - status = DBWrapper::DB()->DescribeTable(table_info); - if (!status.ok()) { - if (status.code(), DB_NOT_FOUND) { - return Status(SERVER_TABLE_NOT_EXIST, TableNotExistMsg(table_name)); - } else { - return status; - } - } - - rc.ElapseFromBegin("check validation"); - - // step 3: check date range, and convert to db dates - std::vector dates; - ErrorCode error_code = SERVER_SUCCESS; - std::string error_msg; - - std::vector<::milvus::grpc::Range> range_array; - range_array.emplace_back(delete_by_range_param_->range()); - status = ConvertTimeRangeToDBDates(range_array, dates); - if (!status.ok()) { - return status; - } - -#ifdef MILVUS_ENABLE_PROFILING - std::string fname = "/tmp/search_nq_" + this->delete_by_range_param_->table_name() + ".profiling"; - ProfilerStart(fname.c_str()); -#endif - status = DBWrapper::DB()->DeleteTable(table_name, dates); - if (!status.ok()) { - return status; - } - } catch (std::exception& ex) { - return Status(SERVER_UNEXPECTED_ERROR, ex.what()); - } - - return Status::OK(); -} - -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -PreloadTableTask::PreloadTableTask(const std::string& table_name) - : GrpcBaseTask(DQL_TASK_GROUP), table_name_(table_name) { -} - -BaseTaskPtr -PreloadTableTask::Create(const std::string& table_name) { - return std::shared_ptr(new PreloadTableTask(table_name)); -} - -Status -PreloadTableTask::OnExecute() { - try { - TimeRecorder rc("PreloadTableTask"); - - // step 1: check arguments - auto status = ValidationUtil::ValidateTableName(table_name_); - if (!status.ok()) { - return status; - } - - // step 2: check table existence - status = DBWrapper::DB()->PreloadTable(table_name_); - if (!status.ok()) { - return status; - } - - rc.ElapseFromBegin("totally cost"); - } catch (std::exception& ex) { - return Status(SERVER_UNEXPECTED_ERROR, ex.what()); - } - - return Status::OK(); -} - -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -DescribeIndexTask::DescribeIndexTask(const std::string& table_name, ::milvus::grpc::IndexParam* index_param) - : GrpcBaseTask(INFO_TASK_GROUP), table_name_(table_name), index_param_(index_param) { -} - -BaseTaskPtr -DescribeIndexTask::Create(const std::string& table_name, ::milvus::grpc::IndexParam* index_param) { - return std::shared_ptr(new DescribeIndexTask(table_name, index_param)); -} - -Status -DescribeIndexTask::OnExecute() { - try { - TimeRecorder rc("DescribeIndexTask"); - - // step 1: check arguments - auto status = ValidationUtil::ValidateTableName(table_name_); - if (!status.ok()) { - return status; - } - - // step 2: check table existence - engine::TableIndex index; - status = DBWrapper::DB()->DescribeIndex(table_name_, index); - if (!status.ok()) { - return status; - } - - index_param_->set_table_name(table_name_); - index_param_->mutable_index()->set_index_type(index.engine_type_); - index_param_->mutable_index()->set_nlist(index.nlist_); - - rc.ElapseFromBegin("totally cost"); - } catch (std::exception& ex) { - return Status(SERVER_UNEXPECTED_ERROR, ex.what()); - } - - return Status::OK(); -} - -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -DropIndexTask::DropIndexTask(const std::string& table_name) - : GrpcBaseTask(DDL_DML_TASK_GROUP), table_name_(table_name) { -} - -BaseTaskPtr -DropIndexTask::Create(const std::string& table_name) { - return std::shared_ptr(new DropIndexTask(table_name)); -} - -Status -DropIndexTask::OnExecute() { - try { - TimeRecorder rc("DropIndexTask"); - - // step 1: check arguments - auto status = ValidationUtil::ValidateTableName(table_name_); - if (!status.ok()) { - return status; - } - - bool has_table = false; - status = DBWrapper::DB()->HasTable(table_name_, has_table); - if (!status.ok()) { - return status; - } - - if (!has_table) { - return Status(SERVER_TABLE_NOT_EXIST, TableNotExistMsg(table_name_)); - } - - // step 2: check table existence - status = DBWrapper::DB()->DropIndex(table_name_); - if (!status.ok()) { - return status; - } - - rc.ElapseFromBegin("totally cost"); - } catch (std::exception& ex) { - return Status(SERVER_UNEXPECTED_ERROR, ex.what()); - } - - return Status::OK(); -} - -} // namespace grpc -} // namespace server -} // namespace milvus diff --git a/core/src/server/grpc_impl/GrpcRequestTask.h b/core/src/server/grpc_impl/GrpcRequestTask.h deleted file mode 100644 index 3732e2ad41..0000000000 --- a/core/src/server/grpc_impl/GrpcRequestTask.h +++ /dev/null @@ -1,272 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#pragma once - -#include "db/Types.h" -#include "server/grpc_impl/GrpcRequestScheduler.h" -#include "utils/Status.h" - -#include "grpc/gen-milvus/milvus.grpc.pb.h" -#include "grpc/gen-status/status.pb.h" - -#include -#include -#include -#include - -namespace milvus { -namespace server { -namespace grpc { - -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -class CreateTableTask : public GrpcBaseTask { - public: - static BaseTaskPtr - Create(const ::milvus::grpc::TableSchema* schema); - - protected: - explicit CreateTableTask(const ::milvus::grpc::TableSchema* schema); - - Status - OnExecute() override; - - private: - const ::milvus::grpc::TableSchema* schema_; -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -class HasTableTask : public GrpcBaseTask { - public: - static BaseTaskPtr - Create(const std::string& table_name, bool& has_table); - - protected: - HasTableTask(const std::string& table_name, bool& has_table); - - Status - OnExecute() override; - - private: - std::string table_name_; - bool& has_table_; -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -class DescribeTableTask : public GrpcBaseTask { - public: - static BaseTaskPtr - Create(const std::string& table_name, ::milvus::grpc::TableSchema* schema); - - protected: - DescribeTableTask(const std::string& table_name, ::milvus::grpc::TableSchema* schema); - - Status - OnExecute() override; - - private: - std::string table_name_; - ::milvus::grpc::TableSchema* schema_; -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -class DropTableTask : public GrpcBaseTask { - public: - static BaseTaskPtr - Create(const std::string& table_name); - - protected: - explicit DropTableTask(const std::string& table_name); - - Status - OnExecute() override; - - private: - std::string table_name_; -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -class CreateIndexTask : public GrpcBaseTask { - public: - static BaseTaskPtr - Create(const ::milvus::grpc::IndexParam* index_param); - - protected: - explicit CreateIndexTask(const ::milvus::grpc::IndexParam* index_param); - - Status - OnExecute() override; - - private: - const ::milvus::grpc::IndexParam* index_param_; -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -class ShowTablesTask : public GrpcBaseTask { - public: - static BaseTaskPtr - Create(::milvus::grpc::TableNameList* table_name_list); - - protected: - explicit ShowTablesTask(::milvus::grpc::TableNameList* table_name_list); - - Status - OnExecute() override; - - private: - ::milvus::grpc::TableNameList* table_name_list_; -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -class InsertTask : public GrpcBaseTask { - public: - static BaseTaskPtr - Create(const ::milvus::grpc::InsertParam* insert_param, ::milvus::grpc::VectorIds* record_ids); - - protected: - InsertTask(const ::milvus::grpc::InsertParam* insert_param, ::milvus::grpc::VectorIds* record_ids); - - Status - OnExecute() override; - - private: - const ::milvus::grpc::InsertParam* insert_param_; - ::milvus::grpc::VectorIds* record_ids_; -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -class SearchTask : public GrpcBaseTask { - public: - static BaseTaskPtr - Create(const ::milvus::grpc::SearchParam* search_param, const std::vector& file_id_array, - ::milvus::grpc::TopKQueryResult* response); - - protected: - SearchTask(const ::milvus::grpc::SearchParam* search_param, const std::vector& file_id_array, - ::milvus::grpc::TopKQueryResult* response); - - Status - OnExecute() override; - - private: - const ::milvus::grpc::SearchParam* search_param_; - std::vector file_id_array_; - ::milvus::grpc::TopKQueryResult* topk_result_; -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -class CountTableTask : public GrpcBaseTask { - public: - static BaseTaskPtr - Create(const std::string& table_name, int64_t& row_count); - - protected: - CountTableTask(const std::string& table_name, int64_t& row_count); - - Status - OnExecute() override; - - private: - std::string table_name_; - int64_t& row_count_; -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -class CmdTask : public GrpcBaseTask { - public: - static BaseTaskPtr - Create(const std::string& cmd, std::string& result); - - protected: - CmdTask(const std::string& cmd, std::string& result); - - Status - OnExecute() override; - - private: - std::string cmd_; - std::string& result_; -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -class DeleteByRangeTask : public GrpcBaseTask { - public: - static BaseTaskPtr - Create(const ::milvus::grpc::DeleteByRangeParam* delete_by_range_param); - - protected: - explicit DeleteByRangeTask(const ::milvus::grpc::DeleteByRangeParam* delete_by_range_param); - - Status - OnExecute() override; - - private: - const ::milvus::grpc::DeleteByRangeParam* delete_by_range_param_; -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -class PreloadTableTask : public GrpcBaseTask { - public: - static BaseTaskPtr - Create(const std::string& table_name); - - protected: - explicit PreloadTableTask(const std::string& table_name); - - Status - OnExecute() override; - - private: - std::string table_name_; -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -class DescribeIndexTask : public GrpcBaseTask { - public: - static BaseTaskPtr - Create(const std::string& table_name, ::milvus::grpc::IndexParam* index_param); - - protected: - DescribeIndexTask(const std::string& table_name, ::milvus::grpc::IndexParam* index_param); - - Status - OnExecute() override; - - private: - std::string table_name_; - ::milvus::grpc::IndexParam* index_param_; -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -class DropIndexTask : public GrpcBaseTask { - public: - static BaseTaskPtr - Create(const std::string& table_name); - - protected: - explicit DropIndexTask(const std::string& table_name); - - Status - OnExecute() override; - - private: - std::string table_name_; -}; - -} // namespace grpc -} // namespace server -} // namespace milvus diff --git a/core/src/server/grpc_impl/request/CmdRequest.cpp b/core/src/server/grpc_impl/request/CmdRequest.cpp new file mode 100644 index 0000000000..b215f94d31 --- /dev/null +++ b/core/src/server/grpc_impl/request/CmdRequest.cpp @@ -0,0 +1,51 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "server/grpc_impl/request/CmdRequest.h" +#include "scheduler/SchedInst.h" + +#include + +namespace milvus { +namespace server { +namespace grpc { + +CmdRequest::CmdRequest(const std::string& cmd, std::string& result) + : GrpcBaseRequest(INFO_REQUEST_GROUP), cmd_(cmd), result_(result) { +} + +BaseRequestPtr +CmdRequest::Create(const std::string& cmd, std::string& result) { + return std::shared_ptr(new CmdRequest(cmd, result)); +} + +Status +CmdRequest::OnExecute() { + if (cmd_ == "version") { + result_ = MILVUS_VERSION; + } else if (cmd_ == "tasktable") { + result_ = scheduler::ResMgrInst::GetInstance()->DumpTaskTables(); + } else { + result_ = "OK"; + } + + return Status::OK(); +} + +} // namespace grpc +} // namespace server +} // namespace milvus diff --git a/core/src/server/grpc_impl/request/CmdRequest.h b/core/src/server/grpc_impl/request/CmdRequest.h new file mode 100644 index 0000000000..960b17d436 --- /dev/null +++ b/core/src/server/grpc_impl/request/CmdRequest.h @@ -0,0 +1,46 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include "server/grpc_impl/request/GrpcBaseRequest.h" + +#include + +namespace milvus { +namespace server { +namespace grpc { + +class CmdRequest : public GrpcBaseRequest { + public: + static BaseRequestPtr + Create(const std::string& cmd, std::string& result); + + protected: + CmdRequest(const std::string& cmd, std::string& result); + + Status + OnExecute() override; + + private: + std::string cmd_; + std::string& result_; +}; + +} // namespace grpc +} // namespace server +} // namespace milvus diff --git a/core/src/server/grpc_impl/request/CountTableRequest.cpp b/core/src/server/grpc_impl/request/CountTableRequest.cpp new file mode 100644 index 0000000000..8559890ad6 --- /dev/null +++ b/core/src/server/grpc_impl/request/CountTableRequest.cpp @@ -0,0 +1,73 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "server/grpc_impl/request/CountTableRequest.h" +#include "server/DBWrapper.h" +#include "utils/Log.h" +#include "utils/TimeRecorder.h" +#include "utils/ValidationUtil.h" + +#include + +namespace milvus { +namespace server { +namespace grpc { + +CountTableRequest::CountTableRequest(const std::string& table_name, int64_t& row_count) + : GrpcBaseRequest(INFO_REQUEST_GROUP), table_name_(table_name), row_count_(row_count) { +} + +BaseRequestPtr +CountTableRequest::Create(const std::string& table_name, int64_t& row_count) { + return std::shared_ptr(new CountTableRequest(table_name, row_count)); +} + +Status +CountTableRequest::OnExecute() { + try { + TimeRecorder rc("CountTableRequest"); + + // step 1: check arguments + auto status = ValidationUtil::ValidateTableName(table_name_); + if (!status.ok()) { + return status; + } + + // step 2: get row count + uint64_t row_count = 0; + status = DBWrapper::DB()->GetTableRowCount(table_name_, row_count); + if (!status.ok()) { + if (status.code(), DB_NOT_FOUND) { + return Status(SERVER_TABLE_NOT_EXIST, TableNotExistMsg(table_name_)); + } else { + return status; + } + } + + row_count_ = static_cast(row_count); + + rc.ElapseFromBegin("total cost"); + } catch (std::exception& ex) { + return Status(SERVER_UNEXPECTED_ERROR, ex.what()); + } + + return Status::OK(); +} + +} // namespace grpc +} // namespace server +} // namespace milvus diff --git a/core/src/server/grpc_impl/request/CountTableRequest.h b/core/src/server/grpc_impl/request/CountTableRequest.h new file mode 100644 index 0000000000..80f8fcfcf5 --- /dev/null +++ b/core/src/server/grpc_impl/request/CountTableRequest.h @@ -0,0 +1,46 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include "server/grpc_impl/request/GrpcBaseRequest.h" + +#include + +namespace milvus { +namespace server { +namespace grpc { + +class CountTableRequest : public GrpcBaseRequest { + public: + static BaseRequestPtr + Create(const std::string& table_name, int64_t& row_count); + + protected: + CountTableRequest(const std::string& table_name, int64_t& row_count); + + Status + OnExecute() override; + + private: + std::string table_name_; + int64_t& row_count_; +}; + +} // namespace grpc +} // namespace server +} // namespace milvus diff --git a/core/src/server/grpc_impl/request/CreateIndexRequest.cpp b/core/src/server/grpc_impl/request/CreateIndexRequest.cpp new file mode 100644 index 0000000000..72678aee87 --- /dev/null +++ b/core/src/server/grpc_impl/request/CreateIndexRequest.cpp @@ -0,0 +1,96 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "server/grpc_impl/request/CreateIndexRequest.h" +#include "server/DBWrapper.h" +#include "utils/Log.h" +#include "utils/TimeRecorder.h" +#include "utils/ValidationUtil.h" + +#include +#include + +namespace milvus { +namespace server { +namespace grpc { + +CreateIndexRequest::CreateIndexRequest(const ::milvus::grpc::IndexParam* index_param) + : GrpcBaseRequest(DDL_DML_REQUEST_GROUP), index_param_(index_param) { +} + +BaseRequestPtr +CreateIndexRequest::Create(const ::milvus::grpc::IndexParam* index_param) { + if (index_param == nullptr) { + SERVER_LOG_ERROR << "grpc input is null!"; + return nullptr; + } + return std::shared_ptr(new CreateIndexRequest(index_param)); +} + +Status +CreateIndexRequest::OnExecute() { + try { + TimeRecorder rc("CreateIndexRequest"); + + // step 1: check arguments + std::string table_name_ = index_param_->table_name(); + auto status = ValidationUtil::ValidateTableName(table_name_); + if (!status.ok()) { + return status; + } + + bool has_table = false; + status = DBWrapper::DB()->HasTable(table_name_, has_table); + if (!status.ok()) { + return status; + } + + if (!has_table) { + return Status(SERVER_TABLE_NOT_EXIST, TableNotExistMsg(table_name_)); + } + + auto& grpc_index = index_param_->index(); + status = ValidationUtil::ValidateTableIndexType(grpc_index.index_type()); + if (!status.ok()) { + return status; + } + + status = ValidationUtil::ValidateTableIndexNlist(grpc_index.nlist()); + if (!status.ok()) { + return status; + } + + // step 2: check table existence + engine::TableIndex index; + index.engine_type_ = grpc_index.index_type(); + index.nlist_ = grpc_index.nlist(); + status = DBWrapper::DB()->CreateIndex(table_name_, index); + if (!status.ok()) { + return status; + } + + rc.ElapseFromBegin("totally cost"); + } catch (std::exception& ex) { + return Status(SERVER_UNEXPECTED_ERROR, ex.what()); + } + + return Status::OK(); +} + +} // namespace grpc +} // namespace server +} // namespace milvus diff --git a/core/src/server/grpc_impl/request/CreateIndexRequest.h b/core/src/server/grpc_impl/request/CreateIndexRequest.h new file mode 100644 index 0000000000..9503cf0d06 --- /dev/null +++ b/core/src/server/grpc_impl/request/CreateIndexRequest.h @@ -0,0 +1,43 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include "server/grpc_impl/request/GrpcBaseRequest.h" + +namespace milvus { +namespace server { +namespace grpc { + +class CreateIndexRequest : public GrpcBaseRequest { + public: + static BaseRequestPtr + Create(const ::milvus::grpc::IndexParam* index_param); + + protected: + explicit CreateIndexRequest(const ::milvus::grpc::IndexParam* index_param); + + Status + OnExecute() override; + + private: + const ::milvus::grpc::IndexParam* index_param_; +}; + +} // namespace grpc +} // namespace server +} // namespace milvus diff --git a/core/src/server/grpc_impl/request/CreatePartitionRequest.cpp b/core/src/server/grpc_impl/request/CreatePartitionRequest.cpp new file mode 100644 index 0000000000..3bd4a86ef6 --- /dev/null +++ b/core/src/server/grpc_impl/request/CreatePartitionRequest.cpp @@ -0,0 +1,85 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "server/grpc_impl/request/CreatePartitionRequest.h" +#include "server/DBWrapper.h" +#include "utils/Log.h" +#include "utils/TimeRecorder.h" +#include "utils/ValidationUtil.h" + +#include + +namespace milvus { +namespace server { +namespace grpc { + +CreatePartitionRequest::CreatePartitionRequest(const ::milvus::grpc::PartitionParam* partition_param) + : GrpcBaseRequest(DDL_DML_REQUEST_GROUP), partition_param_(partition_param) { +} + +BaseRequestPtr +CreatePartitionRequest::Create(const ::milvus::grpc::PartitionParam* partition_param) { + if (partition_param == nullptr) { + SERVER_LOG_ERROR << "grpc input is null!"; + return nullptr; + } + return std::shared_ptr(new CreatePartitionRequest(partition_param)); +} + +Status +CreatePartitionRequest::OnExecute() { + TimeRecorder rc("CreatePartitionRequest"); + + try { + // step 1: check arguments + auto status = ValidationUtil::ValidateTableName(partition_param_->table_name()); + if (!status.ok()) { + return status; + } + + status = ValidationUtil::ValidatePartitionName(partition_param_->partition_name()); + if (!status.ok()) { + return status; + } + + status = ValidationUtil::ValidatePartitionTags({partition_param_->tag()}); + if (!status.ok()) { + return status; + } + + // step 2: create partition + status = DBWrapper::DB()->CreatePartition(partition_param_->table_name(), partition_param_->partition_name(), + partition_param_->tag()); + if (!status.ok()) { + // partition could exist + if (status.code() == DB_ALREADY_EXIST) { + return Status(SERVER_INVALID_TABLE_NAME, status.message()); + } + return status; + } + } catch (std::exception& ex) { + return Status(SERVER_UNEXPECTED_ERROR, ex.what()); + } + + rc.ElapseFromBegin("totally cost"); + + return Status::OK(); +} + +} // namespace grpc +} // namespace server +} // namespace milvus diff --git a/core/src/server/grpc_impl/request/CreatePartitionRequest.h b/core/src/server/grpc_impl/request/CreatePartitionRequest.h new file mode 100644 index 0000000000..70a284bc16 --- /dev/null +++ b/core/src/server/grpc_impl/request/CreatePartitionRequest.h @@ -0,0 +1,43 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include "server/grpc_impl/request/GrpcBaseRequest.h" + +namespace milvus { +namespace server { +namespace grpc { + +class CreatePartitionRequest : public GrpcBaseRequest { + public: + static BaseRequestPtr + Create(const ::milvus::grpc::PartitionParam* partition_param); + + protected: + explicit CreatePartitionRequest(const ::milvus::grpc::PartitionParam* partition_param); + + Status + OnExecute() override; + + private: + const ::milvus::grpc::PartitionParam* partition_param_; +}; + +} // namespace grpc +} // namespace server +} // namespace milvus diff --git a/core/src/server/grpc_impl/request/CreateTableRequest.cpp b/core/src/server/grpc_impl/request/CreateTableRequest.cpp new file mode 100644 index 0000000000..67a3eaa877 --- /dev/null +++ b/core/src/server/grpc_impl/request/CreateTableRequest.cpp @@ -0,0 +1,96 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "server/grpc_impl/request/CreateTableRequest.h" +#include "server/DBWrapper.h" +#include "utils/Log.h" +#include "utils/TimeRecorder.h" +#include "utils/ValidationUtil.h" + +#include + +namespace milvus { +namespace server { +namespace grpc { + +CreateTableRequest::CreateTableRequest(const ::milvus::grpc::TableSchema* schema) + : GrpcBaseRequest(DDL_DML_REQUEST_GROUP), schema_(schema) { +} + +BaseRequestPtr +CreateTableRequest::Create(const ::milvus::grpc::TableSchema* schema) { + if (schema == nullptr) { + SERVER_LOG_ERROR << "grpc input is null!"; + return nullptr; + } + return std::shared_ptr(new CreateTableRequest(schema)); +} + +Status +CreateTableRequest::OnExecute() { + TimeRecorder rc("CreateTableRequest"); + + try { + // step 1: check arguments + auto status = ValidationUtil::ValidateTableName(schema_->table_name()); + if (!status.ok()) { + return status; + } + + status = ValidationUtil::ValidateTableDimension(schema_->dimension()); + if (!status.ok()) { + return status; + } + + status = ValidationUtil::ValidateTableIndexFileSize(schema_->index_file_size()); + if (!status.ok()) { + return status; + } + + status = ValidationUtil::ValidateTableIndexMetricType(schema_->metric_type()); + if (!status.ok()) { + return status; + } + + // step 2: construct table schema + engine::meta::TableSchema table_info; + table_info.table_id_ = schema_->table_name(); + table_info.dimension_ = static_cast(schema_->dimension()); + table_info.index_file_size_ = schema_->index_file_size(); + table_info.metric_type_ = schema_->metric_type(); + + // step 3: create table + status = DBWrapper::DB()->CreateTable(table_info); + if (!status.ok()) { + // table could exist + if (status.code() == DB_ALREADY_EXIST) { + return Status(SERVER_INVALID_TABLE_NAME, status.message()); + } + return status; + } + } catch (std::exception& ex) { + return Status(SERVER_UNEXPECTED_ERROR, ex.what()); + } + + rc.ElapseFromBegin("totally cost"); + + return Status::OK(); +} + +} // namespace grpc +} // namespace server +} // namespace milvus diff --git a/core/src/server/grpc_impl/request/CreateTableRequest.h b/core/src/server/grpc_impl/request/CreateTableRequest.h new file mode 100644 index 0000000000..3a07aae1b7 --- /dev/null +++ b/core/src/server/grpc_impl/request/CreateTableRequest.h @@ -0,0 +1,43 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include "server/grpc_impl/request/GrpcBaseRequest.h" + +namespace milvus { +namespace server { +namespace grpc { + +class CreateTableRequest : public GrpcBaseRequest { + public: + static BaseRequestPtr + Create(const ::milvus::grpc::TableSchema* schema); + + protected: + explicit CreateTableRequest(const ::milvus::grpc::TableSchema* schema); + + Status + OnExecute() override; + + private: + const ::milvus::grpc::TableSchema* schema_; +}; + +} // namespace grpc +} // namespace server +} // namespace milvus diff --git a/core/src/server/grpc_impl/request/DeleteByDateRequest.cpp b/core/src/server/grpc_impl/request/DeleteByDateRequest.cpp new file mode 100644 index 0000000000..4a4d414803 --- /dev/null +++ b/core/src/server/grpc_impl/request/DeleteByDateRequest.cpp @@ -0,0 +1,101 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "server/grpc_impl/request/DeleteByDateRequest.h" +#include "server/DBWrapper.h" +#include "utils/Log.h" +#include "utils/TimeRecorder.h" +#include "utils/ValidationUtil.h" + +#include +#include +#include + +namespace milvus { +namespace server { +namespace grpc { + +DeleteByDateRequest::DeleteByDateRequest(const ::milvus::grpc::DeleteByDateParam* delete_by_range_param) + : GrpcBaseRequest(DDL_DML_REQUEST_GROUP), delete_by_range_param_(delete_by_range_param) { +} + +BaseRequestPtr +DeleteByDateRequest::Create(const ::milvus::grpc::DeleteByDateParam* delete_by_range_param) { + if (delete_by_range_param == nullptr) { + SERVER_LOG_ERROR << "grpc input is null!"; + return nullptr; + } + + return std::shared_ptr(new DeleteByDateRequest(delete_by_range_param)); +} + +Status +DeleteByDateRequest::OnExecute() { + try { + TimeRecorder rc("DeleteByDateRequest"); + + // step 1: check arguments + std::string table_name = delete_by_range_param_->table_name(); + auto status = ValidationUtil::ValidateTableName(table_name); + if (!status.ok()) { + return status; + } + + // step 2: check table existence + engine::meta::TableSchema table_info; + table_info.table_id_ = table_name; + status = DBWrapper::DB()->DescribeTable(table_info); + if (!status.ok()) { + if (status.code(), DB_NOT_FOUND) { + return Status(SERVER_TABLE_NOT_EXIST, TableNotExistMsg(table_name)); + } else { + return status; + } + } + + rc.ElapseFromBegin("check validation"); + + // step 3: check date range, and convert to db dates + std::vector dates; + ErrorCode error_code = SERVER_SUCCESS; + std::string error_msg; + + std::vector<::milvus::grpc::Range> range_array; + range_array.emplace_back(delete_by_range_param_->range()); + status = ConvertTimeRangeToDBDates(range_array, dates); + if (!status.ok()) { + return status; + } + +#ifdef MILVUS_ENABLE_PROFILING + std::string fname = "/tmp/search_nq_" + this->delete_by_range_param_->table_name() + ".profiling"; + ProfilerStart(fname.c_str()); +#endif + status = DBWrapper::DB()->DropTable(table_name, dates); + if (!status.ok()) { + return status; + } + } catch (std::exception& ex) { + return Status(SERVER_UNEXPECTED_ERROR, ex.what()); + } + + return Status::OK(); +} + +} // namespace grpc +} // namespace server +} // namespace milvus diff --git a/core/src/server/grpc_impl/request/DeleteByDateRequest.h b/core/src/server/grpc_impl/request/DeleteByDateRequest.h new file mode 100644 index 0000000000..b3e04c2850 --- /dev/null +++ b/core/src/server/grpc_impl/request/DeleteByDateRequest.h @@ -0,0 +1,43 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include "server/grpc_impl/request/GrpcBaseRequest.h" + +namespace milvus { +namespace server { +namespace grpc { + +class DeleteByDateRequest : public GrpcBaseRequest { + public: + static BaseRequestPtr + Create(const ::milvus::grpc::DeleteByDateParam* delete_by_range_param); + + protected: + explicit DeleteByDateRequest(const ::milvus::grpc::DeleteByDateParam* delete_by_range_param); + + Status + OnExecute() override; + + private: + const ::milvus::grpc::DeleteByDateParam* delete_by_range_param_; +}; + +} // namespace grpc +} // namespace server +} // namespace milvus diff --git a/core/src/server/grpc_impl/request/DescribeIndexRequest.cpp b/core/src/server/grpc_impl/request/DescribeIndexRequest.cpp new file mode 100644 index 0000000000..b3a987c6b0 --- /dev/null +++ b/core/src/server/grpc_impl/request/DescribeIndexRequest.cpp @@ -0,0 +1,71 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "server/grpc_impl/request/DescribeIndexRequest.h" +#include "server/DBWrapper.h" +#include "utils/Log.h" +#include "utils/TimeRecorder.h" +#include "utils/ValidationUtil.h" + +#include + +namespace milvus { +namespace server { +namespace grpc { + +DescribeIndexRequest::DescribeIndexRequest(const std::string& table_name, ::milvus::grpc::IndexParam* index_param) + : GrpcBaseRequest(INFO_REQUEST_GROUP), table_name_(table_name), index_param_(index_param) { +} + +BaseRequestPtr +DescribeIndexRequest::Create(const std::string& table_name, ::milvus::grpc::IndexParam* index_param) { + return std::shared_ptr(new DescribeIndexRequest(table_name, index_param)); +} + +Status +DescribeIndexRequest::OnExecute() { + try { + TimeRecorder rc("DescribeIndexRequest"); + + // step 1: check arguments + auto status = ValidationUtil::ValidateTableName(table_name_); + if (!status.ok()) { + return status; + } + + // step 2: check table existence + engine::TableIndex index; + status = DBWrapper::DB()->DescribeIndex(table_name_, index); + if (!status.ok()) { + return status; + } + + index_param_->set_table_name(table_name_); + index_param_->mutable_index()->set_index_type(index.engine_type_); + index_param_->mutable_index()->set_nlist(index.nlist_); + + rc.ElapseFromBegin("totally cost"); + } catch (std::exception& ex) { + return Status(SERVER_UNEXPECTED_ERROR, ex.what()); + } + + return Status::OK(); +} + +} // namespace grpc +} // namespace server +} // namespace milvus diff --git a/core/src/server/grpc_impl/request/DescribeIndexRequest.h b/core/src/server/grpc_impl/request/DescribeIndexRequest.h new file mode 100644 index 0000000000..fa0d72152f --- /dev/null +++ b/core/src/server/grpc_impl/request/DescribeIndexRequest.h @@ -0,0 +1,46 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include "server/grpc_impl/request/GrpcBaseRequest.h" + +#include + +namespace milvus { +namespace server { +namespace grpc { + +class DescribeIndexRequest : public GrpcBaseRequest { + public: + static BaseRequestPtr + Create(const std::string& table_name, ::milvus::grpc::IndexParam* index_param); + + protected: + DescribeIndexRequest(const std::string& table_name, ::milvus::grpc::IndexParam* index_param); + + Status + OnExecute() override; + + private: + std::string table_name_; + ::milvus::grpc::IndexParam* index_param_; +}; + +} // namespace grpc +} // namespace server +} // namespace milvus diff --git a/core/src/server/grpc_impl/request/DescribeTableRequest.cpp b/core/src/server/grpc_impl/request/DescribeTableRequest.cpp new file mode 100644 index 0000000000..28a5f327c5 --- /dev/null +++ b/core/src/server/grpc_impl/request/DescribeTableRequest.cpp @@ -0,0 +1,73 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "server/grpc_impl/request/DescribeTableRequest.h" +#include "server/DBWrapper.h" +#include "utils/Log.h" +#include "utils/TimeRecorder.h" +#include "utils/ValidationUtil.h" + +#include + +namespace milvus { +namespace server { +namespace grpc { + +DescribeTableRequest::DescribeTableRequest(const std::string& table_name, ::milvus::grpc::TableSchema* schema) + : GrpcBaseRequest(INFO_REQUEST_GROUP), table_name_(table_name), schema_(schema) { +} + +BaseRequestPtr +DescribeTableRequest::Create(const std::string& table_name, ::milvus::grpc::TableSchema* schema) { + return std::shared_ptr(new DescribeTableRequest(table_name, schema)); +} + +Status +DescribeTableRequest::OnExecute() { + TimeRecorder rc("DescribeTableRequest"); + + try { + // step 1: check arguments + auto status = ValidationUtil::ValidateTableName(table_name_); + if (!status.ok()) { + return status; + } + + // step 2: get table info + engine::meta::TableSchema table_info; + table_info.table_id_ = table_name_; + status = DBWrapper::DB()->DescribeTable(table_info); + if (!status.ok()) { + return status; + } + + schema_->set_table_name(table_info.table_id_); + schema_->set_dimension(table_info.dimension_); + schema_->set_index_file_size(table_info.index_file_size_); + schema_->set_metric_type(table_info.metric_type_); + } catch (std::exception& ex) { + return Status(SERVER_UNEXPECTED_ERROR, ex.what()); + } + + rc.ElapseFromBegin("totally cost"); + + return Status::OK(); +} + +} // namespace grpc +} // namespace server +} // namespace milvus diff --git a/core/src/server/grpc_impl/request/DescribeTableRequest.h b/core/src/server/grpc_impl/request/DescribeTableRequest.h new file mode 100644 index 0000000000..0d903fedbd --- /dev/null +++ b/core/src/server/grpc_impl/request/DescribeTableRequest.h @@ -0,0 +1,46 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include "server/grpc_impl/request/GrpcBaseRequest.h" + +#include + +namespace milvus { +namespace server { +namespace grpc { + +class DescribeTableRequest : public GrpcBaseRequest { + public: + static BaseRequestPtr + Create(const std::string& table_name, ::milvus::grpc::TableSchema* schema); + + protected: + DescribeTableRequest(const std::string& table_name, ::milvus::grpc::TableSchema* schema); + + Status + OnExecute() override; + + private: + std::string table_name_; + ::milvus::grpc::TableSchema* schema_; +}; + +} // namespace grpc +} // namespace server +} // namespace milvus diff --git a/core/src/server/grpc_impl/request/DropIndexRequest.cpp b/core/src/server/grpc_impl/request/DropIndexRequest.cpp new file mode 100644 index 0000000000..ab5c83b0e5 --- /dev/null +++ b/core/src/server/grpc_impl/request/DropIndexRequest.cpp @@ -0,0 +1,76 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "server/grpc_impl/request/DropIndexRequest.h" +#include "server/DBWrapper.h" +#include "utils/Log.h" +#include "utils/TimeRecorder.h" +#include "utils/ValidationUtil.h" + +#include + +namespace milvus { +namespace server { +namespace grpc { + +DropIndexRequest::DropIndexRequest(const std::string& table_name) + : GrpcBaseRequest(DDL_DML_REQUEST_GROUP), table_name_(table_name) { +} + +BaseRequestPtr +DropIndexRequest::Create(const std::string& table_name) { + return std::shared_ptr(new DropIndexRequest(table_name)); +} + +Status +DropIndexRequest::OnExecute() { + try { + TimeRecorder rc("DropIndexRequest"); + + // step 1: check arguments + auto status = ValidationUtil::ValidateTableName(table_name_); + if (!status.ok()) { + return status; + } + + bool has_table = false; + status = DBWrapper::DB()->HasTable(table_name_, has_table); + if (!status.ok()) { + return status; + } + + if (!has_table) { + return Status(SERVER_TABLE_NOT_EXIST, TableNotExistMsg(table_name_)); + } + + // step 2: check table existence + status = DBWrapper::DB()->DropIndex(table_name_); + if (!status.ok()) { + return status; + } + + rc.ElapseFromBegin("totally cost"); + } catch (std::exception& ex) { + return Status(SERVER_UNEXPECTED_ERROR, ex.what()); + } + + return Status::OK(); +} + +} // namespace grpc +} // namespace server +} // namespace milvus diff --git a/core/src/server/grpc_impl/request/DropIndexRequest.h b/core/src/server/grpc_impl/request/DropIndexRequest.h new file mode 100644 index 0000000000..c7cd51fcd7 --- /dev/null +++ b/core/src/server/grpc_impl/request/DropIndexRequest.h @@ -0,0 +1,45 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include "server/grpc_impl/request/GrpcBaseRequest.h" + +#include + +namespace milvus { +namespace server { +namespace grpc { + +class DropIndexRequest : public GrpcBaseRequest { + public: + static BaseRequestPtr + Create(const std::string& table_name); + + protected: + explicit DropIndexRequest(const std::string& table_name); + + Status + OnExecute() override; + + private: + std::string table_name_; +}; + +} // namespace grpc +} // namespace server +} // namespace milvus diff --git a/core/src/server/grpc_impl/request/DropPartitionRequest.cpp b/core/src/server/grpc_impl/request/DropPartitionRequest.cpp new file mode 100644 index 0000000000..0e29b6abe8 --- /dev/null +++ b/core/src/server/grpc_impl/request/DropPartitionRequest.cpp @@ -0,0 +1,81 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "server/grpc_impl/request/DropPartitionRequest.h" +#include "server/DBWrapper.h" +#include "utils/Log.h" +#include "utils/TimeRecorder.h" +#include "utils/ValidationUtil.h" + +#include +#include + +namespace milvus { +namespace server { +namespace grpc { + +DropPartitionRequest::DropPartitionRequest(const ::milvus::grpc::PartitionParam* partition_param) + : GrpcBaseRequest(DDL_DML_REQUEST_GROUP), partition_param_(partition_param) { +} + +BaseRequestPtr +DropPartitionRequest::Create(const ::milvus::grpc::PartitionParam* partition_param) { + return std::shared_ptr(new DropPartitionRequest(partition_param)); +} + +Status +DropPartitionRequest::OnExecute() { + std::string table_name = partition_param_->table_name(); + std::string partition_name = partition_param_->partition_name(); + std::string partition_tag = partition_param_->tag(); + if (!partition_name.empty()) { + auto status = ValidationUtil::ValidateTableName(partition_name); + if (!status.ok()) { + return status; + } + + // check partition existence + engine::meta::TableSchema table_info; + table_info.table_id_ = partition_name; + status = DBWrapper::DB()->DescribeTable(table_info); + if (!status.ok()) { + if (status.code() == DB_NOT_FOUND) { + return Status(SERVER_TABLE_NOT_EXIST, + "Table " + table_name + "'s partition " + partition_name + " not found"); + } else { + return status; + } + } + + return DBWrapper::DB()->DropPartition(partition_name); + } else { + auto status = ValidationUtil::ValidateTableName(table_name); + if (!status.ok()) { + return status; + } + + status = ValidationUtil::ValidatePartitionTags({partition_tag}); + if (!status.ok()) { + return status; + } + return DBWrapper::DB()->DropPartitionByTag(table_name, partition_tag); + } +} + +} // namespace grpc +} // namespace server +} // namespace milvus diff --git a/core/src/server/grpc_impl/request/DropPartitionRequest.h b/core/src/server/grpc_impl/request/DropPartitionRequest.h new file mode 100644 index 0000000000..4ac5743555 --- /dev/null +++ b/core/src/server/grpc_impl/request/DropPartitionRequest.h @@ -0,0 +1,43 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include "server/grpc_impl/request/GrpcBaseRequest.h" + +namespace milvus { +namespace server { +namespace grpc { + +class DropPartitionRequest : public GrpcBaseRequest { + public: + static BaseRequestPtr + Create(const ::milvus::grpc::PartitionParam* partition_param); + + protected: + explicit DropPartitionRequest(const ::milvus::grpc::PartitionParam* partition_param); + + Status + OnExecute() override; + + private: + const ::milvus::grpc::PartitionParam* partition_param_; +}; + +} // namespace grpc +} // namespace server +} // namespace milvus diff --git a/core/src/server/grpc_impl/request/DropTableRequest.cpp b/core/src/server/grpc_impl/request/DropTableRequest.cpp new file mode 100644 index 0000000000..a678e5a6f6 --- /dev/null +++ b/core/src/server/grpc_impl/request/DropTableRequest.cpp @@ -0,0 +1,82 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "server/grpc_impl/request/DropTableRequest.h" +#include "server/DBWrapper.h" +#include "utils/Log.h" +#include "utils/TimeRecorder.h" +#include "utils/ValidationUtil.h" + +#include +#include + +namespace milvus { +namespace server { +namespace grpc { + +DropTableRequest::DropTableRequest(const std::string& table_name) + : GrpcBaseRequest(DDL_DML_REQUEST_GROUP), table_name_(table_name) { +} + +BaseRequestPtr +DropTableRequest::Create(const std::string& table_name) { + return std::shared_ptr(new DropTableRequest(table_name)); +} + +Status +DropTableRequest::OnExecute() { + try { + TimeRecorder rc("DropTableRequest"); + + // step 1: check arguments + auto status = ValidationUtil::ValidateTableName(table_name_); + if (!status.ok()) { + return status; + } + + // step 2: check table existence + engine::meta::TableSchema table_info; + table_info.table_id_ = table_name_; + status = DBWrapper::DB()->DescribeTable(table_info); + if (!status.ok()) { + if (status.code() == DB_NOT_FOUND) { + return Status(SERVER_TABLE_NOT_EXIST, TableNotExistMsg(table_name_)); + } else { + return status; + } + } + + rc.ElapseFromBegin("check validation"); + + // step 3: Drop table + std::vector dates; + status = DBWrapper::DB()->DropTable(table_name_, dates); + if (!status.ok()) { + return status; + } + + rc.ElapseFromBegin("total cost"); + } catch (std::exception& ex) { + return Status(SERVER_UNEXPECTED_ERROR, ex.what()); + } + + return Status::OK(); +} + +} // namespace grpc +} // namespace server +} // namespace milvus diff --git a/core/src/server/grpc_impl/request/DropTableRequest.h b/core/src/server/grpc_impl/request/DropTableRequest.h new file mode 100644 index 0000000000..7dd7eca209 --- /dev/null +++ b/core/src/server/grpc_impl/request/DropTableRequest.h @@ -0,0 +1,45 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include "server/grpc_impl/request/GrpcBaseRequest.h" + +#include + +namespace milvus { +namespace server { +namespace grpc { + +class DropTableRequest : public GrpcBaseRequest { + public: + static BaseRequestPtr + Create(const std::string& table_name); + + protected: + explicit DropTableRequest(const std::string& table_name); + + Status + OnExecute() override; + + private: + std::string table_name_; +}; + +} // namespace grpc +} // namespace server +} // namespace milvus diff --git a/core/src/server/grpc_impl/request/GrpcBaseRequest.cpp b/core/src/server/grpc_impl/request/GrpcBaseRequest.cpp new file mode 100644 index 0000000000..e1eb07af5c --- /dev/null +++ b/core/src/server/grpc_impl/request/GrpcBaseRequest.cpp @@ -0,0 +1,107 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "server/grpc_impl/request/GrpcBaseRequest.h" +#include "utils/CommonUtil.h" +#include "utils/Log.h" + +namespace milvus { +namespace server { +namespace grpc { + +constexpr int64_t DAY_SECONDS = 24 * 60 * 60; + +Status +ConvertTimeRangeToDBDates(const std::vector<::milvus::grpc::Range>& range_array, std::vector& dates) { + dates.clear(); + for (auto& range : range_array) { + time_t tt_start, tt_end; + tm tm_start, tm_end; + if (!CommonUtil::TimeStrToTime(range.start_value(), tt_start, tm_start)) { + return Status(SERVER_INVALID_TIME_RANGE, "Invalid time range: " + range.start_value()); + } + + if (!CommonUtil::TimeStrToTime(range.end_value(), tt_end, tm_end)) { + return Status(SERVER_INVALID_TIME_RANGE, "Invalid time range: " + range.start_value()); + } + + int64_t days = (tt_end - tt_start) / DAY_SECONDS; + if (days <= 0) { + return Status(SERVER_INVALID_TIME_RANGE, + "Invalid time range: The start-date should be smaller than end-date!"); + } + + // range: [start_day, end_day) + for (int64_t i = 0; i < days; i++) { + time_t tt_day = tt_start + DAY_SECONDS * i; + tm tm_day; + CommonUtil::ConvertTime(tt_day, tm_day); + + int64_t date = tm_day.tm_year * 10000 + tm_day.tm_mon * 100 + tm_day.tm_mday; // according to db logic + dates.push_back(date); + } + } + + return Status::OK(); +} + +GrpcBaseRequest::GrpcBaseRequest(const std::string& request_group, bool async) + : request_group_(request_group), async_(async), done_(false) { +} + +GrpcBaseRequest::~GrpcBaseRequest() { + WaitToFinish(); +} + +Status +GrpcBaseRequest::Execute() { + status_ = OnExecute(); + Done(); + return status_; +} + +void +GrpcBaseRequest::Done() { + done_ = true; + finish_cond_.notify_all(); +} + +Status +GrpcBaseRequest::SetStatus(ErrorCode error_code, const std::string& error_msg) { + status_ = Status(error_code, error_msg); + SERVER_LOG_ERROR << error_msg; + return status_; +} + +std::string +GrpcBaseRequest::TableNotExistMsg(const std::string& table_name) { + return "Table " + table_name + + " not exist. Use milvus.has_table to verify whether the table exists. You also can check if the table name " + "exists."; +} + +Status +GrpcBaseRequest::WaitToFinish() { + std::unique_lock lock(finish_mtx_); + finish_cond_.wait(lock, [this] { return done_; }); + + return status_; +} + +} // namespace grpc +} // namespace server +} // namespace milvus diff --git a/core/src/server/grpc_impl/request/GrpcBaseRequest.h b/core/src/server/grpc_impl/request/GrpcBaseRequest.h new file mode 100644 index 0000000000..b64662b2cf --- /dev/null +++ b/core/src/server/grpc_impl/request/GrpcBaseRequest.h @@ -0,0 +1,101 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include "db/meta/MetaTypes.h" +#include "grpc/gen-milvus/milvus.grpc.pb.h" +#include "grpc/gen-status/status.grpc.pb.h" +#include "grpc/gen-status/status.pb.h" +#include "utils/Status.h" + +#include +//#include +#include +#include +#include +#include + +namespace milvus { +namespace server { +namespace grpc { + +static const char* DQL_REQUEST_GROUP = "dql"; +static const char* DDL_DML_REQUEST_GROUP = "ddl_dml"; +static const char* INFO_REQUEST_GROUP = "info"; + +using DB_DATE = milvus::engine::meta::DateT; + +Status +ConvertTimeRangeToDBDates(const std::vector<::milvus::grpc::Range>& range_array, std::vector& dates); + +class GrpcBaseRequest { + protected: + explicit GrpcBaseRequest(const std::string& request_group, bool async = false); + + virtual ~GrpcBaseRequest(); + + public: + Status + Execute(); + + void + Done(); + + Status + WaitToFinish(); + + std::string + RequestGroup() const { + return request_group_; + } + + const Status& + status() const { + return status_; + } + + bool + IsAsync() const { + return async_; + } + + protected: + virtual Status + OnExecute() = 0; + + Status + SetStatus(ErrorCode error_code, const std::string& error_msg); + + std::string + TableNotExistMsg(const std::string& table_name); + + protected: + mutable std::mutex finish_mtx_; + std::condition_variable finish_cond_; + + std::string request_group_; + bool async_; + bool done_; + Status status_; +}; + +using BaseRequestPtr = std::shared_ptr; + +} // namespace grpc +} // namespace server +} // namespace milvus diff --git a/core/src/server/grpc_impl/request/HasTableRequest.cpp b/core/src/server/grpc_impl/request/HasTableRequest.cpp new file mode 100644 index 0000000000..2909c93056 --- /dev/null +++ b/core/src/server/grpc_impl/request/HasTableRequest.cpp @@ -0,0 +1,66 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "server/grpc_impl/request/HasTableRequest.h" +#include "server/DBWrapper.h" +#include "utils/Log.h" +#include "utils/TimeRecorder.h" +#include "utils/ValidationUtil.h" + +#include + +namespace milvus { +namespace server { +namespace grpc { + +HasTableRequest::HasTableRequest(const std::string& table_name, bool& has_table) + : GrpcBaseRequest(INFO_REQUEST_GROUP), table_name_(table_name), has_table_(has_table) { +} + +BaseRequestPtr +HasTableRequest::Create(const std::string& table_name, bool& has_table) { + return std::shared_ptr(new HasTableRequest(table_name, has_table)); +} + +Status +HasTableRequest::OnExecute() { + try { + TimeRecorder rc("HasTableRequest"); + + // step 1: check arguments + auto status = ValidationUtil::ValidateTableName(table_name_); + if (!status.ok()) { + return status; + } + + // step 2: check table existence + status = DBWrapper::DB()->HasTable(table_name_, has_table_); + if (!status.ok()) { + return status; + } + + rc.ElapseFromBegin("totally cost"); + } catch (std::exception& ex) { + return Status(SERVER_UNEXPECTED_ERROR, ex.what()); + } + + return Status::OK(); +} + +} // namespace grpc +} // namespace server +} // namespace milvus diff --git a/core/src/server/grpc_impl/request/HasTableRequest.h b/core/src/server/grpc_impl/request/HasTableRequest.h new file mode 100644 index 0000000000..85fa2b345b --- /dev/null +++ b/core/src/server/grpc_impl/request/HasTableRequest.h @@ -0,0 +1,46 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include "server/grpc_impl/request/GrpcBaseRequest.h" + +#include + +namespace milvus { +namespace server { +namespace grpc { + +class HasTableRequest : public GrpcBaseRequest { + public: + static BaseRequestPtr + Create(const std::string& table_name, bool& has_table); + + protected: + HasTableRequest(const std::string& table_name, bool& has_table); + + Status + OnExecute() override; + + private: + std::string table_name_; + bool& has_table_; +}; + +} // namespace grpc +} // namespace server +} // namespace milvus diff --git a/core/src/server/grpc_impl/request/InsertRequest.cpp b/core/src/server/grpc_impl/request/InsertRequest.cpp new file mode 100644 index 0000000000..f436db074e --- /dev/null +++ b/core/src/server/grpc_impl/request/InsertRequest.cpp @@ -0,0 +1,170 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "server/grpc_impl/request/InsertRequest.h" +#include "server/DBWrapper.h" +#include "utils/Log.h" +#include "utils/TimeRecorder.h" +#include "utils/ValidationUtil.h" + +#include +#include +#include + +namespace milvus { +namespace server { +namespace grpc { + +InsertRequest::InsertRequest(const ::milvus::grpc::InsertParam* insert_param, ::milvus::grpc::VectorIds* record_ids) + : GrpcBaseRequest(DDL_DML_REQUEST_GROUP), insert_param_(insert_param), record_ids_(record_ids) { +} + +BaseRequestPtr +InsertRequest::Create(const ::milvus::grpc::InsertParam* insert_param, ::milvus::grpc::VectorIds* record_ids) { + if (insert_param == nullptr) { + SERVER_LOG_ERROR << "grpc input is null!"; + return nullptr; + } + return std::shared_ptr(new InsertRequest(insert_param, record_ids)); +} + +Status +InsertRequest::OnExecute() { + try { + TimeRecorder rc("InsertRequest"); + + // step 1: check arguments + auto status = ValidationUtil::ValidateTableName(insert_param_->table_name()); + if (!status.ok()) { + return status; + } + if (insert_param_->row_record_array().empty()) { + return Status(SERVER_INVALID_ROWRECORD_ARRAY, + "The vector array is empty. Make sure you have entered vector records."); + } + + if (!insert_param_->row_id_array().empty()) { + if (insert_param_->row_id_array().size() != insert_param_->row_record_array_size()) { + return Status(SERVER_ILLEGAL_VECTOR_ID, + "The size of vector ID array must be equal to the size of the vector."); + } + } + + // step 2: check table existence + engine::meta::TableSchema table_info; + table_info.table_id_ = insert_param_->table_name(); + status = DBWrapper::DB()->DescribeTable(table_info); + if (!status.ok()) { + if (status.code() == DB_NOT_FOUND) { + return Status(SERVER_TABLE_NOT_EXIST, TableNotExistMsg(insert_param_->table_name())); + } else { + return status; + } + } + + // step 3: check table flag + // all user provide id, or all internal id + bool user_provide_ids = !insert_param_->row_id_array().empty(); + // user already provided id before, all insert action require user id + if ((table_info.flag_ & engine::meta::FLAG_MASK_HAS_USERID) != 0 && !user_provide_ids) { + return Status(SERVER_ILLEGAL_VECTOR_ID, + "Table vector IDs are user-defined. Please provide IDs for all vectors of this table."); + } + + // user didn't provided id before, no need to provide user id + if ((table_info.flag_ & engine::meta::FLAG_MASK_NO_USERID) != 0 && user_provide_ids) { + return Status( + SERVER_ILLEGAL_VECTOR_ID, + "Table vector IDs are auto-generated. All vectors of this table must use auto-generated IDs."); + } + + rc.RecordSection("check validation"); + +#ifdef MILVUS_ENABLE_PROFILING + std::string fname = + "/tmp/insert_" + std::to_string(this->insert_param_->row_record_array_size()) + ".profiling"; + ProfilerStart(fname.c_str()); +#endif + + // step 4: prepare float data + std::vector vec_f(insert_param_->row_record_array_size() * table_info.dimension_, 0); + + // TODO(yk): change to one dimension array or use multiple-thread to copy the data + for (size_t i = 0; i < insert_param_->row_record_array_size(); i++) { + if (insert_param_->row_record_array(i).vector_data().empty()) { + return Status(SERVER_INVALID_ROWRECORD_ARRAY, + "The vector dimension must be equal to the table dimension."); + } + uint64_t vec_dim = insert_param_->row_record_array(i).vector_data().size(); + if (vec_dim != table_info.dimension_) { + ErrorCode error_code = SERVER_INVALID_VECTOR_DIMENSION; + std::string error_msg = "The vector dimension must be equal to the table dimension."; + return Status(error_code, error_msg); + } + memcpy(&vec_f[i * table_info.dimension_], insert_param_->row_record_array(i).vector_data().data(), + table_info.dimension_ * sizeof(float)); + } + + rc.ElapseFromBegin("prepare vectors data"); + + // step 5: insert vectors + auto vec_count = static_cast(insert_param_->row_record_array_size()); + std::vector vec_ids(insert_param_->row_id_array_size(), 0); + if (!insert_param_->row_id_array().empty()) { + const int64_t* src_data = insert_param_->row_id_array().data(); + int64_t* target_data = vec_ids.data(); + memcpy(target_data, src_data, static_cast(sizeof(int64_t) * insert_param_->row_id_array_size())); + } + + status = DBWrapper::DB()->InsertVectors(insert_param_->table_name(), insert_param_->partition_tag(), vec_count, + vec_f.data(), vec_ids); + rc.ElapseFromBegin("add vectors to engine"); + if (!status.ok()) { + return status; + } + for (int64_t id : vec_ids) { + record_ids_->add_vector_id_array(id); + } + + auto ids_size = record_ids_->vector_id_array_size(); + if (ids_size != vec_count) { + std::string msg = + "Add " + std::to_string(vec_count) + " vectors but only return " + std::to_string(ids_size) + " id"; + return Status(SERVER_ILLEGAL_VECTOR_ID, msg); + } + + // step 6: update table flag + user_provide_ids ? table_info.flag_ |= engine::meta::FLAG_MASK_HAS_USERID + : table_info.flag_ |= engine::meta::FLAG_MASK_NO_USERID; + status = DBWrapper::DB()->UpdateTableFlag(insert_param_->table_name(), table_info.flag_); + +#ifdef MILVUS_ENABLE_PROFILING + ProfilerStop(); +#endif + + rc.RecordSection("add vectors to engine"); + rc.ElapseFromBegin("total cost"); + } catch (std::exception& ex) { + return Status(SERVER_UNEXPECTED_ERROR, ex.what()); + } + + return Status::OK(); +} + +} // namespace grpc +} // namespace server +} // namespace milvus diff --git a/core/src/server/grpc_impl/request/InsertRequest.h b/core/src/server/grpc_impl/request/InsertRequest.h new file mode 100644 index 0000000000..1582615d2e --- /dev/null +++ b/core/src/server/grpc_impl/request/InsertRequest.h @@ -0,0 +1,44 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include "server/grpc_impl/request/GrpcBaseRequest.h" + +namespace milvus { +namespace server { +namespace grpc { + +class InsertRequest : public GrpcBaseRequest { + public: + static BaseRequestPtr + Create(const ::milvus::grpc::InsertParam* insert_param, ::milvus::grpc::VectorIds* record_ids); + + protected: + InsertRequest(const ::milvus::grpc::InsertParam* insert_param, ::milvus::grpc::VectorIds* record_ids); + + Status + OnExecute() override; + + private: + const ::milvus::grpc::InsertParam* insert_param_; + ::milvus::grpc::VectorIds* record_ids_; +}; + +} // namespace grpc +} // namespace server +} // namespace milvus diff --git a/core/src/server/grpc_impl/request/PreloadTableRequest.cpp b/core/src/server/grpc_impl/request/PreloadTableRequest.cpp new file mode 100644 index 0000000000..e26aa8a877 --- /dev/null +++ b/core/src/server/grpc_impl/request/PreloadTableRequest.cpp @@ -0,0 +1,66 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "server/grpc_impl/request/PreloadTableRequest.h" +#include "server/DBWrapper.h" +#include "utils/Log.h" +#include "utils/TimeRecorder.h" +#include "utils/ValidationUtil.h" + +#include + +namespace milvus { +namespace server { +namespace grpc { + +PreloadTableRequest::PreloadTableRequest(const std::string& table_name) + : GrpcBaseRequest(DQL_REQUEST_GROUP), table_name_(table_name) { +} + +BaseRequestPtr +PreloadTableRequest::Create(const std::string& table_name) { + return std::shared_ptr(new PreloadTableRequest(table_name)); +} + +Status +PreloadTableRequest::OnExecute() { + try { + TimeRecorder rc("PreloadTableRequest"); + + // step 1: check arguments + auto status = ValidationUtil::ValidateTableName(table_name_); + if (!status.ok()) { + return status; + } + + // step 2: check table existence + status = DBWrapper::DB()->PreloadTable(table_name_); + if (!status.ok()) { + return status; + } + + rc.ElapseFromBegin("totally cost"); + } catch (std::exception& ex) { + return Status(SERVER_UNEXPECTED_ERROR, ex.what()); + } + + return Status::OK(); +} + +} // namespace grpc +} // namespace server +} // namespace milvus diff --git a/core/src/server/grpc_impl/request/PreloadTableRequest.h b/core/src/server/grpc_impl/request/PreloadTableRequest.h new file mode 100644 index 0000000000..208c7477ad --- /dev/null +++ b/core/src/server/grpc_impl/request/PreloadTableRequest.h @@ -0,0 +1,45 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include "server/grpc_impl/request/GrpcBaseRequest.h" + +#include + +namespace milvus { +namespace server { +namespace grpc { + +class PreloadTableRequest : public GrpcBaseRequest { + public: + static BaseRequestPtr + Create(const std::string& table_name); + + protected: + explicit PreloadTableRequest(const std::string& table_name); + + Status + OnExecute() override; + + private: + std::string table_name_; +}; + +} // namespace grpc +} // namespace server +} // namespace milvus diff --git a/core/src/server/grpc_impl/request/SearchRequest.cpp b/core/src/server/grpc_impl/request/SearchRequest.cpp new file mode 100644 index 0000000000..28f4ff723e --- /dev/null +++ b/core/src/server/grpc_impl/request/SearchRequest.cpp @@ -0,0 +1,188 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "server/grpc_impl/request/SearchRequest.h" +#include "server/DBWrapper.h" +#include "utils/Log.h" +#include "utils/TimeRecorder.h" +#include "utils/ValidationUtil.h" + +#include + +namespace milvus { +namespace server { +namespace grpc { + +SearchRequest::SearchRequest(const ::milvus::grpc::SearchParam* search_vector_infos, + const std::vector& file_id_array, ::milvus::grpc::TopKQueryResult* response) + : GrpcBaseRequest(DQL_REQUEST_GROUP), + search_param_(search_vector_infos), + file_id_array_(file_id_array), + topk_result_(response) { +} + +BaseRequestPtr +SearchRequest::Create(const ::milvus::grpc::SearchParam* search_vector_infos, + const std::vector& file_id_array, ::milvus::grpc::TopKQueryResult* response) { + if (search_vector_infos == nullptr) { + SERVER_LOG_ERROR << "grpc input is null!"; + return nullptr; + } + return std::shared_ptr(new SearchRequest(search_vector_infos, file_id_array, response)); +} + +Status +SearchRequest::OnExecute() { + try { + int64_t top_k = search_param_->topk(); + int64_t nprobe = search_param_->nprobe(); + + std::string hdr = "SearchRequest(k=" + std::to_string(top_k) + ", nprob=" + std::to_string(nprobe) + ")"; + TimeRecorder rc(hdr); + + // step 1: check table name + std::string table_name_ = search_param_->table_name(); + auto status = ValidationUtil::ValidateTableName(table_name_); + if (!status.ok()) { + return status; + } + + // step 2: check table existence + engine::meta::TableSchema table_info; + table_info.table_id_ = table_name_; + status = DBWrapper::DB()->DescribeTable(table_info); + if (!status.ok()) { + if (status.code() == DB_NOT_FOUND) { + return Status(SERVER_TABLE_NOT_EXIST, TableNotExistMsg(table_name_)); + } else { + return status; + } + } + + // step 3: check search parameter + status = ValidationUtil::ValidateSearchTopk(top_k, table_info); + if (!status.ok()) { + return status; + } + + status = ValidationUtil::ValidateSearchNprobe(nprobe, table_info); + if (!status.ok()) { + return status; + } + + if (search_param_->query_record_array().empty()) { + return Status(SERVER_INVALID_ROWRECORD_ARRAY, + "The vector array is empty. Make sure you have entered vector records."); + } + + // step 4: check date range, and convert to db dates + std::vector dates; + std::vector<::milvus::grpc::Range> range_array; + for (size_t i = 0; i < search_param_->query_range_array_size(); i++) { + range_array.emplace_back(search_param_->query_range_array(i)); + } + + status = ConvertTimeRangeToDBDates(range_array, dates); + if (!status.ok()) { + return status; + } + + rc.RecordSection("check validation"); + + // step 5: prepare float data + auto record_array_size = search_param_->query_record_array_size(); + std::vector vec_f(record_array_size * table_info.dimension_, 0); + for (size_t i = 0; i < record_array_size; i++) { + if (search_param_->query_record_array(i).vector_data().empty()) { + return Status(SERVER_INVALID_ROWRECORD_ARRAY, + "The vector dimension must be equal to the table dimension."); + } + uint64_t query_vec_dim = search_param_->query_record_array(i).vector_data().size(); + if (query_vec_dim != table_info.dimension_) { + ErrorCode error_code = SERVER_INVALID_VECTOR_DIMENSION; + std::string error_msg = "The vector dimension must be equal to the table dimension."; + return Status(error_code, error_msg); + } + + memcpy(&vec_f[i * table_info.dimension_], search_param_->query_record_array(i).vector_data().data(), + table_info.dimension_ * sizeof(float)); + } + rc.RecordSection("prepare vector data"); + + // step 6: search vectors + engine::ResultIds result_ids; + engine::ResultDistances result_distances; + auto record_count = (uint64_t)search_param_->query_record_array().size(); + +#ifdef MILVUS_ENABLE_PROFILING + std::string fname = + "/tmp/search_nq_" + std::to_string(this->search_param_->query_record_array_size()) + ".profiling"; + ProfilerStart(fname.c_str()); +#endif + + if (file_id_array_.empty()) { + std::vector partition_tags; + for (size_t i = 0; i < search_param_->partition_tag_array_size(); i++) { + partition_tags.emplace_back(search_param_->partition_tag_array(i)); + } + + status = ValidationUtil::ValidatePartitionTags(partition_tags); + if (!status.ok()) { + return status; + } + + status = DBWrapper::DB()->Query(table_name_, partition_tags, (size_t)top_k, record_count, nprobe, + vec_f.data(), dates, result_ids, result_distances); + } else { + status = DBWrapper::DB()->QueryByFileID(table_name_, file_id_array_, (size_t)top_k, record_count, nprobe, + vec_f.data(), dates, result_ids, result_distances); + } + +#ifdef MILVUS_ENABLE_PROFILING + ProfilerStop(); +#endif + + rc.RecordSection("search vectors from engine"); + if (!status.ok()) { + return status; + } + + if (result_ids.empty()) { + return Status::OK(); // empty table + } + + // step 7: construct result array + topk_result_->set_row_num(record_count); + topk_result_->mutable_ids()->Resize(static_cast(result_ids.size()), -1); + memcpy(topk_result_->mutable_ids()->mutable_data(), result_ids.data(), result_ids.size() * sizeof(int64_t)); + topk_result_->mutable_distances()->Resize(static_cast(result_distances.size()), 0.0); + memcpy(topk_result_->mutable_distances()->mutable_data(), result_distances.data(), + result_distances.size() * sizeof(float)); + + // step 8: print time cost percent + rc.RecordSection("construct result and send"); + rc.ElapseFromBegin("totally cost"); + } catch (std::exception& ex) { + return Status(SERVER_UNEXPECTED_ERROR, ex.what()); + } + + return Status::OK(); +} + +} // namespace grpc +} // namespace server +} // namespace milvus diff --git a/core/src/server/grpc_impl/request/SearchRequest.h b/core/src/server/grpc_impl/request/SearchRequest.h new file mode 100644 index 0000000000..667c7e62ee --- /dev/null +++ b/core/src/server/grpc_impl/request/SearchRequest.h @@ -0,0 +1,50 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include "server/grpc_impl/request/GrpcBaseRequest.h" + +#include +#include + +namespace milvus { +namespace server { +namespace grpc { + +class SearchRequest : public GrpcBaseRequest { + public: + static BaseRequestPtr + Create(const ::milvus::grpc::SearchParam* search_param, const std::vector& file_id_array, + ::milvus::grpc::TopKQueryResult* response); + + protected: + SearchRequest(const ::milvus::grpc::SearchParam* search_param, const std::vector& file_id_array, + ::milvus::grpc::TopKQueryResult* response); + + Status + OnExecute() override; + + private: + const ::milvus::grpc::SearchParam* search_param_; + std::vector file_id_array_; + ::milvus::grpc::TopKQueryResult* topk_result_; +}; + +} // namespace grpc +} // namespace server +} // namespace milvus diff --git a/core/src/server/grpc_impl/request/ShowPartitionsRequest.cpp b/core/src/server/grpc_impl/request/ShowPartitionsRequest.cpp new file mode 100644 index 0000000000..32fa0672c5 --- /dev/null +++ b/core/src/server/grpc_impl/request/ShowPartitionsRequest.cpp @@ -0,0 +1,65 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "server/grpc_impl/request/ShowPartitionsRequest.h" +#include "server/DBWrapper.h" +#include "utils/Log.h" +#include "utils/TimeRecorder.h" +#include "utils/ValidationUtil.h" + +#include +#include + +namespace milvus { +namespace server { +namespace grpc { + +ShowPartitionsRequest::ShowPartitionsRequest(const std::string& table_name, + ::milvus::grpc::PartitionList* partition_list) + : GrpcBaseRequest(INFO_REQUEST_GROUP), table_name_(table_name), partition_list_(partition_list) { +} + +BaseRequestPtr +ShowPartitionsRequest::Create(const std::string& table_name, ::milvus::grpc::PartitionList* partition_list) { + return std::shared_ptr(new ShowPartitionsRequest(table_name, partition_list)); +} + +Status +ShowPartitionsRequest::OnExecute() { + auto status = ValidationUtil::ValidateTableName(table_name_); + if (!status.ok()) { + return status; + } + + std::vector schema_array; + auto statuts = DBWrapper::DB()->ShowPartitions(table_name_, schema_array); + if (!statuts.ok()) { + return statuts; + } + + for (auto& schema : schema_array) { + ::milvus::grpc::PartitionParam* param = partition_list_->add_partition_array(); + param->set_table_name(schema.owner_table_); + param->set_partition_name(schema.table_id_); + param->set_tag(schema.partition_tag_); + } + return Status::OK(); +} + +} // namespace grpc +} // namespace server +} // namespace milvus diff --git a/core/src/server/grpc_impl/request/ShowPartitionsRequest.h b/core/src/server/grpc_impl/request/ShowPartitionsRequest.h new file mode 100644 index 0000000000..dd9768b9cb --- /dev/null +++ b/core/src/server/grpc_impl/request/ShowPartitionsRequest.h @@ -0,0 +1,46 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include "server/grpc_impl/request/GrpcBaseRequest.h" + +#include + +namespace milvus { +namespace server { +namespace grpc { + +class ShowPartitionsRequest : public GrpcBaseRequest { + public: + static BaseRequestPtr + Create(const std::string& table_name, ::milvus::grpc::PartitionList* partition_list); + + protected: + ShowPartitionsRequest(const std::string& table_name, ::milvus::grpc::PartitionList* partition_list); + + Status + OnExecute() override; + + private: + std::string table_name_; + ::milvus::grpc::PartitionList* partition_list_; +}; + +} // namespace grpc +} // namespace server +} // namespace milvus diff --git a/core/src/server/grpc_impl/request/ShowTablesRequest.cpp b/core/src/server/grpc_impl/request/ShowTablesRequest.cpp new file mode 100644 index 0000000000..90dbad981f --- /dev/null +++ b/core/src/server/grpc_impl/request/ShowTablesRequest.cpp @@ -0,0 +1,55 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "server/grpc_impl/request/ShowTablesRequest.h" +#include "server/DBWrapper.h" +#include "utils/Log.h" +#include "utils/TimeRecorder.h" + +#include +#include + +namespace milvus { +namespace server { +namespace grpc { + +ShowTablesRequest::ShowTablesRequest(::milvus::grpc::TableNameList* table_name_list) + : GrpcBaseRequest(INFO_REQUEST_GROUP), table_name_list_(table_name_list) { +} + +BaseRequestPtr +ShowTablesRequest::Create(::milvus::grpc::TableNameList* table_name_list) { + return std::shared_ptr(new ShowTablesRequest(table_name_list)); +} + +Status +ShowTablesRequest::OnExecute() { + std::vector schema_array; + auto statuts = DBWrapper::DB()->AllTables(schema_array); + if (!statuts.ok()) { + return statuts; + } + + for (auto& schema : schema_array) { + table_name_list_->add_table_names(schema.table_id_); + } + return Status::OK(); +} + +} // namespace grpc +} // namespace server +} // namespace milvus diff --git a/core/src/server/grpc_impl/request/ShowTablesRequest.h b/core/src/server/grpc_impl/request/ShowTablesRequest.h new file mode 100644 index 0000000000..4818f8bb9d --- /dev/null +++ b/core/src/server/grpc_impl/request/ShowTablesRequest.h @@ -0,0 +1,43 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include "server/grpc_impl/request/GrpcBaseRequest.h" + +namespace milvus { +namespace server { +namespace grpc { + +class ShowTablesRequest : public GrpcBaseRequest { + public: + static BaseRequestPtr + Create(::milvus::grpc::TableNameList* table_name_list); + + protected: + explicit ShowTablesRequest(::milvus::grpc::TableNameList* table_name_list); + + Status + OnExecute() override; + + private: + ::milvus::grpc::TableNameList* table_name_list_; +}; + +} // namespace grpc +} // namespace server +} // namespace milvus diff --git a/core/src/utils/BlockingQueue.inl b/core/src/utils/BlockingQueue.inl index c4318c5fc6..cf0356e467 100644 --- a/core/src/utils/BlockingQueue.inl +++ b/core/src/utils/BlockingQueue.inl @@ -15,32 +15,26 @@ // specific language governing permissions and limitations // under the License. - #pragma once - namespace milvus { namespace server { -template +template void -BlockingQueue::Put(const T &task) { +BlockingQueue::Put(const T& task) { std::unique_lock lock(mtx); - full_.wait(lock, [this] { - return (queue_.size() < capacity_); - }); + full_.wait(lock, [this] { return (queue_.size() < capacity_); }); queue_.push(task); empty_.notify_all(); } -template +template T BlockingQueue::Take() { std::unique_lock lock(mtx); - empty_.wait(lock, [this] { - return !queue_.empty(); - }); + empty_.wait(lock, [this] { return !queue_.empty(); }); T front(queue_.front()); queue_.pop(); @@ -48,51 +42,45 @@ BlockingQueue::Take() { return front; } -template +template size_t BlockingQueue::Size() { std::lock_guard lock(mtx); return queue_.size(); } -template +template T BlockingQueue::Front() { std::unique_lock lock(mtx); - empty_.wait(lock, [this] { - return !queue_.empty(); - }); + empty_.wait(lock, [this] { return !queue_.empty(); }); T front(queue_.front()); return front; } -template +template T BlockingQueue::Back() { std::unique_lock lock(mtx); - empty_.wait(lock, [this] { - return !queue_.empty(); - }); + empty_.wait(lock, [this] { return !queue_.empty(); }); T back(queue_.back()); return back; } -template +template bool BlockingQueue::Empty() { std::unique_lock lock(mtx); return queue_.empty(); } -template +template void BlockingQueue::SetCapacity(const size_t capacity) { capacity_ = (capacity > 0 ? capacity : capacity_); } -} // namespace server -} // namespace milvus - - +} // namespace server +} // namespace milvus diff --git a/core/src/utils/CommonUtil.cpp b/core/src/utils/CommonUtil.cpp index fbf3112aeb..26e43619fb 100644 --- a/core/src/utils/CommonUtil.cpp +++ b/core/src/utils/CommonUtil.cpp @@ -54,7 +54,7 @@ CommonUtil::GetSystemMemInfo(uint64_t& total_mem, uint64_t& free_mem) { } bool -CommonUtil::GetSystemAvailableThreads(uint32_t& thread_count) { +CommonUtil::GetSystemAvailableThreads(int64_t& thread_count) { // threadCnt = std::thread::hardware_concurrency(); thread_count = sysconf(_SC_NPROCESSORS_CONF); thread_count *= THREAD_MULTIPLY_CPU; diff --git a/core/src/utils/CommonUtil.h b/core/src/utils/CommonUtil.h index 939bdd6d31..121196986a 100644 --- a/core/src/utils/CommonUtil.h +++ b/core/src/utils/CommonUtil.h @@ -30,7 +30,7 @@ class CommonUtil { static bool GetSystemMemInfo(uint64_t& total_mem, uint64_t& free_mem); static bool - GetSystemAvailableThreads(uint32_t& thread_count); + GetSystemAvailableThreads(int64_t& thread_count); static bool IsFileExist(const std::string& path); diff --git a/core/src/utils/Json.h b/core/src/utils/Json.h index e5e74a3b67..235d6ac9c1 100644 --- a/core/src/utils/Json.h +++ b/core/src/utils/Json.h @@ -16,7 +16,7 @@ // under the License. #pragma once -#include "external/nlohmann/json.hpp" +#include "nlohmann/json.hpp" namespace milvus { using json = nlohmann::json; diff --git a/core/src/utils/Log.h b/core/src/utils/Log.h index 4aa3146b01..cef72a866d 100644 --- a/core/src/utils/Log.h +++ b/core/src/utils/Log.h @@ -17,7 +17,7 @@ #pragma once -#include "external/easyloggingpp/easylogging++.h" +#include "easyloggingpp/easylogging++.h" namespace milvus { diff --git a/core/src/utils/LogUtil.h b/core/src/utils/LogUtil.h index 7e5afd087a..69607c4631 100644 --- a/core/src/utils/LogUtil.h +++ b/core/src/utils/LogUtil.h @@ -17,7 +17,7 @@ #pragma once -#include "external/easyloggingpp/easylogging++.h" +#include "easyloggingpp/easylogging++.h" #include "utils/Status.h" #include diff --git a/core/src/utils/StringHelpFunctions.cpp b/core/src/utils/StringHelpFunctions.cpp index 2db37829d6..af5b2e3b4d 100644 --- a/core/src/utils/StringHelpFunctions.cpp +++ b/core/src/utils/StringHelpFunctions.cpp @@ -17,6 +17,7 @@ #include "utils/StringHelpFunctions.h" +#include #include namespace milvus { @@ -136,5 +137,22 @@ StringHelpFunctions::SplitStringByQuote(const std::string& str, const std::strin return Status::OK(); } +bool +StringHelpFunctions::IsRegexMatch(const std::string& target_str, const std::string& pattern_str) { + // if target_str equals pattern_str, return true + if (target_str == pattern_str) { + return true; + } + + // regex match + std::regex pattern(pattern_str); + std::smatch results; + if (std::regex_search(target_str, results, pattern)) { + return true; + } else { + return false; + } +} + } // namespace server } // namespace milvus diff --git a/core/src/utils/StringHelpFunctions.h b/core/src/utils/StringHelpFunctions.h index 0cb303bb4f..3a41e53f4b 100644 --- a/core/src/utils/StringHelpFunctions.h +++ b/core/src/utils/StringHelpFunctions.h @@ -59,6 +59,11 @@ class StringHelpFunctions { static Status SplitStringByQuote(const std::string& str, const std::string& delimeter, const std::string& quote, std::vector& result); + + // std regex match function + // regex grammar reference: http://www.cplusplus.com/reference/regex/ECMAScript/ + static bool + IsRegexMatch(const std::string& target_str, const std::string& pattern); }; } // namespace server diff --git a/core/src/utils/ValidationUtil.cpp b/core/src/utils/ValidationUtil.cpp index 68088d6c93..12b2372fc5 100644 --- a/core/src/utils/ValidationUtil.cpp +++ b/core/src/utils/ValidationUtil.cpp @@ -18,9 +18,12 @@ #include "utils/ValidationUtil.h" #include "Log.h" #include "db/engine/ExecutionEngine.h" +#include "utils/StringHelpFunctions.h" #include +#ifdef MILVUS_GPU_VERSION #include +#endif #include #include #include @@ -167,7 +170,36 @@ ValidationUtil::ValidateSearchNprobe(int64_t nprobe, const engine::meta::TableSc } Status -ValidationUtil::ValidateGpuIndex(uint32_t gpu_index) { +ValidationUtil::ValidatePartitionName(const std::string& partition_name) { + if (partition_name.empty()) { + std::string msg = "Partition name should not be empty."; + SERVER_LOG_ERROR << msg; + return Status(SERVER_INVALID_TABLE_NAME, msg); + } + + return ValidateTableName(partition_name); +} + +Status +ValidationUtil::ValidatePartitionTags(const std::vector& partition_tags) { + for (const std::string& tag : partition_tags) { + // trim side-blank of tag, only compare valid characters + // for example: " ab cd " is treated as "ab cd" + std::string valid_tag = tag; + StringHelpFunctions::TrimStringBlank(valid_tag); + if (valid_tag.empty()) { + std::string msg = "Invalid partition tag: " + valid_tag + ". " + "Partition tag should not be empty."; + SERVER_LOG_ERROR << msg; + return Status(SERVER_INVALID_NPROBE, msg); + } + } + + return Status::OK(); +} + +Status +ValidationUtil::ValidateGpuIndex(int32_t gpu_index) { +#ifdef MILVUS_GPU_VERSION int num_devices = 0; auto cuda_err = cudaGetDeviceCount(&num_devices); if (cuda_err != cudaSuccess) { @@ -181,21 +213,27 @@ ValidationUtil::ValidateGpuIndex(uint32_t gpu_index) { SERVER_LOG_ERROR << msg; return Status(SERVER_INVALID_ARGUMENT, msg); } +#endif return Status::OK(); } Status -ValidationUtil::GetGpuMemory(uint32_t gpu_index, size_t& memory) { +ValidationUtil::GetGpuMemory(int32_t gpu_index, size_t& memory) { +#ifdef MILVUS_GPU_VERSION + cudaDeviceProp deviceProp; auto cuda_err = cudaGetDeviceProperties(&deviceProp, gpu_index); if (cuda_err) { - std::string msg = "Failed to get gpu properties, cuda error:" + std::to_string(cuda_err); + std::string msg = "Failed to get gpu properties for gpu" + std::to_string(gpu_index) + + " , cuda error:" + std::to_string(cuda_err); SERVER_LOG_ERROR << msg; return Status(SERVER_UNEXPECTED_ERROR, msg); } memory = deviceProp.totalGlobalMem; +#endif + return Status::OK(); } diff --git a/core/src/utils/ValidationUtil.h b/core/src/utils/ValidationUtil.h index 7b24c93fb5..ab32c35c40 100644 --- a/core/src/utils/ValidationUtil.h +++ b/core/src/utils/ValidationUtil.h @@ -21,6 +21,7 @@ #include "utils/Status.h" #include +#include namespace milvus { namespace server { @@ -55,10 +56,16 @@ class ValidationUtil { ValidateSearchNprobe(int64_t nprobe, const engine::meta::TableSchema& table_schema); static Status - ValidateGpuIndex(uint32_t gpu_index); + ValidatePartitionName(const std::string& partition_name); static Status - GetGpuMemory(uint32_t gpu_index, size_t& memory); + ValidatePartitionTags(const std::vector& partition_tags); + + static Status + ValidateGpuIndex(int32_t gpu_index); + + static Status + GetGpuMemory(int32_t gpu_index, size_t& memory); static Status ValidateIpAddress(const std::string& ip_address); diff --git a/core/src/version.h.in b/core/src/version.h.in new file mode 100644 index 0000000000..989010cde8 --- /dev/null +++ b/core/src/version.h.in @@ -0,0 +1,20 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#cmakedefine MILVUS_VERSION "@MILVUS_VERSION@" +#cmakedefine BUILD_TYPE "@BUILD_TYPE@" +#cmakedefine BUILD_TIME @BUILD_TIME@ \ No newline at end of file diff --git a/core/src/wrapper/ConfAdapter.cpp b/core/src/wrapper/ConfAdapter.cpp index 4ac8e22f52..aa4b3c12b8 100644 --- a/core/src/wrapper/ConfAdapter.cpp +++ b/core/src/wrapper/ConfAdapter.cpp @@ -16,6 +16,7 @@ // under the License. #include "wrapper/ConfAdapter.h" +#include "WrapperException.h" #include "knowhere/index/vector_index/helpers/IndexParameter.h" #include "utils/Log.h" @@ -46,7 +47,8 @@ ConfAdapter::Match(const TempMetaConf& metaconf) { auto conf = std::make_shared(); conf->d = metaconf.dim; conf->metric_type = metaconf.metric_type; - conf->gpu_id = conf->gpu_id; + conf->gpu_id = metaconf.gpu_id; + conf->k = metaconf.k; MatchBase(conf); return conf; } @@ -64,7 +66,7 @@ IVFConfAdapter::Match(const TempMetaConf& metaconf) { conf->nlist = MatchNlist(metaconf.size, metaconf.nlist); conf->d = metaconf.dim; conf->metric_type = metaconf.metric_type; - conf->gpu_id = conf->gpu_id; + conf->gpu_id = metaconf.gpu_id; MatchBase(conf); return conf; } @@ -76,7 +78,7 @@ IVFConfAdapter::MatchNlist(const int64_t& size, const int64_t& nlist) { if (size <= TYPICAL_COUNT / 16384 + 1) { // handle less row count, avoid nlist set to 0 return 1; - } else if (int(size / TYPICAL_COUNT) * nlist == 0) { + } else if (int(size / TYPICAL_COUNT) * nlist <= 0) { // calculate a proper nlist if nlist not specified or size less than TYPICAL_COUNT return int(size / TYPICAL_COUNT * 16384); } @@ -87,7 +89,11 @@ knowhere::Config IVFConfAdapter::MatchSearch(const TempMetaConf& metaconf, const IndexType& type) { auto conf = std::make_shared(); conf->k = metaconf.k; - conf->nprobe = metaconf.nprobe; + + if (metaconf.nprobe <= 0) + conf->nprobe = 16; // hardcode here + else + conf->nprobe = metaconf.nprobe; switch (type) { case IndexType::FAISS_IVFFLAT_GPU: @@ -121,33 +127,66 @@ IVFPQConfAdapter::Match(const TempMetaConf& metaconf) { conf->nlist = MatchNlist(metaconf.size, metaconf.nlist); conf->d = metaconf.dim; conf->metric_type = metaconf.metric_type; - conf->gpu_id = conf->gpu_id; + conf->gpu_id = metaconf.gpu_id; conf->nbits = 8; - conf->m = 8; + + if (!(conf->d % 4)) + conf->m = conf->d / 4; // compression radio = 16 + else if (!(conf->d % 2)) + conf->m = conf->d / 2; // compression radio = 8 + else if (!(conf->d % 3)) + conf->m = conf->d / 3; // compression radio = 12 + else + conf->m = conf->d; // same as SQ8, compression radio = 4 + MatchBase(conf); return conf; } +knowhere::Config +IVFPQConfAdapter::MatchSearch(const TempMetaConf& metaconf, const IndexType& type) { + auto conf = std::make_shared(); + conf->k = metaconf.k; + + if (metaconf.nprobe <= 0) { + WRAPPER_LOG_ERROR << "The nprobe of PQ is wrong!"; + throw WrapperException("The nprobe of PQ is wrong!"); + } else { + conf->nprobe = metaconf.nprobe; + } + + return conf; +} + +int64_t +IVFPQConfAdapter::MatchNlist(const int64_t& size, const int64_t& nlist) { + if (size <= TYPICAL_COUNT / 16384 + 1) { + // handle less row count, avoid nlist set to 0 + return 1; + } else if (int(size / TYPICAL_COUNT) * nlist <= 0) { + // calculate a proper nlist if nlist not specified or size less than TYPICAL_COUNT + return int(size / TYPICAL_COUNT * 16384); + } + return nlist; +} + knowhere::Config NSGConfAdapter::Match(const TempMetaConf& metaconf) { auto conf = std::make_shared(); conf->nlist = MatchNlist(metaconf.size, metaconf.nlist); conf->d = metaconf.dim; conf->metric_type = metaconf.metric_type; - conf->gpu_id = conf->gpu_id; + conf->gpu_id = metaconf.gpu_id; + conf->k = metaconf.k; auto scale_factor = round(metaconf.dim / 128.0); scale_factor = scale_factor >= 4 ? 4 : scale_factor; - conf->nprobe = 6 + 10 * scale_factor; - conf->knng = 100 + 100 * scale_factor; + conf->nprobe = int64_t(conf->nlist * 0.01); + conf->knng = 40 + 10 * scale_factor; // the size of knng conf->search_length = 40 + 5 * scale_factor; conf->out_degree = 50 + 5 * scale_factor; conf->candidate_pool_size = 200 + 100 * scale_factor; MatchBase(conf); - - // WRAPPER_LOG_DEBUG << "nlist: " << conf->nlist - // << ", gpu_id: " << conf->gpu_id << ", d: " << conf->d - // << ", nprobe: " << conf->nprobe << ", knng: " << conf->knng; return conf; } @@ -156,6 +195,39 @@ NSGConfAdapter::MatchSearch(const TempMetaConf& metaconf, const IndexType& type) auto conf = std::make_shared(); conf->k = metaconf.k; conf->search_length = metaconf.search_length; + if (metaconf.search_length == TEMPMETA_DEFAULT_VALUE) { + conf->search_length = 30; // TODO(linxj): hardcode here. + } + return conf; +} + +knowhere::Config +SPTAGKDTConfAdapter::Match(const TempMetaConf& metaconf) { + auto conf = std::make_shared(); + conf->d = metaconf.dim; + conf->metric_type = metaconf.metric_type; + return conf; +} + +knowhere::Config +SPTAGKDTConfAdapter::MatchSearch(const TempMetaConf& metaconf, const IndexType& type) { + auto conf = std::make_shared(); + conf->k = metaconf.k; + return conf; +} + +knowhere::Config +SPTAGBKTConfAdapter::Match(const TempMetaConf& metaconf) { + auto conf = std::make_shared(); + conf->d = metaconf.dim; + conf->metric_type = metaconf.metric_type; + return conf; +} + +knowhere::Config +SPTAGBKTConfAdapter::MatchSearch(const TempMetaConf& metaconf, const IndexType& type) { + auto conf = std::make_shared(); + conf->k = metaconf.k; return conf; } diff --git a/core/src/wrapper/ConfAdapter.h b/core/src/wrapper/ConfAdapter.h index 4c8e528a2d..46fc27eb3b 100644 --- a/core/src/wrapper/ConfAdapter.h +++ b/core/src/wrapper/ConfAdapter.h @@ -46,9 +46,6 @@ class ConfAdapter { virtual knowhere::Config MatchSearch(const TempMetaConf& metaconf, const IndexType& type); - // virtual void - // Dump(){} - protected: static void MatchBase(knowhere::Config conf); @@ -79,6 +76,13 @@ class IVFPQConfAdapter : public IVFConfAdapter { public: knowhere::Config Match(const TempMetaConf& metaconf) override; + + knowhere::Config + MatchSearch(const TempMetaConf& metaconf, const IndexType& type) override; + + protected: + static int64_t + MatchNlist(const int64_t& size, const int64_t& nlist); }; class NSGConfAdapter : public IVFConfAdapter { @@ -90,5 +94,23 @@ class NSGConfAdapter : public IVFConfAdapter { MatchSearch(const TempMetaConf& metaconf, const IndexType& type) final; }; +class SPTAGKDTConfAdapter : public ConfAdapter { + public: + knowhere::Config + Match(const TempMetaConf& metaconf) override; + + knowhere::Config + MatchSearch(const TempMetaConf& metaconf, const IndexType& type) override; +}; + +class SPTAGBKTConfAdapter : public ConfAdapter { + public: + knowhere::Config + Match(const TempMetaConf& metaconf) override; + + knowhere::Config + MatchSearch(const TempMetaConf& metaconf, const IndexType& type) override; +}; + } // namespace engine } // namespace milvus diff --git a/core/src/wrapper/ConfAdapterMgr.cpp b/core/src/wrapper/ConfAdapterMgr.cpp index b329588cf2..cf58c0110f 100644 --- a/core/src/wrapper/ConfAdapterMgr.cpp +++ b/core/src/wrapper/ConfAdapterMgr.cpp @@ -53,8 +53,12 @@ AdapterMgr::RegisterAdapter() { REGISTER_CONF_ADAPTER(IVFPQConfAdapter, IndexType::FAISS_IVFPQ_CPU, ivfpq_cpu); REGISTER_CONF_ADAPTER(IVFPQConfAdapter, IndexType::FAISS_IVFPQ_GPU, ivfpq_gpu); + REGISTER_CONF_ADAPTER(IVFPQConfAdapter, IndexType::FAISS_IVFPQ_MIX, ivfpq_mix); REGISTER_CONF_ADAPTER(NSGConfAdapter, IndexType::NSG_MIX, nsg_mix); + + REGISTER_CONF_ADAPTER(SPTAGKDTConfAdapter, IndexType::SPTAG_KDT_RNT_CPU, sptag_kdt); + REGISTER_CONF_ADAPTER(SPTAGBKTConfAdapter, IndexType::SPTAG_BKT_RNT_CPU, sptag_bkt); } } // namespace engine diff --git a/core/src/wrapper/KnowhereResource.cpp b/core/src/wrapper/KnowhereResource.cpp index 650ae727c1..5a2296b16e 100644 --- a/core/src/wrapper/KnowhereResource.cpp +++ b/core/src/wrapper/KnowhereResource.cpp @@ -16,7 +16,11 @@ // under the License. #include "wrapper/KnowhereResource.h" +#ifdef MILVUS_GPU_VERSION #include "knowhere/index/vector_index/helpers/FaissGpuResourceMgr.h" +#endif + +#include "scheduler/Utils.h" #include "server/Config.h" #include @@ -32,6 +36,7 @@ constexpr int64_t M_BYTE = 1024 * 1024; Status KnowhereResource::Initialize() { +#ifdef MILVUS_GPU_VERSION struct GpuResourceSetting { int64_t pinned_memory = 300 * M_BYTE; int64_t temp_memory = 300 * M_BYTE; @@ -43,27 +48,22 @@ KnowhereResource::Initialize() { // get build index gpu resource server::Config& config = server::Config::GetInstance(); - - int32_t build_index_gpu; - s = config.GetResourceConfigIndexBuildDevice(build_index_gpu); + std::vector build_index_gpus; + s = config.GetGpuResourceConfigBuildIndexResources(build_index_gpus); if (!s.ok()) return s; - gpu_resources.insert(std::make_pair(build_index_gpu, GpuResourceSetting())); + for (auto gpu_id : build_index_gpus) { + gpu_resources.insert(std::make_pair(gpu_id, GpuResourceSetting())); + } // get search gpu resource - std::vector pool; - s = config.GetResourceConfigSearchResources(pool); + std::vector search_gpus; + s = config.GetGpuResourceConfigSearchResources(search_gpus); if (!s.ok()) return s; - std::set gpu_ids; - for (auto& resource : pool) { - if (resource.length() < 4 || resource.substr(0, 3) != "gpu") { - // invalid - continue; - } - auto gpu_id = std::stoi(resource.substr(3)); + for (auto& gpu_id : search_gpus) { gpu_resources.insert(std::make_pair(gpu_id, GpuResourceSetting())); } @@ -73,12 +73,16 @@ KnowhereResource::Initialize() { iter->second.temp_memory, iter->second.resource_num); } +#endif + return Status::OK(); } Status KnowhereResource::Finalize() { +#ifdef MILVUS_GPU_VERSION knowhere::FaissGpuResourceMgr::GetInstance().Free(); // free gpu resource. +#endif return Status::OK(); } diff --git a/core/src/wrapper/VecImpl.cpp b/core/src/wrapper/VecImpl.cpp index 931eef9dbf..74e9e94a2f 100644 --- a/core/src/wrapper/VecImpl.cpp +++ b/core/src/wrapper/VecImpl.cpp @@ -18,11 +18,18 @@ #include "wrapper/VecImpl.h" #include "DataTransfer.h" #include "knowhere/common/Exception.h" -#include "knowhere/index/vector_index/IndexGPUIVF.h" #include "knowhere/index/vector_index/IndexIDMAP.h" +#include "utils/Log.h" +#include "wrapper/WrapperException.h" +#include "wrapper/gpu/GPUVecImpl.h" + +#ifdef MILVUS_GPU_VERSION + +#include "knowhere/index/vector_index/IndexGPUIVF.h" #include "knowhere/index/vector_index/IndexIVFSQHybrid.h" #include "knowhere/index/vector_index/helpers/Cloner.h" -#include "utils/Log.h" + +#endif /* * no parameter check in this layer. @@ -79,8 +86,8 @@ VecIndexImpl::Search(const int64_t& nq, const float* xq, float* dist, int64_t* i Config search_cfg = cfg; auto res = index_->Search(dataset, search_cfg); - auto ids_array = res->array()[0]; - auto dis_array = res->array()[1]; + // auto ids_array = res->array()[0]; + // auto dis_array = res->array()[1]; //{ // auto& ids = ids_array; @@ -99,12 +106,14 @@ VecIndexImpl::Search(const int64_t& nq, const float* xq, float* dist, int64_t* i // std::cout << "dist\n" << ss_dist.str() << std::endl; //} - auto p_ids = ids_array->data()->GetValues(1, 0); - auto p_dist = dis_array->data()->GetValues(1, 0); + // auto p_ids = ids_array->data()->GetValues(1, 0); + // auto p_dist = dis_array->data()->GetValues(1, 0); // TODO(linxj): avoid copy here. - memcpy(ids, p_ids, sizeof(int64_t) * nq * k); - memcpy(dist, p_dist, sizeof(float) * nq * k); + memcpy(ids, res->ids(), sizeof(int64_t) * nq * k); + memcpy(dist, res->dist(), sizeof(float) * nq * k); + free(res->ids()); + free(res->dist()); } catch (knowhere::KnowhereException& e) { WRAPPER_LOG_ERROR << e.what(); return Status(KNOWHERE_UNEXPECTED_ERROR, e.what()); @@ -146,19 +155,29 @@ VecIndexImpl::GetType() { VecIndexPtr VecIndexImpl::CopyToGpu(const int64_t& device_id, const Config& cfg) { // TODO(linxj): exception handle +#ifdef MILVUS_GPU_VERSION auto gpu_index = knowhere::cloner::CopyCpuToGpu(index_, device_id, cfg); auto new_index = std::make_shared(gpu_index, ConvertToGpuIndexType(type)); new_index->dim = dim; return new_index; +#else + WRAPPER_LOG_ERROR << "Calling VecIndexImpl::CopyToGpu when we are using CPU version"; + throw WrapperException("Calling VecIndexImpl::CopyToGpu when we are using CPU version"); +#endif } VecIndexPtr VecIndexImpl::CopyToCpu(const Config& cfg) { // TODO(linxj): exception handle +#ifdef MILVUS_GPU_VERSION auto cpu_index = knowhere::cloner::CopyGpuToCpu(index_, cfg); auto new_index = std::make_shared(cpu_index, ConvertToCpuIndexType(type)); new_index->dim = dim; return new_index; +#else + WRAPPER_LOG_ERROR << "Calling VecIndexImpl::CopyToCpu when we are using CPU version"; + throw WrapperException("Calling VecIndexImpl::CopyToCpu when we are using CPU version"); +#endif } VecIndexPtr @@ -171,11 +190,14 @@ VecIndexImpl::Clone() { int64_t VecIndexImpl::GetDeviceId() { +#ifdef MILVUS_GPU_VERSION if (auto device_idx = std::dynamic_pointer_cast(index_)) { return device_idx->GetGpuDevice(); } +#else // else return -1; // -1 == cpu +#endif } float* @@ -226,130 +248,5 @@ BFIndex::BuildAll(const int64_t& nb, const float* xb, const int64_t* ids, const return Status::OK(); } -// TODO(linxj): add lock here. -Status -IVFMixIndex::BuildAll(const int64_t& nb, const float* xb, const int64_t* ids, const Config& cfg, const int64_t& nt, - const float* xt) { - try { - dim = cfg->d; - auto dataset = GenDatasetWithIds(nb, dim, xb, ids); - - auto preprocessor = index_->BuildPreprocessor(dataset, cfg); - index_->set_preprocessor(preprocessor); - auto model = index_->Train(dataset, cfg); - index_->set_index_model(model); - index_->Add(dataset, cfg); - - if (auto device_index = std::dynamic_pointer_cast(index_)) { - auto host_index = device_index->CopyGpuToCpu(Config()); - index_ = host_index; - type = ConvertToCpuIndexType(type); - } else { - WRAPPER_LOG_ERROR << "Build IVFMIXIndex Failed"; - return Status(KNOWHERE_ERROR, "Build IVFMIXIndex Failed"); - } - } catch (knowhere::KnowhereException& e) { - WRAPPER_LOG_ERROR << e.what(); - return Status(KNOWHERE_UNEXPECTED_ERROR, e.what()); - } catch (std::exception& e) { - WRAPPER_LOG_ERROR << e.what(); - return Status(KNOWHERE_ERROR, e.what()); - } - return Status::OK(); -} - -Status -IVFMixIndex::Load(const knowhere::BinarySet& index_binary) { - index_->Load(index_binary); - dim = Dimension(); - return Status::OK(); -} - -knowhere::QuantizerPtr -IVFHybridIndex::LoadQuantizer(const Config& conf) { - // TODO(linxj): Hardcode here - if (auto new_idx = std::dynamic_pointer_cast(index_)) { - return new_idx->LoadQuantizer(conf); - } else { - WRAPPER_LOG_ERROR << "Hybrid mode not supported for index type: " << int(type); - } -} - -Status -IVFHybridIndex::SetQuantizer(const knowhere::QuantizerPtr& q) { - try { - // TODO(linxj): Hardcode here - if (auto new_idx = std::dynamic_pointer_cast(index_)) { - new_idx->SetQuantizer(q); - } else { - WRAPPER_LOG_ERROR << "Hybrid mode not supported for index type: " << int(type); - return Status(KNOWHERE_ERROR, "not supported"); - } - } catch (knowhere::KnowhereException& e) { - WRAPPER_LOG_ERROR << e.what(); - return Status(KNOWHERE_UNEXPECTED_ERROR, e.what()); - } catch (std::exception& e) { - WRAPPER_LOG_ERROR << e.what(); - return Status(KNOWHERE_ERROR, e.what()); - } - return Status::OK(); -} - -Status -IVFHybridIndex::UnsetQuantizer() { - try { - // TODO(linxj): Hardcode here - if (auto new_idx = std::dynamic_pointer_cast(index_)) { - new_idx->UnsetQuantizer(); - } else { - WRAPPER_LOG_ERROR << "Hybrid mode not supported for index type: " << int(type); - return Status(KNOWHERE_ERROR, "not supported"); - } - } catch (knowhere::KnowhereException& e) { - WRAPPER_LOG_ERROR << e.what(); - return Status(KNOWHERE_UNEXPECTED_ERROR, e.what()); - } catch (std::exception& e) { - WRAPPER_LOG_ERROR << e.what(); - return Status(KNOWHERE_ERROR, e.what()); - } - return Status::OK(); -} - -VecIndexPtr -IVFHybridIndex::LoadData(const knowhere::QuantizerPtr& q, const Config& conf) { - try { - // TODO(linxj): Hardcode here - if (auto new_idx = std::dynamic_pointer_cast(index_)) { - return std::make_shared(new_idx->LoadData(q, conf), type); - } else { - WRAPPER_LOG_ERROR << "Hybrid mode not supported for index type: " << int(type); - } - } catch (knowhere::KnowhereException& e) { - WRAPPER_LOG_ERROR << e.what(); - } catch (std::exception& e) { - WRAPPER_LOG_ERROR << e.what(); - } - return nullptr; -} - -std::pair -IVFHybridIndex::CopyToGpuWithQuantizer(const int64_t& device_id, const Config& cfg) { - try { - // TODO(linxj): Hardcode here - if (auto hybrid_idx = std::dynamic_pointer_cast(index_)) { - auto pair = hybrid_idx->CopyCpuToGpuWithQuantizer(device_id, cfg); - auto new_idx = std::make_shared(pair.first, type); - return std::make_pair(new_idx, pair.second); - } else { - WRAPPER_LOG_ERROR << "Hybrid mode not supported for index type: " << int(type); - } - } catch (knowhere::KnowhereException& e) { - WRAPPER_LOG_ERROR << e.what(); - } catch (std::exception& e) { - WRAPPER_LOG_ERROR << e.what(); - } - return std::make_pair(nullptr, nullptr); -} - } // namespace engine } // namespace milvus diff --git a/core/src/wrapper/VecImpl.h b/core/src/wrapper/VecImpl.h index 22d734cf92..2b6f07827e 100644 --- a/core/src/wrapper/VecImpl.h +++ b/core/src/wrapper/VecImpl.h @@ -77,41 +77,6 @@ class VecIndexImpl : public VecIndex { std::shared_ptr index_ = nullptr; }; -class IVFMixIndex : public VecIndexImpl { - public: - explicit IVFMixIndex(std::shared_ptr index, const IndexType& type) - : VecIndexImpl(std::move(index), type) { - } - - Status - BuildAll(const int64_t& nb, const float* xb, const int64_t* ids, const Config& cfg, const int64_t& nt, - const float* xt) override; - - Status - Load(const knowhere::BinarySet& index_binary) override; -}; - -class IVFHybridIndex : public IVFMixIndex { - public: - explicit IVFHybridIndex(std::shared_ptr index, const IndexType& type) - : IVFMixIndex(std::move(index), type) { - } - - knowhere::QuantizerPtr - LoadQuantizer(const Config& conf) override; - - Status - SetQuantizer(const knowhere::QuantizerPtr& q) override; - - Status - UnsetQuantizer() override; - std::pair - CopyToGpuWithQuantizer(const int64_t& device_id, const Config& cfg) override; - - VecIndexPtr - LoadData(const knowhere::QuantizerPtr& q, const Config& conf) override; -}; - class BFIndex : public VecIndexImpl { public: explicit BFIndex(std::shared_ptr index) diff --git a/core/src/wrapper/VecIndex.cpp b/core/src/wrapper/VecIndex.cpp index abf97e69e5..75f75fb983 100644 --- a/core/src/wrapper/VecIndex.cpp +++ b/core/src/wrapper/VecIndex.cpp @@ -18,19 +18,23 @@ #include "wrapper/VecIndex.h" #include "VecImpl.h" #include "knowhere/common/Exception.h" -#include "knowhere/index/vector_index/IndexGPUIVF.h" -#include "knowhere/index/vector_index/IndexGPUIVFPQ.h" -#include "knowhere/index/vector_index/IndexGPUIVFSQ.h" #include "knowhere/index/vector_index/IndexIDMAP.h" #include "knowhere/index/vector_index/IndexIVF.h" #include "knowhere/index/vector_index/IndexIVFPQ.h" #include "knowhere/index/vector_index/IndexIVFSQ.h" -#include "knowhere/index/vector_index/IndexIVFSQHybrid.h" -#include "knowhere/index/vector_index/IndexKDT.h" #include "knowhere/index/vector_index/IndexNSG.h" +#include "knowhere/index/vector_index/IndexSPTAG.h" #include "utils/Log.h" +#ifdef MILVUS_GPU_VERSION #include +#include "knowhere/index/vector_index/IndexGPUIDMAP.h" +#include "knowhere/index/vector_index/IndexGPUIVF.h" +#include "knowhere/index/vector_index/IndexGPUIVFPQ.h" +#include "knowhere/index/vector_index/IndexGPUIVFSQ.h" +#include "knowhere/index/vector_index/IndexIVFSQHybrid.h" +#include "wrapper/gpu/GPUVecImpl.h" +#endif namespace milvus { namespace engine { @@ -119,43 +123,54 @@ GetVecIndexFactory(const IndexType& type, const Config& cfg) { index = std::make_shared(); break; } - case IndexType::FAISS_IVFFLAT_GPU: { - index = std::make_shared(gpu_device); - break; - } - case IndexType::FAISS_IVFFLAT_MIX: { - index = std::make_shared(gpu_device); - return std::make_shared(index, IndexType::FAISS_IVFFLAT_MIX); - } case IndexType::FAISS_IVFPQ_CPU: { index = std::make_shared(); break; } + case IndexType::SPTAG_KDT_RNT_CPU: { + index = std::make_shared("KDT"); + break; + } + case IndexType::SPTAG_BKT_RNT_CPU: { + index = std::make_shared("BKT"); + break; + } + case IndexType::FAISS_IVFSQ8_CPU: { + index = std::make_shared(); + break; + } + +#ifdef MILVUS_GPU_VERSION + case IndexType::FAISS_IVFFLAT_GPU: { + index = std::make_shared(gpu_device); + break; + } case IndexType::FAISS_IVFPQ_GPU: { index = std::make_shared(gpu_device); break; } - case IndexType::SPTAG_KDT_RNT_CPU: { - index = std::make_shared(); - break; + case IndexType::FAISS_IVFPQ_MIX: { + index = std::make_shared(gpu_device); + return std::make_shared(index, IndexType::FAISS_IVFPQ_MIX); } case IndexType::FAISS_IVFSQ8_MIX: { index = std::make_shared(gpu_device); return std::make_shared(index, IndexType::FAISS_IVFSQ8_MIX); } - case IndexType::FAISS_IVFSQ8_CPU: { - index = std::make_shared(); - break; - } case IndexType::FAISS_IVFSQ8_GPU: { index = std::make_shared(gpu_device); break; } + case IndexType::FAISS_IVFFLAT_MIX: { + index = std::make_shared(gpu_device); + return std::make_shared(index, IndexType::FAISS_IVFFLAT_MIX); + } #ifdef CUSTOMIZATION case IndexType::FAISS_IVFSQ8_HYBRID: { index = std::make_shared(gpu_device); return std::make_shared(index, IndexType::FAISS_IVFSQ8_HYBRID); } +#endif #endif case IndexType::NSG_MIX: { // TODO(linxj): bug. index = std::make_shared(gpu_device); @@ -269,6 +284,10 @@ ConvertToCpuIndexType(const IndexType& type) { case IndexType::FAISS_IVFSQ8_MIX: { return IndexType::FAISS_IVFSQ8_CPU; } + case IndexType::FAISS_IVFPQ_GPU: + case IndexType::FAISS_IVFPQ_MIX: { + return IndexType::FAISS_IVFPQ_CPU; + } default: { return type; } } } @@ -284,9 +303,12 @@ ConvertToGpuIndexType(const IndexType& type) { case IndexType::FAISS_IVFSQ8_CPU: { return IndexType::FAISS_IVFSQ8_GPU; } + case IndexType::FAISS_IVFPQ_MIX: + case IndexType::FAISS_IVFPQ_CPU: { + return IndexType::FAISS_IVFPQ_GPU; + } default: { return type; } } } - } // namespace engine } // namespace milvus diff --git a/core/src/wrapper/VecIndex.h b/core/src/wrapper/VecIndex.h index cd9ba86951..e69655b087 100644 --- a/core/src/wrapper/VecIndex.h +++ b/core/src/wrapper/VecIndex.h @@ -33,6 +33,7 @@ namespace engine { using Config = knowhere::Config; +// TODO(linxj): replace with string, Do refactor serialization enum class IndexType { INVALID = 0, FAISS_IDMAP = 1, @@ -47,6 +48,8 @@ enum class IndexType { FAISS_IVFSQ8_GPU, FAISS_IVFSQ8_HYBRID, // only support build on gpu. NSG_MIX, + FAISS_IVFPQ_MIX, + SPTAG_BKT_RNT_CPU, }; class VecIndex; @@ -137,6 +140,9 @@ write_index(VecIndexPtr index, const std::string& location); extern VecIndexPtr read_index(const std::string& location); +VecIndexPtr +read_index(const std::string& location, knowhere::BinarySet& index_binary); + extern VecIndexPtr GetVecIndexFactory(const IndexType& type, const Config& cfg = Config()); diff --git a/core/src/wrapper/WrapperException.cpp b/core/src/wrapper/WrapperException.cpp new file mode 100644 index 0000000000..0c256638dc --- /dev/null +++ b/core/src/wrapper/WrapperException.cpp @@ -0,0 +1,32 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "wrapper/WrapperException.h" + +namespace milvus { +namespace engine { + +WrapperException::WrapperException(const std::string& msg) : msg(msg) { +} + +const char* +WrapperException::what() const noexcept { + return msg.c_str(); +} + +} // namespace engine +} // namespace milvus diff --git a/core/src/wrapper/WrapperException.h b/core/src/wrapper/WrapperException.h new file mode 100644 index 0000000000..60b1aae71f --- /dev/null +++ b/core/src/wrapper/WrapperException.h @@ -0,0 +1,37 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include + +namespace milvus { +namespace engine { + +class WrapperException : public std::exception { + public: + explicit WrapperException(const std::string& msg); + + const char* + what() const noexcept override; + + const std::string msg; +}; + +} // namespace engine +} // namespace milvus diff --git a/core/src/wrapper/gpu/GPUVecImpl.cpp b/core/src/wrapper/gpu/GPUVecImpl.cpp new file mode 100644 index 0000000000..e40f5038ea --- /dev/null +++ b/core/src/wrapper/gpu/GPUVecImpl.cpp @@ -0,0 +1,163 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "wrapper/gpu/GPUVecImpl.h" +#include "knowhere/common/Exception.h" +#include "knowhere/index/vector_index/IndexGPUIDMAP.h" +#include "knowhere/index/vector_index/IndexGPUIVF.h" +#include "knowhere/index/vector_index/IndexIDMAP.h" +#include "knowhere/index/vector_index/IndexIVFSQHybrid.h" +#include "knowhere/index/vector_index/helpers/Cloner.h" +#include "src/wrapper/DataTransfer.h" +#include "utils/Log.h" +#include "wrapper/VecImpl.h" + +/* + * no parameter check in this layer. + * only responible for index combination + */ + +namespace milvus { +namespace engine { + +// TODO(linxj): add lock here. +Status +IVFMixIndex::BuildAll(const int64_t& nb, const float* xb, const int64_t* ids, const Config& cfg, const int64_t& nt, + const float* xt) { + try { + dim = cfg->d; + auto dataset = GenDatasetWithIds(nb, dim, xb, ids); + + auto preprocessor = index_->BuildPreprocessor(dataset, cfg); + index_->set_preprocessor(preprocessor); + auto model = index_->Train(dataset, cfg); + index_->set_index_model(model); + index_->Add(dataset, cfg); + + if (auto device_index = std::dynamic_pointer_cast(index_)) { + auto host_index = device_index->CopyGpuToCpu(Config()); + index_ = host_index; + type = ConvertToCpuIndexType(type); + } else { + WRAPPER_LOG_ERROR << "Build IVFMIXIndex Failed"; + return Status(KNOWHERE_ERROR, "Build IVFMIXIndex Failed"); + } + } catch (knowhere::KnowhereException& e) { + WRAPPER_LOG_ERROR << e.what(); + return Status(KNOWHERE_UNEXPECTED_ERROR, e.what()); + } catch (std::exception& e) { + WRAPPER_LOG_ERROR << e.what(); + return Status(KNOWHERE_ERROR, e.what()); + } + return Status::OK(); +} + +Status +IVFMixIndex::Load(const knowhere::BinarySet& index_binary) { + index_->Load(index_binary); + dim = Dimension(); + return Status::OK(); +} + +knowhere::QuantizerPtr +IVFHybridIndex::LoadQuantizer(const Config& conf) { + // TODO(linxj): Hardcode here + if (auto new_idx = std::dynamic_pointer_cast(index_)) { + return new_idx->LoadQuantizer(conf); + } else { + WRAPPER_LOG_ERROR << "Hybrid mode not support for index type: " << int(type); + } +} + +Status +IVFHybridIndex::SetQuantizer(const knowhere::QuantizerPtr& q) { + try { + // TODO(linxj): Hardcode here + if (auto new_idx = std::dynamic_pointer_cast(index_)) { + new_idx->SetQuantizer(q); + } else { + WRAPPER_LOG_ERROR << "Hybrid mode not support for index type: " << int(type); + return Status(KNOWHERE_ERROR, "not support"); + } + } catch (knowhere::KnowhereException& e) { + WRAPPER_LOG_ERROR << e.what(); + return Status(KNOWHERE_UNEXPECTED_ERROR, e.what()); + } catch (std::exception& e) { + WRAPPER_LOG_ERROR << e.what(); + return Status(KNOWHERE_ERROR, e.what()); + } + return Status::OK(); +} + +Status +IVFHybridIndex::UnsetQuantizer() { + try { + // TODO(linxj): Hardcode here + if (auto new_idx = std::dynamic_pointer_cast(index_)) { + new_idx->UnsetQuantizer(); + } else { + WRAPPER_LOG_ERROR << "Hybrid mode not support for index type: " << int(type); + return Status(KNOWHERE_ERROR, "not support"); + } + } catch (knowhere::KnowhereException& e) { + WRAPPER_LOG_ERROR << e.what(); + return Status(KNOWHERE_UNEXPECTED_ERROR, e.what()); + } catch (std::exception& e) { + WRAPPER_LOG_ERROR << e.what(); + return Status(KNOWHERE_ERROR, e.what()); + } + return Status::OK(); +} + +VecIndexPtr +IVFHybridIndex::LoadData(const knowhere::QuantizerPtr& q, const Config& conf) { + try { + // TODO(linxj): Hardcode here + if (auto new_idx = std::dynamic_pointer_cast(index_)) { + return std::make_shared(new_idx->LoadData(q, conf), type); + } else { + WRAPPER_LOG_ERROR << "Hybrid mode not support for index type: " << int(type); + } + } catch (knowhere::KnowhereException& e) { + WRAPPER_LOG_ERROR << e.what(); + } catch (std::exception& e) { + WRAPPER_LOG_ERROR << e.what(); + } + return nullptr; +} + +std::pair +IVFHybridIndex::CopyToGpuWithQuantizer(const int64_t& device_id, const Config& cfg) { + try { + // TODO(linxj): Hardcode here + if (auto hybrid_idx = std::dynamic_pointer_cast(index_)) { + auto pair = hybrid_idx->CopyCpuToGpuWithQuantizer(device_id, cfg); + auto new_idx = std::make_shared(pair.first, type); + return std::make_pair(new_idx, pair.second); + } else { + WRAPPER_LOG_ERROR << "Hybrid mode not support for index type: " << int(type); + } + } catch (knowhere::KnowhereException& e) { + WRAPPER_LOG_ERROR << e.what(); + } catch (std::exception& e) { + WRAPPER_LOG_ERROR << e.what(); + } + return std::make_pair(nullptr, nullptr); +} + +} // namespace engine +} // namespace milvus diff --git a/core/src/wrapper/gpu/GPUVecImpl.h b/core/src/wrapper/gpu/GPUVecImpl.h new file mode 100644 index 0000000000..82dd06aba4 --- /dev/null +++ b/core/src/wrapper/gpu/GPUVecImpl.h @@ -0,0 +1,67 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include + +#include "knowhere/index/vector_index/VectorIndex.h" +#include "wrapper/VecImpl.h" +#include "wrapper/VecIndex.h" + +namespace milvus { +namespace engine { + +class IVFMixIndex : public VecIndexImpl { + public: + explicit IVFMixIndex(std::shared_ptr index, const IndexType& type) + : VecIndexImpl(std::move(index), type) { + } + + Status + BuildAll(const int64_t& nb, const float* xb, const int64_t* ids, const Config& cfg, const int64_t& nt, + const float* xt) override; + + Status + Load(const knowhere::BinarySet& index_binary) override; +}; + +class IVFHybridIndex : public IVFMixIndex { + public: + explicit IVFHybridIndex(std::shared_ptr index, const IndexType& type) + : IVFMixIndex(std::move(index), type) { + } + + knowhere::QuantizerPtr + LoadQuantizer(const Config& conf) override; + + Status + SetQuantizer(const knowhere::QuantizerPtr& q) override; + + Status + UnsetQuantizer() override; + + std::pair + CopyToGpuWithQuantizer(const int64_t& device_id, const Config& cfg) override; + + VecIndexPtr + LoadData(const knowhere::QuantizerPtr& q, const Config& conf) override; +}; + +} // namespace engine +} // namespace milvus diff --git a/core/thirdparty/easyloggingpp/easylogging++.cc b/core/thirdparty/easyloggingpp/easylogging++.cc new file mode 100644 index 0000000000..4c6df12686 --- /dev/null +++ b/core/thirdparty/easyloggingpp/easylogging++.cc @@ -0,0 +1,3299 @@ +// +// Bismillah ar-Rahmaan ar-Raheem +// +// Easylogging++ v9.96.7 +// Cross-platform logging library for C++ applications +// +// Copyright (c) 2012-2018 Zuhd Web Services +// Copyright (c) 2012-2018 @abumusamq +// +// This library is released under the MIT Licence. +// https://github.com/zuhd-org/easyloggingpp/blob/master/LICENSE +// +// https://zuhd.org +// http://muflihun.com +// + +#include "easylogging++.h" + +#if defined(AUTO_INITIALIZE_EASYLOGGINGPP) +INITIALIZE_EASYLOGGINGPP +#endif + +namespace el { + +// el::base +namespace base { +// el::base::consts +namespace consts { + +// Level log values - These are values that are replaced in place of %level format specifier +// Extra spaces after format specifiers are only for readability purposes in log files +static const base::type::char_t* kInfoLevelLogValue = ELPP_LITERAL("INFO"); +static const base::type::char_t* kDebugLevelLogValue = ELPP_LITERAL("DEBUG"); +static const base::type::char_t* kWarningLevelLogValue = ELPP_LITERAL("WARNING"); +static const base::type::char_t* kErrorLevelLogValue = ELPP_LITERAL("ERROR"); +static const base::type::char_t* kFatalLevelLogValue = ELPP_LITERAL("FATAL"); +static const base::type::char_t* kVerboseLevelLogValue = + ELPP_LITERAL("VERBOSE"); // will become VERBOSE-x where x = verbose level +static const base::type::char_t* kTraceLevelLogValue = ELPP_LITERAL("TRACE"); +static const base::type::char_t* kInfoLevelShortLogValue = ELPP_LITERAL("I"); +static const base::type::char_t* kDebugLevelShortLogValue = ELPP_LITERAL("D"); +static const base::type::char_t* kWarningLevelShortLogValue = ELPP_LITERAL("W"); +static const base::type::char_t* kErrorLevelShortLogValue = ELPP_LITERAL("E"); +static const base::type::char_t* kFatalLevelShortLogValue = ELPP_LITERAL("F"); +static const base::type::char_t* kVerboseLevelShortLogValue = ELPP_LITERAL("V"); +static const base::type::char_t* kTraceLevelShortLogValue = ELPP_LITERAL("T"); +// Format specifiers - These are used to define log format +static const base::type::char_t* kAppNameFormatSpecifier = ELPP_LITERAL("%app"); +static const base::type::char_t* kLoggerIdFormatSpecifier = ELPP_LITERAL("%logger"); +static const base::type::char_t* kThreadIdFormatSpecifier = ELPP_LITERAL("%thread"); +static const base::type::char_t* kSeverityLevelFormatSpecifier = ELPP_LITERAL("%level"); +static const base::type::char_t* kSeverityLevelShortFormatSpecifier = ELPP_LITERAL("%levshort"); +static const base::type::char_t* kDateTimeFormatSpecifier = ELPP_LITERAL("%datetime"); +static const base::type::char_t* kLogFileFormatSpecifier = ELPP_LITERAL("%file"); +static const base::type::char_t* kLogFileBaseFormatSpecifier = ELPP_LITERAL("%fbase"); +static const base::type::char_t* kLogLineFormatSpecifier = ELPP_LITERAL("%line"); +static const base::type::char_t* kLogLocationFormatSpecifier = ELPP_LITERAL("%loc"); +static const base::type::char_t* kLogFunctionFormatSpecifier = ELPP_LITERAL("%func"); +static const base::type::char_t* kCurrentUserFormatSpecifier = ELPP_LITERAL("%user"); +static const base::type::char_t* kCurrentHostFormatSpecifier = ELPP_LITERAL("%host"); +static const base::type::char_t* kMessageFormatSpecifier = ELPP_LITERAL("%msg"); +static const base::type::char_t* kVerboseLevelFormatSpecifier = ELPP_LITERAL("%vlevel"); +static const char* kDateTimeFormatSpecifierForFilename = "%datetime"; +// Date/time +static const char* kDays[7] = {"Sundayaaa", "Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday"}; +static const char* kDaysAbbrev[7] = {"Sun", "Mon", "Tue", "Wed", "Thu", "Fri", "Sat"}; +static const char* kMonths[12] = {"January", "February", "March", "Apri", "May", "June", + "July", "August", "September", "October", "November", "December"}; +static const char* kMonthsAbbrev[12] = {"Jan", "Feb", "Mar", "Apr", "May", "Jun", + "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"}; +static const char* kDefaultDateTimeFormat = "%Y-%M-%d %H:%m:%s,%g"; +static const char* kDefaultDateTimeFormatInFilename = "%Y-%M-%d_%H-%m"; +static const int kYearBase = 1900; +static const char* kAm = "AM"; +static const char* kPm = "PM"; +// Miscellaneous constants + +static const char* kNullPointer = "nullptr"; +#if ELPP_VARIADIC_TEMPLATES_SUPPORTED +#endif // ELPP_VARIADIC_TEMPLATES_SUPPORTED +static const base::type::VerboseLevel kMaxVerboseLevel = 9; +static const char* kUnknownUser = "user"; +static const char* kUnknownHost = "unknown-host"; + +//---------------- DEFAULT LOG FILE ----------------------- + +#if defined(ELPP_NO_DEFAULT_LOG_FILE) +#if ELPP_OS_UNIX +static const char* kDefaultLogFile = "/dev/null"; +#elif ELPP_OS_WINDOWS +static const char* kDefaultLogFile = "nul"; +#endif // ELPP_OS_UNIX +#elif defined(ELPP_DEFAULT_LOG_FILE) +static const char* kDefaultLogFile = ELPP_DEFAULT_LOG_FILE; +#else +static const char* kDefaultLogFile = "myeasylog.log"; +#endif // defined(ELPP_NO_DEFAULT_LOG_FILE) + +#if !defined(ELPP_DISABLE_LOG_FILE_FROM_ARG) +static const char* kDefaultLogFileParam = "--default-log-file"; +#endif // !defined(ELPP_DISABLE_LOG_FILE_FROM_ARG) +#if defined(ELPP_LOGGING_FLAGS_FROM_ARG) +static const char* kLoggingFlagsParam = "--logging-flags"; +#endif // defined(ELPP_LOGGING_FLAGS_FROM_ARG) +static const char* kValidLoggerIdSymbols = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-._"; +static const char* kConfigurationComment = "##"; +static const char* kConfigurationLevel = "*"; +static const char* kConfigurationLoggerId = "--"; +} // namespace consts +// el::base::utils +namespace utils { + +/// @brief Aborts application due with user-defined status +static void +abort(int status, const std::string& reason) { + // Both status and reason params are there for debugging with tools like gdb etc + ELPP_UNUSED(status); + ELPP_UNUSED(reason); +#if defined(ELPP_COMPILER_MSVC) && defined(_M_IX86) && defined(_DEBUG) + // Ignore msvc critical error dialog - break instead (on debug mode) + _asm int 3 +#else + ::abort(); +#endif // defined(ELPP_COMPILER_MSVC) && defined(_M_IX86) && defined(_DEBUG) +} + +} // namespace utils +} // namespace base + +// el + +// LevelHelper + +const char* +LevelHelper::convertToString(Level level) { + // Do not use switch over strongly typed enums because Intel C++ compilers dont support them yet. + if (level == Level::Global) + return "GLOBAL"; + if (level == Level::Debug) + return "DEBUG"; + if (level == Level::Info) + return "INFO"; + if (level == Level::Warning) + return "WARNING"; + if (level == Level::Error) + return "ERROR"; + if (level == Level::Fatal) + return "FATAL"; + if (level == Level::Verbose) + return "VERBOSE"; + if (level == Level::Trace) + return "TRACE"; + return "UNKNOWN"; +} + +struct StringToLevelItem { + const char* levelString; + Level level; +}; + +static struct StringToLevelItem stringToLevelMap[] = { + {"global", Level::Global}, {"debug", Level::Debug}, {"info", Level::Info}, {"warning", Level::Warning}, + {"error", Level::Error}, {"fatal", Level::Fatal}, {"verbose", Level::Verbose}, {"trace", Level::Trace}}; + +Level +LevelHelper::convertFromString(const char* levelStr) { + for (auto& item : stringToLevelMap) { + if (base::utils::Str::cStringCaseEq(levelStr, item.levelString)) { + return item.level; + } + } + return Level::Unknown; +} + +void +LevelHelper::forEachLevel(base::type::EnumType* startIndex, const std::function& fn) { + base::type::EnumType lIndexMax = LevelHelper::kMaxValid; + do { + if (fn()) { + break; + } + *startIndex = static_cast(*startIndex << 1); + } while (*startIndex <= lIndexMax); +} + +// ConfigurationTypeHelper + +const char* +ConfigurationTypeHelper::convertToString(ConfigurationType configurationType) { + // Do not use switch over strongly typed enums because Intel C++ compilers dont support them yet. + if (configurationType == ConfigurationType::Enabled) + return "ENABLED"; + if (configurationType == ConfigurationType::Filename) + return "FILENAME"; + if (configurationType == ConfigurationType::Format) + return "FORMAT"; + if (configurationType == ConfigurationType::ToFile) + return "TO_FILE"; + if (configurationType == ConfigurationType::ToStandardOutput) + return "TO_STANDARD_OUTPUT"; + if (configurationType == ConfigurationType::SubsecondPrecision) + return "SUBSECOND_PRECISION"; + if (configurationType == ConfigurationType::PerformanceTracking) + return "PERFORMANCE_TRACKING"; + if (configurationType == ConfigurationType::MaxLogFileSize) + return "MAX_LOG_FILE_SIZE"; + if (configurationType == ConfigurationType::LogFlushThreshold) + return "LOG_FLUSH_THRESHOLD"; + return "UNKNOWN"; +} + +struct ConfigurationStringToTypeItem { + const char* configString; + ConfigurationType configType; +}; + +static struct ConfigurationStringToTypeItem configStringToTypeMap[] = { + {"enabled", ConfigurationType::Enabled}, + {"to_file", ConfigurationType::ToFile}, + {"to_standard_output", ConfigurationType::ToStandardOutput}, + {"format", ConfigurationType::Format}, + {"filename", ConfigurationType::Filename}, + {"subsecond_precision", ConfigurationType::SubsecondPrecision}, + {"milliseconds_width", ConfigurationType::MillisecondsWidth}, + {"performance_tracking", ConfigurationType::PerformanceTracking}, + {"max_log_file_size", ConfigurationType::MaxLogFileSize}, + {"log_flush_threshold", ConfigurationType::LogFlushThreshold}, +}; + +ConfigurationType +ConfigurationTypeHelper::convertFromString(const char* configStr) { + for (auto& item : configStringToTypeMap) { + if (base::utils::Str::cStringCaseEq(configStr, item.configString)) { + return item.configType; + } + } + return ConfigurationType::Unknown; +} + +void +ConfigurationTypeHelper::forEachConfigType(base::type::EnumType* startIndex, const std::function& fn) { + base::type::EnumType cIndexMax = ConfigurationTypeHelper::kMaxValid; + do { + if (fn()) { + break; + } + *startIndex = static_cast(*startIndex << 1); + } while (*startIndex <= cIndexMax); +} + +// Configuration + +Configuration::Configuration(const Configuration& c) + : m_level(c.m_level), m_configurationType(c.m_configurationType), m_value(c.m_value) { +} + +Configuration& +Configuration::operator=(const Configuration& c) { + if (&c != this) { + m_level = c.m_level; + m_configurationType = c.m_configurationType; + m_value = c.m_value; + } + return *this; +} + +/// @brief Full constructor used to sets value of configuration +Configuration::Configuration(Level level, ConfigurationType configurationType, const std::string& value) + : m_level(level), m_configurationType(configurationType), m_value(value) { +} + +void +Configuration::log(el::base::type::ostream_t& os) const { + os << LevelHelper::convertToString(m_level) << ELPP_LITERAL(" ") + << ConfigurationTypeHelper::convertToString(m_configurationType) << ELPP_LITERAL(" = ") << m_value.c_str(); +} + +/// @brief Used to find configuration from configuration (pointers) repository. Avoid using it. +Configuration::Predicate::Predicate(Level level, ConfigurationType configurationType) + : m_level(level), m_configurationType(configurationType) { +} + +bool +Configuration::Predicate::operator()(const Configuration* conf) const { + return ((conf != nullptr) && (conf->level() == m_level) && (conf->configurationType() == m_configurationType)); +} + +// Configurations + +Configurations::Configurations(void) : m_configurationFile(std::string()), m_isFromFile(false) { +} + +Configurations::Configurations(const std::string& configurationFile, bool useDefaultsForRemaining, Configurations* base) + : m_configurationFile(configurationFile), m_isFromFile(false) { + parseFromFile(configurationFile, base); + if (useDefaultsForRemaining) { + setRemainingToDefault(); + } +} + +bool +Configurations::parseFromFile(const std::string& configurationFile, Configurations* base) { + // We initial assertion with true because if we have assertion diabled, we want to pass this + // check and if assertion is enabled we will have values re-assigned any way. + bool assertionPassed = true; + ELPP_ASSERT((assertionPassed = base::utils::File::pathExists(configurationFile.c_str(), true)) == true, + "Configuration file [" << configurationFile << "] does not exist!"); + if (!assertionPassed) { + return false; + } + bool success = Parser::parseFromFile(configurationFile, this, base); + m_isFromFile = success; + return success; +} + +bool +Configurations::parseFromText(const std::string& configurationsString, Configurations* base) { + bool success = Parser::parseFromText(configurationsString, this, base); + if (success) { + m_isFromFile = false; + } + return success; +} + +void +Configurations::setFromBase(Configurations* base) { + if (base == nullptr || base == this) { + return; + } + base::threading::ScopedLock scopedLock(base->lock()); + for (Configuration*& conf : base->list()) { + set(conf); + } +} + +bool +Configurations::hasConfiguration(ConfigurationType configurationType) { + base::type::EnumType lIndex = LevelHelper::kMinValid; + bool result = false; + LevelHelper::forEachLevel(&lIndex, [&](void) -> bool { + if (hasConfiguration(LevelHelper::castFromInt(lIndex), configurationType)) { + result = true; + } + return result; + }); + return result; +} + +bool +Configurations::hasConfiguration(Level level, ConfigurationType configurationType) { + base::threading::ScopedLock scopedLock(lock()); +#if ELPP_COMPILER_INTEL + // We cant specify template types here, Intel C++ throws compilation error + // "error: type name is not allowed" + return RegistryWithPred::get(level, configurationType) != nullptr; +#else + return RegistryWithPred::get(level, configurationType) != nullptr; +#endif // ELPP_COMPILER_INTEL +} + +void +Configurations::set(Level level, ConfigurationType configurationType, const std::string& value) { + base::threading::ScopedLock scopedLock(lock()); + unsafeSet(level, configurationType, value); // This is not unsafe anymore as we have locked mutex + if (level == Level::Global) { + unsafeSetGlobally(configurationType, value, false); // Again this is not unsafe either + } +} + +void +Configurations::set(Configuration* conf) { + if (conf == nullptr) { + return; + } + set(conf->level(), conf->configurationType(), conf->value()); +} + +void +Configurations::setToDefault(void) { + setGlobally(ConfigurationType::Enabled, std::string("true"), true); + setGlobally(ConfigurationType::Filename, std::string(base::consts::kDefaultLogFile), true); +#if defined(ELPP_NO_LOG_TO_FILE) + setGlobally(ConfigurationType::ToFile, std::string("false"), true); +#else + setGlobally(ConfigurationType::ToFile, std::string("true"), true); +#endif // defined(ELPP_NO_LOG_TO_FILE) + setGlobally(ConfigurationType::ToStandardOutput, std::string("true"), true); + setGlobally(ConfigurationType::SubsecondPrecision, std::string("3"), true); + setGlobally(ConfigurationType::PerformanceTracking, std::string("true"), true); + setGlobally(ConfigurationType::MaxLogFileSize, std::string("0"), true); + setGlobally(ConfigurationType::LogFlushThreshold, std::string("0"), true); + + setGlobally(ConfigurationType::Format, std::string("%datetime %level [%logger] %msg"), true); + set(Level::Debug, ConfigurationType::Format, + std::string("%datetime %level [%logger] [%user@%host] [%func] [%loc] %msg")); + // INFO and WARNING are set to default by Level::Global + set(Level::Error, ConfigurationType::Format, std::string("%datetime %level [%logger] %msg")); + set(Level::Fatal, ConfigurationType::Format, std::string("%datetime %level [%logger] %msg")); + set(Level::Verbose, ConfigurationType::Format, std::string("%datetime %level-%vlevel [%logger] %msg")); + set(Level::Trace, ConfigurationType::Format, std::string("%datetime %level [%logger] [%func] [%loc] %msg")); +} + +void +Configurations::setRemainingToDefault(void) { + base::threading::ScopedLock scopedLock(lock()); +#if defined(ELPP_NO_LOG_TO_FILE) + unsafeSetIfNotExist(Level::Global, ConfigurationType::Enabled, std::string("false")); +#else + unsafeSetIfNotExist(Level::Global, ConfigurationType::Enabled, std::string("true")); +#endif // defined(ELPP_NO_LOG_TO_FILE) + unsafeSetIfNotExist(Level::Global, ConfigurationType::Filename, std::string(base::consts::kDefaultLogFile)); + unsafeSetIfNotExist(Level::Global, ConfigurationType::ToStandardOutput, std::string("true")); + unsafeSetIfNotExist(Level::Global, ConfigurationType::SubsecondPrecision, std::string("3")); + unsafeSetIfNotExist(Level::Global, ConfigurationType::PerformanceTracking, std::string("true")); + unsafeSetIfNotExist(Level::Global, ConfigurationType::MaxLogFileSize, std::string("0")); + unsafeSetIfNotExist(Level::Global, ConfigurationType::Format, std::string("%datetime %level [%logger] %msg")); + unsafeSetIfNotExist(Level::Debug, ConfigurationType::Format, + std::string("%datetime %level [%logger] [%user@%host] [%func] [%loc] %msg")); + // INFO and WARNING are set to default by Level::Global + unsafeSetIfNotExist(Level::Error, ConfigurationType::Format, std::string("%datetime %level [%logger] %msg")); + unsafeSetIfNotExist(Level::Fatal, ConfigurationType::Format, std::string("%datetime %level [%logger] %msg")); + unsafeSetIfNotExist(Level::Verbose, ConfigurationType::Format, + std::string("%datetime %level-%vlevel [%logger] %msg")); + unsafeSetIfNotExist(Level::Trace, ConfigurationType::Format, + std::string("%datetime %level [%logger] [%func] [%loc] %msg")); +} + +bool +Configurations::Parser::parseFromFile(const std::string& configurationFile, Configurations* sender, + Configurations* base) { + sender->setFromBase(base); + std::ifstream fileStream_(configurationFile.c_str(), std::ifstream::in); + ELPP_ASSERT(fileStream_.is_open(), "Unable to open configuration file [" << configurationFile << "] for parsing."); + bool parsedSuccessfully = false; + std::string line = std::string(); + Level currLevel = Level::Unknown; + std::string currConfigStr = std::string(); + std::string currLevelStr = std::string(); + while (fileStream_.good()) { + std::getline(fileStream_, line); + parsedSuccessfully = parseLine(&line, &currConfigStr, &currLevelStr, &currLevel, sender); + ELPP_ASSERT(parsedSuccessfully, "Unable to parse configuration line: " << line); + } + return parsedSuccessfully; +} + +bool +Configurations::Parser::parseFromText(const std::string& configurationsString, Configurations* sender, + Configurations* base) { + sender->setFromBase(base); + bool parsedSuccessfully = false; + std::stringstream ss(configurationsString); + std::string line = std::string(); + Level currLevel = Level::Unknown; + std::string currConfigStr = std::string(); + std::string currLevelStr = std::string(); + while (std::getline(ss, line)) { + parsedSuccessfully = parseLine(&line, &currConfigStr, &currLevelStr, &currLevel, sender); + ELPP_ASSERT(parsedSuccessfully, "Unable to parse configuration line: " << line); + } + return parsedSuccessfully; +} + +void +Configurations::Parser::ignoreComments(std::string* line) { + std::size_t foundAt = 0; + std::size_t quotesStart = line->find("\""); + std::size_t quotesEnd = std::string::npos; + if (quotesStart != std::string::npos) { + quotesEnd = line->find("\"", quotesStart + 1); + while (quotesEnd != std::string::npos && line->at(quotesEnd - 1) == '\\') { + // Do not erase slash yet - we will erase it in parseLine(..) while loop + quotesEnd = line->find("\"", quotesEnd + 2); + } + } + if ((foundAt = line->find(base::consts::kConfigurationComment)) != std::string::npos) { + if (foundAt < quotesEnd) { + foundAt = line->find(base::consts::kConfigurationComment, quotesEnd + 1); + } + *line = line->substr(0, foundAt); + } +} + +bool +Configurations::Parser::isLevel(const std::string& line) { + return base::utils::Str::startsWith(line, std::string(base::consts::kConfigurationLevel)); +} + +bool +Configurations::Parser::isComment(const std::string& line) { + return base::utils::Str::startsWith(line, std::string(base::consts::kConfigurationComment)); +} + +bool +Configurations::Parser::isConfig(const std::string& line) { + std::size_t assignment = line.find('='); + return line != "" && ((line[0] >= 'A' && line[0] <= 'Z') || (line[0] >= 'a' && line[0] <= 'z')) && + (assignment != std::string::npos) && (line.size() > assignment); +} + +bool +Configurations::Parser::parseLine(std::string* line, std::string* currConfigStr, std::string* currLevelStr, + Level* currLevel, Configurations* conf) { + ConfigurationType currConfig = ConfigurationType::Unknown; + std::string currValue = std::string(); + *line = base::utils::Str::trim(*line); + if (isComment(*line)) + return true; + ignoreComments(line); + *line = base::utils::Str::trim(*line); + if (line->empty()) { + // Comment ignored + return true; + } + if (isLevel(*line)) { + if (line->size() <= 2) { + return true; + } + *currLevelStr = line->substr(1, line->size() - 2); + *currLevelStr = base::utils::Str::toUpper(*currLevelStr); + *currLevelStr = base::utils::Str::trim(*currLevelStr); + *currLevel = LevelHelper::convertFromString(currLevelStr->c_str()); + return true; + } + if (isConfig(*line)) { + std::size_t assignment = line->find('='); + *currConfigStr = line->substr(0, assignment); + *currConfigStr = base::utils::Str::toUpper(*currConfigStr); + *currConfigStr = base::utils::Str::trim(*currConfigStr); + currConfig = ConfigurationTypeHelper::convertFromString(currConfigStr->c_str()); + currValue = line->substr(assignment + 1); + currValue = base::utils::Str::trim(currValue); + std::size_t quotesStart = currValue.find("\"", 0); + std::size_t quotesEnd = std::string::npos; + if (quotesStart != std::string::npos) { + quotesEnd = currValue.find("\"", quotesStart + 1); + while (quotesEnd != std::string::npos && currValue.at(quotesEnd - 1) == '\\') { + currValue = currValue.erase(quotesEnd - 1, 1); + quotesEnd = currValue.find("\"", quotesEnd + 2); + } + } + if (quotesStart != std::string::npos && quotesEnd != std::string::npos) { + // Quote provided - check and strip if valid + ELPP_ASSERT((quotesStart < quotesEnd), + "Configuration error - No ending quote found in [" << currConfigStr << "]"); + ELPP_ASSERT((quotesStart + 1 != quotesEnd), "Empty configuration value for [" << currConfigStr << "]"); + if ((quotesStart != quotesEnd) && (quotesStart + 1 != quotesEnd)) { + // Explicit check in case if assertion is disabled + currValue = currValue.substr(quotesStart + 1, quotesEnd - 1); + } + } + } + ELPP_ASSERT(*currLevel != Level::Unknown, "Unrecognized severity level [" << *currLevelStr << "]"); + ELPP_ASSERT(currConfig != ConfigurationType::Unknown, "Unrecognized configuration [" << *currConfigStr << "]"); + if (*currLevel == Level::Unknown || currConfig == ConfigurationType::Unknown) { + return false; // unrecognizable level or config + } + conf->set(*currLevel, currConfig, currValue); + return true; +} + +void +Configurations::unsafeSetIfNotExist(Level level, ConfigurationType configurationType, const std::string& value) { + Configuration* conf = RegistryWithPred::get(level, configurationType); + if (conf == nullptr) { + unsafeSet(level, configurationType, value); + } +} + +void +Configurations::unsafeSet(Level level, ConfigurationType configurationType, const std::string& value) { + Configuration* conf = RegistryWithPred::get(level, configurationType); + if (conf == nullptr) { + registerNew(new Configuration(level, configurationType, value)); + } else { + conf->setValue(value); + } + if (level == Level::Global) { + unsafeSetGlobally(configurationType, value, false); + } +} + +void +Configurations::setGlobally(ConfigurationType configurationType, const std::string& value, bool includeGlobalLevel) { + if (includeGlobalLevel) { + set(Level::Global, configurationType, value); + } + base::type::EnumType lIndex = LevelHelper::kMinValid; + LevelHelper::forEachLevel(&lIndex, [&](void) -> bool { + set(LevelHelper::castFromInt(lIndex), configurationType, value); + return false; // Do not break lambda function yet as we need to set all levels regardless + }); +} + +void +Configurations::unsafeSetGlobally(ConfigurationType configurationType, const std::string& value, + bool includeGlobalLevel) { + if (includeGlobalLevel) { + unsafeSet(Level::Global, configurationType, value); + } + base::type::EnumType lIndex = LevelHelper::kMinValid; + LevelHelper::forEachLevel(&lIndex, [&](void) -> bool { + unsafeSet(LevelHelper::castFromInt(lIndex), configurationType, value); + return false; // Do not break lambda function yet as we need to set all levels regardless + }); +} + +// LogBuilder + +void +LogBuilder::convertToColoredOutput(base::type::string_t* logLine, Level level) { + if (!m_termSupportsColor) + return; + const base::type::char_t* resetColor = ELPP_LITERAL("\x1b[0m"); + if (level == Level::Error || level == Level::Fatal) + *logLine = ELPP_LITERAL("\x1b[31m") + *logLine + resetColor; + else if (level == Level::Warning) + *logLine = ELPP_LITERAL("\x1b[33m") + *logLine + resetColor; + else if (level == Level::Debug) + *logLine = ELPP_LITERAL("\x1b[32m") + *logLine + resetColor; + else if (level == Level::Info) + *logLine = ELPP_LITERAL("\x1b[36m") + *logLine + resetColor; + else if (level == Level::Trace) + *logLine = ELPP_LITERAL("\x1b[35m") + *logLine + resetColor; +} + +// Logger + +Logger::Logger(const std::string& id, base::LogStreamsReferenceMap* logStreamsReference) + : m_id(id), + m_typedConfigurations(nullptr), + m_parentApplicationName(std::string()), + m_isConfigured(false), + m_logStreamsReference(logStreamsReference) { + initUnflushedCount(); +} + +Logger::Logger(const std::string& id, const Configurations& configurations, + base::LogStreamsReferenceMap* logStreamsReference) + : m_id(id), + m_typedConfigurations(nullptr), + m_parentApplicationName(std::string()), + m_isConfigured(false), + m_logStreamsReference(logStreamsReference) { + initUnflushedCount(); + configure(configurations); +} + +Logger::Logger(const Logger& logger) { + base::utils::safeDelete(m_typedConfigurations); + m_id = logger.m_id; + m_typedConfigurations = logger.m_typedConfigurations; + m_parentApplicationName = logger.m_parentApplicationName; + m_isConfigured = logger.m_isConfigured; + m_configurations = logger.m_configurations; + m_unflushedCount = logger.m_unflushedCount; + m_logStreamsReference = logger.m_logStreamsReference; +} + +Logger& +Logger::operator=(const Logger& logger) { + if (&logger != this) { + base::utils::safeDelete(m_typedConfigurations); + m_id = logger.m_id; + m_typedConfigurations = logger.m_typedConfigurations; + m_parentApplicationName = logger.m_parentApplicationName; + m_isConfigured = logger.m_isConfigured; + m_configurations = logger.m_configurations; + m_unflushedCount = logger.m_unflushedCount; + m_logStreamsReference = logger.m_logStreamsReference; + } + return *this; +} + +void +Logger::configure(const Configurations& configurations) { + m_isConfigured = false; // we set it to false in case if we fail + initUnflushedCount(); + if (m_typedConfigurations != nullptr) { + Configurations* c = const_cast(m_typedConfigurations->configurations()); + if (c->hasConfiguration(Level::Global, ConfigurationType::Filename)) { + flush(); + } + } + base::threading::ScopedLock scopedLock(lock()); + if (m_configurations != configurations) { + m_configurations.setFromBase(const_cast(&configurations)); + } + base::utils::safeDelete(m_typedConfigurations); + m_typedConfigurations = new base::TypedConfigurations(&m_configurations, m_logStreamsReference); + resolveLoggerFormatSpec(); + m_isConfigured = true; +} + +void +Logger::reconfigure(void) { + ELPP_INTERNAL_INFO(1, "Reconfiguring logger [" << m_id << "]"); + configure(m_configurations); +} + +bool +Logger::isValidId(const std::string& id) { + for (std::string::const_iterator it = id.begin(); it != id.end(); ++it) { + if (!base::utils::Str::contains(base::consts::kValidLoggerIdSymbols, *it)) { + return false; + } + } + return true; +} + +void +Logger::flush(void) { + ELPP_INTERNAL_INFO(3, "Flushing logger [" << m_id << "] all levels"); + base::threading::ScopedLock scopedLock(lock()); + base::type::EnumType lIndex = LevelHelper::kMinValid; + LevelHelper::forEachLevel(&lIndex, [&](void) -> bool { + flush(LevelHelper::castFromInt(lIndex), nullptr); + return false; + }); +} + +void +Logger::flush(Level level, base::type::fstream_t* fs) { + if (fs == nullptr && m_typedConfigurations->toFile(level)) { + fs = m_typedConfigurations->fileStream(level); + } + if (fs != nullptr) { + fs->flush(); + std::unordered_map::iterator iter = m_unflushedCount.find(level); + if (iter != m_unflushedCount.end()) { + iter->second = 0; + } + Helpers::validateFileRolling(this, level); + } +} + +void +Logger::initUnflushedCount(void) { + m_unflushedCount.clear(); + base::type::EnumType lIndex = LevelHelper::kMinValid; + LevelHelper::forEachLevel(&lIndex, [&](void) -> bool { + m_unflushedCount.insert(std::make_pair(LevelHelper::castFromInt(lIndex), 0)); + return false; + }); +} + +void +Logger::resolveLoggerFormatSpec(void) const { + base::type::EnumType lIndex = LevelHelper::kMinValid; + LevelHelper::forEachLevel(&lIndex, [&](void) -> bool { + base::LogFormat* logFormat = + const_cast(&m_typedConfigurations->logFormat(LevelHelper::castFromInt(lIndex))); + base::utils::Str::replaceFirstWithEscape(logFormat->m_format, base::consts::kLoggerIdFormatSpecifier, m_id); + return false; + }); +} + +// el::base +namespace base { + +// el::base::utils +namespace utils { + +// File + +base::type::fstream_t* +File::newFileStream(const std::string& filename) { + base::type::fstream_t* fs = new base::type::fstream_t(filename.c_str(), base::type::fstream_t::out +#if !defined(ELPP_FRESH_LOG_FILE) + | base::type::fstream_t::app +#endif + ); +#if defined(ELPP_UNICODE) + std::locale elppUnicodeLocale(""); +#if ELPP_OS_WINDOWS + std::locale elppUnicodeLocaleWindows(elppUnicodeLocale, new std::codecvt_utf8_utf16); + elppUnicodeLocale = elppUnicodeLocaleWindows; +#endif // ELPP_OS_WINDOWS + fs->imbue(elppUnicodeLocale); +#endif // defined(ELPP_UNICODE) + if (fs->is_open()) { + fs->flush(); + } else { + base::utils::safeDelete(fs); + ELPP_INTERNAL_ERROR("Bad file [" << filename << "]", true); + } + return fs; +} + +std::size_t +File::getSizeOfFile(base::type::fstream_t* fs) { + if (fs == nullptr) { + return 0; + } + // Since the file stream is appended to or truncated, the current + // offset is the file size. + std::size_t size = static_cast(fs->tellg()); + return size; +} + +bool +File::pathExists(const char* path, bool considerFile) { + if (path == nullptr) { + return false; + } +#if ELPP_OS_UNIX + ELPP_UNUSED(considerFile); + struct stat st; + return (stat(path, &st) == 0); +#elif ELPP_OS_WINDOWS + DWORD fileType = GetFileAttributesA(path); + if (fileType == INVALID_FILE_ATTRIBUTES) { + return false; + } + return considerFile ? true : ((fileType & FILE_ATTRIBUTE_DIRECTORY) == 0 ? false : true); +#endif // ELPP_OS_UNIX +} + +bool +File::createPath(const std::string& path) { + if (path.empty()) { + return false; + } + if (base::utils::File::pathExists(path.c_str())) { + return true; + } + int status = -1; + + char* currPath = const_cast(path.c_str()); + std::string builtPath = std::string(); +#if ELPP_OS_UNIX + if (path[0] == '/') { + builtPath = "/"; + } + currPath = STRTOK(currPath, base::consts::kFilePathSeperator, 0); +#elif ELPP_OS_WINDOWS + // Use secure functions API + char* nextTok_ = nullptr; + currPath = STRTOK(currPath, base::consts::kFilePathSeperator, &nextTok_); + ELPP_UNUSED(nextTok_); +#endif // ELPP_OS_UNIX + while (currPath != nullptr) { + builtPath.append(currPath); + builtPath.append(base::consts::kFilePathSeperator); +#if ELPP_OS_UNIX + status = mkdir(builtPath.c_str(), ELPP_LOG_PERMS); + currPath = STRTOK(nullptr, base::consts::kFilePathSeperator, 0); +#elif ELPP_OS_WINDOWS + status = _mkdir(builtPath.c_str()); + currPath = STRTOK(nullptr, base::consts::kFilePathSeperator, &nextTok_); +#endif // ELPP_OS_UNIX + } + if (status == -1) { + ELPP_INTERNAL_ERROR("Error while creating path [" << path << "]", true); + return false; + } + return true; +} + +std::string +File::extractPathFromFilename(const std::string& fullPath, const char* separator) { + if ((fullPath == "") || (fullPath.find(separator) == std::string::npos)) { + return fullPath; + } + std::size_t lastSlashAt = fullPath.find_last_of(separator); + if (lastSlashAt == 0) { + return std::string(separator); + } + return fullPath.substr(0, lastSlashAt + 1); +} + +void +File::buildStrippedFilename(const char* filename, char buff[], std::size_t limit) { + std::size_t sizeOfFilename = strlen(filename); + if (sizeOfFilename >= limit) { + filename += (sizeOfFilename - limit); + if (filename[0] != '.' && filename[1] != '.') { // prepend if not already + filename += 3; // 3 = '..' + STRCAT(buff, "..", limit); + } + } + STRCAT(buff, filename, limit); +} + +void +File::buildBaseFilename(const std::string& fullPath, char buff[], std::size_t limit, const char* separator) { + const char* filename = fullPath.c_str(); + std::size_t lastSlashAt = fullPath.find_last_of(separator); + filename += lastSlashAt ? lastSlashAt + 1 : 0; + std::size_t sizeOfFilename = strlen(filename); + if (sizeOfFilename >= limit) { + filename += (sizeOfFilename - limit); + if (filename[0] != '.' && filename[1] != '.') { // prepend if not already + filename += 3; // 3 = '..' + STRCAT(buff, "..", limit); + } + } + STRCAT(buff, filename, limit); +} + +// Str + +bool +Str::wildCardMatch(const char* str, const char* pattern) { + while (*pattern) { + switch (*pattern) { + case '?': + if (!*str) + return false; + ++str; + ++pattern; + break; + case '*': + if (wildCardMatch(str, pattern + 1)) + return true; + if (*str && wildCardMatch(str + 1, pattern)) + return true; + return false; + default: + if (*str++ != *pattern++) + return false; + break; + } + } + return !*str && !*pattern; +} + +std::string& +Str::ltrim(std::string& str) { + str.erase(str.begin(), std::find_if(str.begin(), str.end(), [](char c) { return !std::isspace(c); })); + return str; +} + +std::string& +Str::rtrim(std::string& str) { + str.erase(std::find_if(str.rbegin(), str.rend(), [](char c) { return !std::isspace(c); }).base(), str.end()); + return str; +} + +std::string& +Str::trim(std::string& str) { + return ltrim(rtrim(str)); +} + +bool +Str::startsWith(const std::string& str, const std::string& start) { + return (str.length() >= start.length()) && (str.compare(0, start.length(), start) == 0); +} + +bool +Str::endsWith(const std::string& str, const std::string& end) { + return (str.length() >= end.length()) && (str.compare(str.length() - end.length(), end.length(), end) == 0); +} + +std::string& +Str::replaceAll(std::string& str, char replaceWhat, char replaceWith) { + std::replace(str.begin(), str.end(), replaceWhat, replaceWith); + return str; +} + +std::string& +Str::replaceAll(std::string& str, const std::string& replaceWhat, const std::string& replaceWith) { + if (replaceWhat == replaceWith) + return str; + std::size_t foundAt = std::string::npos; + while ((foundAt = str.find(replaceWhat, foundAt + 1)) != std::string::npos) { + str.replace(foundAt, replaceWhat.length(), replaceWith); + } + return str; +} + +void +Str::replaceFirstWithEscape(base::type::string_t& str, const base::type::string_t& replaceWhat, + const base::type::string_t& replaceWith) { + std::size_t foundAt = base::type::string_t::npos; + while ((foundAt = str.find(replaceWhat, foundAt + 1)) != base::type::string_t::npos) { + if (foundAt > 0 && str[foundAt - 1] == base::consts::kFormatSpecifierChar) { + str.erase(foundAt - 1, 1); + ++foundAt; + } else { + str.replace(foundAt, replaceWhat.length(), replaceWith); + return; + } + } +} +#if defined(ELPP_UNICODE) +void +Str::replaceFirstWithEscape(base::type::string_t& str, const base::type::string_t& replaceWhat, + const std::string& replaceWith) { + replaceFirstWithEscape(str, replaceWhat, base::type::string_t(replaceWith.begin(), replaceWith.end())); +} +#endif // defined(ELPP_UNICODE) + +std::string& +Str::toUpper(std::string& str) { + std::transform(str.begin(), str.end(), str.begin(), [](char c) { return static_cast(::toupper(c)); }); + return str; +} + +bool +Str::cStringEq(const char* s1, const char* s2) { + if (s1 == nullptr && s2 == nullptr) + return true; + if (s1 == nullptr || s2 == nullptr) + return false; + return strcmp(s1, s2) == 0; +} + +bool +Str::cStringCaseEq(const char* s1, const char* s2) { + if (s1 == nullptr && s2 == nullptr) + return true; + if (s1 == nullptr || s2 == nullptr) + return false; + + // With thanks to cygwin for this code + int d = 0; + + while (true) { + const int c1 = toupper(*s1++); + const int c2 = toupper(*s2++); + + if (((d = c1 - c2) != 0) || (c2 == '\0')) { + break; + } + } + + return d == 0; +} + +bool +Str::contains(const char* str, char c) { + for (; *str; ++str) { + if (*str == c) + return true; + } + return false; +} + +char* +Str::convertAndAddToBuff(std::size_t n, int len, char* buf, const char* bufLim, bool zeroPadded) { + char localBuff[10] = ""; + char* p = localBuff + sizeof(localBuff) - 2; + if (n > 0) { + for (; n > 0 && p > localBuff && len > 0; n /= 10, --len) *--p = static_cast(n % 10 + '0'); + } else { + *--p = '0'; + --len; + } + if (zeroPadded) + while (p > localBuff && len-- > 0) *--p = static_cast('0'); + return addToBuff(p, buf, bufLim); +} + +char* +Str::addToBuff(const char* str, char* buf, const char* bufLim) { + while ((buf < bufLim) && ((*buf = *str++) != '\0')) ++buf; + return buf; +} + +char* +Str::clearBuff(char buff[], std::size_t lim) { + STRCPY(buff, "", lim); + ELPP_UNUSED(lim); // For *nix we dont have anything using lim in above STRCPY macro + return buff; +} + +/// @brief Converst wchar* to char* +/// NOTE: Need to free return value after use! +char* +Str::wcharPtrToCharPtr(const wchar_t* line) { + std::size_t len_ = wcslen(line) + 1; + char* buff_ = static_cast(malloc(len_ + 1)); +#if ELPP_OS_UNIX || (ELPP_OS_WINDOWS && !ELPP_CRT_DBG_WARNINGS) + std::wcstombs(buff_, line, len_); +#elif ELPP_OS_WINDOWS + std::size_t convCount_ = 0; + mbstate_t mbState_; + ::memset(static_cast(&mbState_), 0, sizeof(mbState_)); + wcsrtombs_s(&convCount_, buff_, len_, &line, len_, &mbState_); +#endif // ELPP_OS_UNIX || (ELPP_OS_WINDOWS && !ELPP_CRT_DBG_WARNINGS) + return buff_; +} + +// OS + +#if ELPP_OS_WINDOWS +/// @brief Gets environment variables for Windows based OS. +/// We are not using getenv(const char*) because of CRT deprecation +/// @param varname Variable name to get environment variable value for +/// @return If variable exist the value of it otherwise nullptr +const char* +OS::getWindowsEnvironmentVariable(const char* varname) { + const DWORD bufferLen = 50; + static char buffer[bufferLen]; + if (GetEnvironmentVariableA(varname, buffer, bufferLen)) { + return buffer; + } + return nullptr; +} +#endif // ELPP_OS_WINDOWS +#if ELPP_OS_ANDROID +std::string +OS::getProperty(const char* prop) { + char propVal[PROP_VALUE_MAX + 1]; + int ret = __system_property_get(prop, propVal); + return ret == 0 ? std::string() : std::string(propVal); +} + +std::string +OS::getDeviceName(void) { + std::stringstream ss; + std::string manufacturer = getProperty("ro.product.manufacturer"); + std::string model = getProperty("ro.product.model"); + if (manufacturer.empty() || model.empty()) { + return std::string(); + } + ss << manufacturer << "-" << model; + return ss.str(); +} +#endif // ELPP_OS_ANDROID + +const std::string +OS::getBashOutput(const char* command) { +#if (ELPP_OS_UNIX && !ELPP_OS_ANDROID && !ELPP_CYGWIN) + if (command == nullptr) { + return std::string(); + } + FILE* proc = nullptr; + if ((proc = popen(command, "r")) == nullptr) { + ELPP_INTERNAL_ERROR("\nUnable to run command [" << command << "]", true); + return std::string(); + } + char hBuff[4096]; + if (fgets(hBuff, sizeof(hBuff), proc) != nullptr) { + pclose(proc); + const std::size_t buffLen = strlen(hBuff); + if (buffLen > 0 && hBuff[buffLen - 1] == '\n') { + hBuff[buffLen - 1] = '\0'; + } + return std::string(hBuff); + } else { + pclose(proc); + } + return std::string(); +#else + ELPP_UNUSED(command); + return std::string(); +#endif // (ELPP_OS_UNIX && !ELPP_OS_ANDROID && !ELPP_CYGWIN) +} + +std::string +OS::getEnvironmentVariable(const char* variableName, const char* defaultVal, const char* alternativeBashCommand) { +#if ELPP_OS_UNIX + const char* val = getenv(variableName); +#elif ELPP_OS_WINDOWS + const char* val = getWindowsEnvironmentVariable(variableName); +#endif // ELPP_OS_UNIX + if ((val == nullptr) || ((strcmp(val, "") == 0))) { +#if ELPP_OS_UNIX && defined(ELPP_FORCE_ENV_VAR_FROM_BASH) + // Try harder on unix-based systems + std::string valBash = base::utils::OS::getBashOutput(alternativeBashCommand); + if (valBash.empty()) { + return std::string(defaultVal); + } else { + return valBash; + } +#elif ELPP_OS_WINDOWS || ELPP_OS_UNIX + ELPP_UNUSED(alternativeBashCommand); + return std::string(defaultVal); +#endif // ELPP_OS_UNIX && defined(ELPP_FORCE_ENV_VAR_FROM_BASH) + } + return std::string(val); +} + +std::string +OS::currentUser(void) { +#if ELPP_OS_UNIX && !ELPP_OS_ANDROID + return getEnvironmentVariable("USER", base::consts::kUnknownUser, "whoami"); +#elif ELPP_OS_WINDOWS + return getEnvironmentVariable("USERNAME", base::consts::kUnknownUser); +#elif ELPP_OS_ANDROID + ELPP_UNUSED(base::consts::kUnknownUser); + return std::string("android"); +#else + return std::string(); +#endif // ELPP_OS_UNIX && !ELPP_OS_ANDROID +} + +std::string +OS::currentHost(void) { +#if ELPP_OS_UNIX && !ELPP_OS_ANDROID + return getEnvironmentVariable("HOSTNAME", base::consts::kUnknownHost, "hostname"); +#elif ELPP_OS_WINDOWS + return getEnvironmentVariable("COMPUTERNAME", base::consts::kUnknownHost); +#elif ELPP_OS_ANDROID + ELPP_UNUSED(base::consts::kUnknownHost); + return getDeviceName(); +#else + return std::string(); +#endif // ELPP_OS_UNIX && !ELPP_OS_ANDROID +} + +bool +OS::termSupportsColor(void) { + std::string term = getEnvironmentVariable("TERM", ""); + return term == "xterm" || term == "xterm-color" || term == "xterm-256color" || term == "screen" || + term == "linux" || term == "cygwin" || term == "screen-256color"; +} + +// DateTime + +void +DateTime::gettimeofday(struct timeval* tv) { +#if ELPP_OS_WINDOWS + if (tv != nullptr) { +#if ELPP_COMPILER_MSVC || defined(_MSC_EXTENSIONS) + const unsigned __int64 delta_ = 11644473600000000Ui64; +#else + const unsigned __int64 delta_ = 11644473600000000ULL; +#endif // ELPP_COMPILER_MSVC || defined(_MSC_EXTENSIONS) + const double secOffSet = 0.000001; + const unsigned long usecOffSet = 1000000; + FILETIME fileTime; + GetSystemTimeAsFileTime(&fileTime); + unsigned __int64 present = 0; + present |= fileTime.dwHighDateTime; + present = present << 32; + present |= fileTime.dwLowDateTime; + present /= 10; // mic-sec + // Subtract the difference + present -= delta_; + tv->tv_sec = static_cast(present * secOffSet); + tv->tv_usec = static_cast(present % usecOffSet); + } +#else + ::gettimeofday(tv, nullptr); +#endif // ELPP_OS_WINDOWS +} + +std::string +DateTime::getDateTime(const char* format, const base::SubsecondPrecision* ssPrec) { + struct timeval currTime; + gettimeofday(&currTime); + return timevalToString(currTime, format, ssPrec); +} + +std::string +DateTime::timevalToString(struct timeval tval, const char* format, const el::base::SubsecondPrecision* ssPrec) { + struct ::tm timeInfo; + buildTimeInfo(&tval, &timeInfo); + const int kBuffSize = 30; + char buff_[kBuffSize] = ""; + parseFormat(buff_, kBuffSize, format, &timeInfo, static_cast(tval.tv_usec / ssPrec->m_offset), ssPrec); + return std::string(buff_); +} + +base::type::string_t +DateTime::formatTime(unsigned long long time, base::TimestampUnit timestampUnit) { + base::type::EnumType start = static_cast(timestampUnit); + const base::type::char_t* unit = base::consts::kTimeFormats[start].unit; + for (base::type::EnumType i = start; i < base::consts::kTimeFormatsCount - 1; ++i) { + if (time <= base::consts::kTimeFormats[i].value) { + break; + } + if (base::consts::kTimeFormats[i].value == 1000.0f && time / 1000.0f < 1.9f) { + break; + } + time /= static_cast(base::consts::kTimeFormats[i].value); + unit = base::consts::kTimeFormats[i + 1].unit; + } + base::type::stringstream_t ss; + ss << time << " " << unit; + return ss.str(); +} + +unsigned long long +DateTime::getTimeDifference(const struct timeval& endTime, const struct timeval& startTime, + base::TimestampUnit timestampUnit) { + if (timestampUnit == base::TimestampUnit::Microsecond) { + return static_cast( + static_cast(1000000 * endTime.tv_sec + endTime.tv_usec) - + static_cast(1000000 * startTime.tv_sec + startTime.tv_usec)); + } + // milliseconds + auto conv = [](const struct timeval& tim) { + return static_cast((tim.tv_sec * 1000) + (tim.tv_usec / 1000)); + }; + return static_cast(conv(endTime) - conv(startTime)); +} + +struct ::tm* +DateTime::buildTimeInfo(struct timeval* currTime, struct ::tm* timeInfo) { +#if ELPP_OS_UNIX + time_t rawTime = currTime->tv_sec; + ::elpptime_r(&rawTime, timeInfo); + return timeInfo; +#else +#if ELPP_COMPILER_MSVC + ELPP_UNUSED(currTime); + time_t t; +#if defined(_USE_32BIT_TIME_T) + _time32(&t); +#else + _time64(&t); +#endif + elpptime_s(timeInfo, &t); + return timeInfo; +#else + // For any other compilers that don't have CRT warnings issue e.g, MinGW or TDM GCC- we use different method + time_t rawTime = currTime->tv_sec; + struct tm* tmInf = elpptime(&rawTime); + *timeInfo = *tmInf; + return timeInfo; +#endif // ELPP_COMPILER_MSVC +#endif // ELPP_OS_UNIX +} + +char* +DateTime::parseFormat(char* buf, std::size_t bufSz, const char* format, const struct tm* tInfo, std::size_t msec, + const base::SubsecondPrecision* ssPrec) { + const char* bufLim = buf + bufSz; + for (; *format; ++format) { + if (*format == base::consts::kFormatSpecifierChar) { + switch (*++format) { + case base::consts::kFormatSpecifierChar: // Escape + break; + case '\0': // End + --format; + break; + case 'd': // Day + buf = base::utils::Str::convertAndAddToBuff(tInfo->tm_mday, 2, buf, bufLim); + continue; + case 'a': // Day of week (short) + buf = base::utils::Str::addToBuff(base::consts::kDaysAbbrev[tInfo->tm_wday], buf, bufLim); + continue; + case 'A': // Day of week (long) + buf = base::utils::Str::addToBuff(base::consts::kDays[tInfo->tm_wday], buf, bufLim); + continue; + case 'M': // month + buf = base::utils::Str::convertAndAddToBuff(tInfo->tm_mon + 1, 2, buf, bufLim); + continue; + case 'b': // month (short) + buf = base::utils::Str::addToBuff(base::consts::kMonthsAbbrev[tInfo->tm_mon], buf, bufLim); + continue; + case 'B': // month (long) + buf = base::utils::Str::addToBuff(base::consts::kMonths[tInfo->tm_mon], buf, bufLim); + continue; + case 'y': // year (two digits) + buf = + base::utils::Str::convertAndAddToBuff(tInfo->tm_year + base::consts::kYearBase, 2, buf, bufLim); + continue; + case 'Y': // year (four digits) + buf = + base::utils::Str::convertAndAddToBuff(tInfo->tm_year + base::consts::kYearBase, 4, buf, bufLim); + continue; + case 'h': // hour (12-hour) + buf = base::utils::Str::convertAndAddToBuff(tInfo->tm_hour % 12, 2, buf, bufLim); + continue; + case 'H': // hour (24-hour) + buf = base::utils::Str::convertAndAddToBuff(tInfo->tm_hour, 2, buf, bufLim); + continue; + case 'm': // minute + buf = base::utils::Str::convertAndAddToBuff(tInfo->tm_min, 2, buf, bufLim); + continue; + case 's': // second + buf = base::utils::Str::convertAndAddToBuff(tInfo->tm_sec, 2, buf, bufLim); + continue; + case 'z': // subsecond part + case 'g': + buf = base::utils::Str::convertAndAddToBuff(msec, ssPrec->m_width, buf, bufLim); + continue; + case 'F': // AM/PM + buf = base::utils::Str::addToBuff((tInfo->tm_hour >= 12) ? base::consts::kPm : base::consts::kAm, + buf, bufLim); + continue; + default: + continue; + } + } + if (buf == bufLim) + break; + *buf++ = *format; + } + return buf; +} + +// CommandLineArgs + +void +CommandLineArgs::setArgs(int argc, char** argv) { + m_params.clear(); + m_paramsWithValue.clear(); + if (argc == 0 || argv == nullptr) { + return; + } + m_argc = argc; + m_argv = argv; + for (int i = 1; i < m_argc; ++i) { + const char* v = (strstr(m_argv[i], "=")); + if (v != nullptr && strlen(v) > 0) { + std::string key = std::string(m_argv[i]); + key = key.substr(0, key.find_first_of('=')); + if (hasParamWithValue(key.c_str())) { + ELPP_INTERNAL_INFO(1, "Skipping [" << key << "] arg since it already has value [" + << getParamValue(key.c_str()) << "]"); + } else { + m_paramsWithValue.insert(std::make_pair(key, std::string(v + 1))); + } + } + if (v == nullptr) { + if (hasParam(m_argv[i])) { + ELPP_INTERNAL_INFO(1, "Skipping [" << m_argv[i] << "] arg since it already exists"); + } else { + m_params.push_back(std::string(m_argv[i])); + } + } + } +} + +bool +CommandLineArgs::hasParamWithValue(const char* paramKey) const { + return m_paramsWithValue.find(std::string(paramKey)) != m_paramsWithValue.end(); +} + +const char* +CommandLineArgs::getParamValue(const char* paramKey) const { + std::unordered_map::const_iterator iter = m_paramsWithValue.find(std::string(paramKey)); + return iter != m_paramsWithValue.end() ? iter->second.c_str() : ""; +} + +bool +CommandLineArgs::hasParam(const char* paramKey) const { + return std::find(m_params.begin(), m_params.end(), std::string(paramKey)) != m_params.end(); +} + +bool +CommandLineArgs::empty(void) const { + return m_params.empty() && m_paramsWithValue.empty(); +} + +std::size_t +CommandLineArgs::size(void) const { + return m_params.size() + m_paramsWithValue.size(); +} + +base::type::ostream_t& +operator<<(base::type::ostream_t& os, const CommandLineArgs& c) { + for (int i = 1; i < c.m_argc; ++i) { + os << ELPP_LITERAL("[") << c.m_argv[i] << ELPP_LITERAL("]"); + if (i < c.m_argc - 1) { + os << ELPP_LITERAL(" "); + } + } + return os; +} + +} // namespace utils + +// el::base::threading +namespace threading { + +#if ELPP_THREADING_ENABLED +#if ELPP_USE_STD_THREADING +#if ELPP_ASYNC_LOGGING +static void +msleep(int ms) { + // Only when async logging enabled - this is because async is strict on compiler +#if defined(ELPP_NO_SLEEP_FOR) + usleep(ms * 1000); +#else + std::this_thread::sleep_for(std::chrono::milliseconds(ms)); +#endif // defined(ELPP_NO_SLEEP_FOR) +} +#endif // ELPP_ASYNC_LOGGING +#endif // !ELPP_USE_STD_THREADING +#endif // ELPP_THREADING_ENABLED + +} // namespace threading + +// el::base + +// SubsecondPrecision + +void +SubsecondPrecision::init(int width) { + if (width < 1 || width > 6) { + width = base::consts::kDefaultSubsecondPrecision; + } + m_width = width; + switch (m_width) { + case 3: + m_offset = 1000; + break; + case 4: + m_offset = 100; + break; + case 5: + m_offset = 10; + break; + case 6: + m_offset = 1; + break; + default: + m_offset = 1000; + break; + } +} + +// LogFormat + +LogFormat::LogFormat(void) + : m_level(Level::Unknown), + m_userFormat(base::type::string_t()), + m_format(base::type::string_t()), + m_dateTimeFormat(std::string()), + m_flags(0x0), + m_currentUser(base::utils::OS::currentUser()), + m_currentHost(base::utils::OS::currentHost()) { +} + +LogFormat::LogFormat(Level level, const base::type::string_t& format) + : m_level(level), + m_userFormat(format), + m_currentUser(base::utils::OS::currentUser()), + m_currentHost(base::utils::OS::currentHost()) { + parseFromFormat(m_userFormat); +} + +LogFormat::LogFormat(const LogFormat& logFormat) + : m_level(logFormat.m_level), + m_userFormat(logFormat.m_userFormat), + m_format(logFormat.m_format), + m_dateTimeFormat(logFormat.m_dateTimeFormat), + m_flags(logFormat.m_flags), + m_currentUser(logFormat.m_currentUser), + m_currentHost(logFormat.m_currentHost) { +} + +LogFormat::LogFormat(LogFormat&& logFormat) { + m_level = std::move(logFormat.m_level); + m_userFormat = std::move(logFormat.m_userFormat); + m_format = std::move(logFormat.m_format); + m_dateTimeFormat = std::move(logFormat.m_dateTimeFormat); + m_flags = std::move(logFormat.m_flags); + m_currentUser = std::move(logFormat.m_currentUser); + m_currentHost = std::move(logFormat.m_currentHost); +} + +LogFormat& +LogFormat::operator=(const LogFormat& logFormat) { + if (&logFormat != this) { + m_level = logFormat.m_level; + m_userFormat = logFormat.m_userFormat; + m_dateTimeFormat = logFormat.m_dateTimeFormat; + m_flags = logFormat.m_flags; + m_currentUser = logFormat.m_currentUser; + m_currentHost = logFormat.m_currentHost; + } + return *this; +} + +bool +LogFormat::operator==(const LogFormat& other) { + return m_level == other.m_level && m_userFormat == other.m_userFormat && m_format == other.m_format && + m_dateTimeFormat == other.m_dateTimeFormat && m_flags == other.m_flags; +} + +/// @brief Updates format to be used while logging. +/// @param userFormat User provided format +void +LogFormat::parseFromFormat(const base::type::string_t& userFormat) { + // We make copy because we will be changing the format + // i.e, removing user provided date format from original format + // and then storing it. + base::type::string_t formatCopy = userFormat; + m_flags = 0x0; + auto conditionalAddFlag = [&](const base::type::char_t* specifier, base::FormatFlags flag) { + std::size_t foundAt = base::type::string_t::npos; + while ((foundAt = formatCopy.find(specifier, foundAt + 1)) != base::type::string_t::npos) { + if (foundAt > 0 && formatCopy[foundAt - 1] == base::consts::kFormatSpecifierChar) { + if (hasFlag(flag)) { + // If we already have flag we remove the escape chars so that '%%' is turned to '%' + // even after specifier resolution - this is because we only replaceFirst specifier + formatCopy.erase(foundAt - 1, 1); + ++foundAt; + } + } else { + if (!hasFlag(flag)) + addFlag(flag); + } + } + }; + conditionalAddFlag(base::consts::kAppNameFormatSpecifier, base::FormatFlags::AppName); + conditionalAddFlag(base::consts::kSeverityLevelFormatSpecifier, base::FormatFlags::Level); + conditionalAddFlag(base::consts::kSeverityLevelShortFormatSpecifier, base::FormatFlags::LevelShort); + conditionalAddFlag(base::consts::kLoggerIdFormatSpecifier, base::FormatFlags::LoggerId); + conditionalAddFlag(base::consts::kThreadIdFormatSpecifier, base::FormatFlags::ThreadId); + conditionalAddFlag(base::consts::kLogFileFormatSpecifier, base::FormatFlags::File); + conditionalAddFlag(base::consts::kLogFileBaseFormatSpecifier, base::FormatFlags::FileBase); + conditionalAddFlag(base::consts::kLogLineFormatSpecifier, base::FormatFlags::Line); + conditionalAddFlag(base::consts::kLogLocationFormatSpecifier, base::FormatFlags::Location); + conditionalAddFlag(base::consts::kLogFunctionFormatSpecifier, base::FormatFlags::Function); + conditionalAddFlag(base::consts::kCurrentUserFormatSpecifier, base::FormatFlags::User); + conditionalAddFlag(base::consts::kCurrentHostFormatSpecifier, base::FormatFlags::Host); + conditionalAddFlag(base::consts::kMessageFormatSpecifier, base::FormatFlags::LogMessage); + conditionalAddFlag(base::consts::kVerboseLevelFormatSpecifier, base::FormatFlags::VerboseLevel); + // For date/time we need to extract user's date format first + std::size_t dateIndex = std::string::npos; + if ((dateIndex = formatCopy.find(base::consts::kDateTimeFormatSpecifier)) != std::string::npos) { + while (dateIndex > 0 && formatCopy[dateIndex - 1] == base::consts::kFormatSpecifierChar) { + dateIndex = formatCopy.find(base::consts::kDateTimeFormatSpecifier, dateIndex + 1); + } + if (dateIndex != std::string::npos) { + addFlag(base::FormatFlags::DateTime); + updateDateFormat(dateIndex, formatCopy); + } + } + m_format = formatCopy; + updateFormatSpec(); +} + +void +LogFormat::updateDateFormat(std::size_t index, base::type::string_t& currFormat) { + if (hasFlag(base::FormatFlags::DateTime)) { + index += ELPP_STRLEN(base::consts::kDateTimeFormatSpecifier); + } + const base::type::char_t* ptr = currFormat.c_str() + index; + if ((currFormat.size() > index) && (ptr[0] == '{')) { + // User has provided format for date/time + ++ptr; + int count = 1; // Start by 1 in order to remove starting brace + std::stringstream ss; + for (; *ptr; ++ptr, ++count) { + if (*ptr == '}') { + ++count; // In order to remove ending brace + break; + } + ss << static_cast(*ptr); + } + currFormat.erase(index, count); + m_dateTimeFormat = ss.str(); + } else { + // No format provided, use default + if (hasFlag(base::FormatFlags::DateTime)) { + m_dateTimeFormat = std::string(base::consts::kDefaultDateTimeFormat); + } + } +} + +void +LogFormat::updateFormatSpec(void) { + // Do not use switch over strongly typed enums because Intel C++ compilers dont support them yet. + if (m_level == Level::Debug) { + base::utils::Str::replaceFirstWithEscape(m_format, base::consts::kSeverityLevelFormatSpecifier, + base::consts::kDebugLevelLogValue); + base::utils::Str::replaceFirstWithEscape(m_format, base::consts::kSeverityLevelShortFormatSpecifier, + base::consts::kDebugLevelShortLogValue); + } else if (m_level == Level::Info) { + base::utils::Str::replaceFirstWithEscape(m_format, base::consts::kSeverityLevelFormatSpecifier, + base::consts::kInfoLevelLogValue); + base::utils::Str::replaceFirstWithEscape(m_format, base::consts::kSeverityLevelShortFormatSpecifier, + base::consts::kInfoLevelShortLogValue); + } else if (m_level == Level::Warning) { + base::utils::Str::replaceFirstWithEscape(m_format, base::consts::kSeverityLevelFormatSpecifier, + base::consts::kWarningLevelLogValue); + base::utils::Str::replaceFirstWithEscape(m_format, base::consts::kSeverityLevelShortFormatSpecifier, + base::consts::kWarningLevelShortLogValue); + } else if (m_level == Level::Error) { + base::utils::Str::replaceFirstWithEscape(m_format, base::consts::kSeverityLevelFormatSpecifier, + base::consts::kErrorLevelLogValue); + base::utils::Str::replaceFirstWithEscape(m_format, base::consts::kSeverityLevelShortFormatSpecifier, + base::consts::kErrorLevelShortLogValue); + } else if (m_level == Level::Fatal) { + base::utils::Str::replaceFirstWithEscape(m_format, base::consts::kSeverityLevelFormatSpecifier, + base::consts::kFatalLevelLogValue); + base::utils::Str::replaceFirstWithEscape(m_format, base::consts::kSeverityLevelShortFormatSpecifier, + base::consts::kFatalLevelShortLogValue); + } else if (m_level == Level::Verbose) { + base::utils::Str::replaceFirstWithEscape(m_format, base::consts::kSeverityLevelFormatSpecifier, + base::consts::kVerboseLevelLogValue); + base::utils::Str::replaceFirstWithEscape(m_format, base::consts::kSeverityLevelShortFormatSpecifier, + base::consts::kVerboseLevelShortLogValue); + } else if (m_level == Level::Trace) { + base::utils::Str::replaceFirstWithEscape(m_format, base::consts::kSeverityLevelFormatSpecifier, + base::consts::kTraceLevelLogValue); + base::utils::Str::replaceFirstWithEscape(m_format, base::consts::kSeverityLevelShortFormatSpecifier, + base::consts::kTraceLevelShortLogValue); + } + if (hasFlag(base::FormatFlags::User)) { + base::utils::Str::replaceFirstWithEscape(m_format, base::consts::kCurrentUserFormatSpecifier, m_currentUser); + } + if (hasFlag(base::FormatFlags::Host)) { + base::utils::Str::replaceFirstWithEscape(m_format, base::consts::kCurrentHostFormatSpecifier, m_currentHost); + } + // Ignore Level::Global and Level::Unknown +} + +// TypedConfigurations + +TypedConfigurations::TypedConfigurations(Configurations* configurations, + base::LogStreamsReferenceMap* logStreamsReference) { + m_configurations = configurations; + m_logStreamsReference = logStreamsReference; + build(m_configurations); +} + +TypedConfigurations::TypedConfigurations(const TypedConfigurations& other) { + this->m_configurations = other.m_configurations; + this->m_logStreamsReference = other.m_logStreamsReference; + build(m_configurations); +} + +bool +TypedConfigurations::enabled(Level level) { + return getConfigByVal(level, &m_enabledMap, "enabled"); +} + +bool +TypedConfigurations::toFile(Level level) { + return getConfigByVal(level, &m_toFileMap, "toFile"); +} + +const std::string& +TypedConfigurations::filename(Level level) { + return getConfigByRef(level, &m_filenameMap, "filename"); +} + +bool +TypedConfigurations::toStandardOutput(Level level) { + return getConfigByVal(level, &m_toStandardOutputMap, "toStandardOutput"); +} + +const base::LogFormat& +TypedConfigurations::logFormat(Level level) { + return getConfigByRef(level, &m_logFormatMap, "logFormat"); +} + +const base::SubsecondPrecision& +TypedConfigurations::subsecondPrecision(Level level) { + return getConfigByRef(level, &m_subsecondPrecisionMap, "subsecondPrecision"); +} + +const base::MillisecondsWidth& +TypedConfigurations::millisecondsWidth(Level level) { + return getConfigByRef(level, &m_subsecondPrecisionMap, "millisecondsWidth"); +} + +bool +TypedConfigurations::performanceTracking(Level level) { + return getConfigByVal(level, &m_performanceTrackingMap, "performanceTracking"); +} + +base::type::fstream_t* +TypedConfigurations::fileStream(Level level) { + return getConfigByRef(level, &m_fileStreamMap, "fileStream").get(); +} + +std::size_t +TypedConfigurations::maxLogFileSize(Level level) { + return getConfigByVal(level, &m_maxLogFileSizeMap, "maxLogFileSize"); +} + +std::size_t +TypedConfigurations::logFlushThreshold(Level level) { + return getConfigByVal(level, &m_logFlushThresholdMap, "logFlushThreshold"); +} + +void +TypedConfigurations::build(Configurations* configurations) { + base::threading::ScopedLock scopedLock(lock()); + auto getBool = [](std::string boolStr) -> bool { // Pass by value for trimming + base::utils::Str::trim(boolStr); + return (boolStr == "TRUE" || boolStr == "true" || boolStr == "1"); + }; + std::vector withFileSizeLimit; + for (Configurations::const_iterator it = configurations->begin(); it != configurations->end(); ++it) { + Configuration* conf = *it; + // We cannot use switch on strong enums because Intel C++ dont support them yet + if (conf->configurationType() == ConfigurationType::Enabled) { + setValue(conf->level(), getBool(conf->value()), &m_enabledMap); + } else if (conf->configurationType() == ConfigurationType::ToFile) { + setValue(conf->level(), getBool(conf->value()), &m_toFileMap); + } else if (conf->configurationType() == ConfigurationType::ToStandardOutput) { + setValue(conf->level(), getBool(conf->value()), &m_toStandardOutputMap); + } else if (conf->configurationType() == ConfigurationType::Filename) { + // We do not yet configure filename but we will configure in another + // loop. This is because if file cannot be created, we will force ToFile + // to be false. Because configuring logger is not necessarily performance + // sensative operation, we can live with another loop; (by the way this loop + // is not very heavy either) + } else if (conf->configurationType() == ConfigurationType::Format) { + setValue(conf->level(), + base::LogFormat(conf->level(), base::type::string_t(conf->value().begin(), conf->value().end())), + &m_logFormatMap); + } else if (conf->configurationType() == ConfigurationType::SubsecondPrecision) { + setValue(Level::Global, base::SubsecondPrecision(static_cast(getULong(conf->value()))), + &m_subsecondPrecisionMap); + } else if (conf->configurationType() == ConfigurationType::PerformanceTracking) { + setValue(Level::Global, getBool(conf->value()), &m_performanceTrackingMap); + } else if (conf->configurationType() == ConfigurationType::MaxLogFileSize) { + auto v = getULong(conf->value()); + setValue(conf->level(), static_cast(v), &m_maxLogFileSizeMap); + if (v != 0) { + withFileSizeLimit.push_back(conf); + } + } else if (conf->configurationType() == ConfigurationType::LogFlushThreshold) { + setValue(conf->level(), static_cast(getULong(conf->value())), &m_logFlushThresholdMap); + } + } + // As mentioned earlier, we will now set filename configuration in separate loop to deal with non-existent files + for (Configurations::const_iterator it = configurations->begin(); it != configurations->end(); ++it) { + Configuration* conf = *it; + if (conf->configurationType() == ConfigurationType::Filename) { + insertFile(conf->level(), conf->value()); + } + } + for (std::vector::iterator conf = withFileSizeLimit.begin(); conf != withFileSizeLimit.end(); + ++conf) { + // This is not unsafe as mutex is locked in currect scope + unsafeValidateFileRolling((*conf)->level(), base::defaultPreRollOutCallback); + } +} + +unsigned long +TypedConfigurations::getULong(std::string confVal) { + bool valid = true; + base::utils::Str::trim(confVal); + valid = !confVal.empty() && std::find_if(confVal.begin(), confVal.end(), + [](char c) { return !base::utils::Str::isDigit(c); }) == confVal.end(); + if (!valid) { + valid = false; + ELPP_ASSERT(valid, "Configuration value not a valid integer [" << confVal << "]"); + return 0; + } + return atol(confVal.c_str()); +} + +std::string +TypedConfigurations::resolveFilename(const std::string& filename) { + std::string resultingFilename = filename; + std::size_t dateIndex = std::string::npos; + std::string dateTimeFormatSpecifierStr = std::string(base::consts::kDateTimeFormatSpecifierForFilename); + if ((dateIndex = resultingFilename.find(dateTimeFormatSpecifierStr.c_str())) != std::string::npos) { + while (dateIndex > 0 && resultingFilename[dateIndex - 1] == base::consts::kFormatSpecifierChar) { + dateIndex = resultingFilename.find(dateTimeFormatSpecifierStr.c_str(), dateIndex + 1); + } + if (dateIndex != std::string::npos) { + const char* ptr = resultingFilename.c_str() + dateIndex; + // Goto end of specifier + ptr += dateTimeFormatSpecifierStr.size(); + std::string fmt; + if ((resultingFilename.size() > dateIndex) && (ptr[0] == '{')) { + // User has provided format for date/time + ++ptr; + int count = 1; // Start by 1 in order to remove starting brace + std::stringstream ss; + for (; *ptr; ++ptr, ++count) { + if (*ptr == '}') { + ++count; // In order to remove ending brace + break; + } + ss << *ptr; + } + resultingFilename.erase(dateIndex + dateTimeFormatSpecifierStr.size(), count); + fmt = ss.str(); + } else { + fmt = std::string(base::consts::kDefaultDateTimeFormatInFilename); + } + base::SubsecondPrecision ssPrec(3); + std::string now = base::utils::DateTime::getDateTime(fmt.c_str(), &ssPrec); + base::utils::Str::replaceAll(now, '/', '-'); // Replace path element since we are dealing with filename + base::utils::Str::replaceAll(resultingFilename, dateTimeFormatSpecifierStr, now); + } + } + return resultingFilename; +} + +void +TypedConfigurations::insertFile(Level level, const std::string& fullFilename) { + std::string resolvedFilename = resolveFilename(fullFilename); + if (resolvedFilename.empty()) { + std::cerr << "Could not load empty file for logging, please re-check your configurations for level [" + << LevelHelper::convertToString(level) << "]"; + } + std::string filePath = + base::utils::File::extractPathFromFilename(resolvedFilename, base::consts::kFilePathSeperator); + if (filePath.size() < resolvedFilename.size()) { + base::utils::File::createPath(filePath); + } + auto create = [&](Level level) { + base::LogStreamsReferenceMap::iterator filestreamIter = m_logStreamsReference->find(resolvedFilename); + base::type::fstream_t* fs = nullptr; + if (filestreamIter == m_logStreamsReference->end()) { + // We need a completely new stream, nothing to share with + fs = base::utils::File::newFileStream(resolvedFilename); + m_filenameMap.insert(std::make_pair(level, resolvedFilename)); + m_fileStreamMap.insert(std::make_pair(level, base::FileStreamPtr(fs))); + m_logStreamsReference->insert( + std::make_pair(resolvedFilename, base::FileStreamPtr(m_fileStreamMap.at(level)))); + } else { + // Woops! we have an existing one, share it! + m_filenameMap.insert(std::make_pair(level, filestreamIter->first)); + m_fileStreamMap.insert(std::make_pair(level, base::FileStreamPtr(filestreamIter->second))); + fs = filestreamIter->second.get(); + } + if (fs == nullptr) { + // We display bad file error from newFileStream() + ELPP_INTERNAL_ERROR("Setting [TO_FILE] of [" << LevelHelper::convertToString(level) << "] to FALSE", false); + setValue(level, false, &m_toFileMap); + } + }; + // If we dont have file conf for any level, create it for Level::Global first + // otherwise create for specified level + create(m_filenameMap.empty() && m_fileStreamMap.empty() ? Level::Global : level); +} + +bool +TypedConfigurations::unsafeValidateFileRolling(Level level, const PreRollOutCallback& preRollOutCallback) { + base::type::fstream_t* fs = unsafeGetConfigByRef(level, &m_fileStreamMap, "fileStream").get(); + if (fs == nullptr) { + return true; + } + std::size_t maxLogFileSize = unsafeGetConfigByVal(level, &m_maxLogFileSizeMap, "maxLogFileSize"); + std::size_t currFileSize = base::utils::File::getSizeOfFile(fs); + if (maxLogFileSize != 0 && currFileSize >= maxLogFileSize) { + std::string fname = unsafeGetConfigByRef(level, &m_filenameMap, "filename"); + ELPP_INTERNAL_INFO(1, "Truncating log file [" << fname << "] as a result of configurations for level [" + << LevelHelper::convertToString(level) << "]"); + fs->close(); + preRollOutCallback(fname.c_str(), currFileSize, level); + fs->open(fname, std::fstream::out | std::fstream::trunc); + return true; + } + return false; +} + +// RegisteredHitCounters + +bool +RegisteredHitCounters::validateEveryN(const char* filename, base::type::LineNumber lineNumber, std::size_t n) { + base::threading::ScopedLock scopedLock(lock()); + base::HitCounter* counter = get(filename, lineNumber); + if (counter == nullptr) { + registerNew(counter = new base::HitCounter(filename, lineNumber)); + } + counter->validateHitCounts(n); + bool result = (n >= 1 && counter->hitCounts() != 0 && counter->hitCounts() % n == 0); + return result; +} + +/// @brief Validates counter for hits >= N, i.e, registers new if does not exist otherwise updates original one +/// @return True if validation resulted in triggering hit. Meaning logs should be written everytime true is returned +bool +RegisteredHitCounters::validateAfterN(const char* filename, base::type::LineNumber lineNumber, std::size_t n) { + base::threading::ScopedLock scopedLock(lock()); + base::HitCounter* counter = get(filename, lineNumber); + if (counter == nullptr) { + registerNew(counter = new base::HitCounter(filename, lineNumber)); + } + // Do not use validateHitCounts here since we do not want to reset counter here + // Note the >= instead of > because we are incrementing + // after this check + if (counter->hitCounts() >= n) + return true; + counter->increment(); + return false; +} + +/// @brief Validates counter for hits are <= n, i.e, registers new if does not exist otherwise updates original one +/// @return True if validation resulted in triggering hit. Meaning logs should be written everytime true is returned +bool +RegisteredHitCounters::validateNTimes(const char* filename, base::type::LineNumber lineNumber, std::size_t n) { + base::threading::ScopedLock scopedLock(lock()); + base::HitCounter* counter = get(filename, lineNumber); + if (counter == nullptr) { + registerNew(counter = new base::HitCounter(filename, lineNumber)); + } + counter->increment(); + // Do not use validateHitCounts here since we do not want to reset counter here + if (counter->hitCounts() <= n) + return true; + return false; +} + +// RegisteredLoggers + +RegisteredLoggers::RegisteredLoggers(const LogBuilderPtr& defaultLogBuilder) : m_defaultLogBuilder(defaultLogBuilder) { + m_defaultConfigurations.setToDefault(); +} + +Logger* +RegisteredLoggers::get(const std::string& id, bool forceCreation) { + base::threading::ScopedLock scopedLock(lock()); + Logger* logger_ = base::utils::Registry::get(id); + if (logger_ == nullptr && forceCreation) { + bool validId = Logger::isValidId(id); + if (!validId) { + ELPP_ASSERT(validId, "Invalid logger ID [" << id << "]. Not registering this logger."); + return nullptr; + } + logger_ = new Logger(id, m_defaultConfigurations, &m_logStreamsReference); + logger_->m_logBuilder = m_defaultLogBuilder; + registerNew(id, logger_); + LoggerRegistrationCallback* callback = nullptr; + for (const std::pair& h : + m_loggerRegistrationCallbacks) { + callback = h.second.get(); + if (callback != nullptr && callback->enabled()) { + callback->handle(logger_); + } + } + } + return logger_; +} + +bool +RegisteredLoggers::remove(const std::string& id) { + if (id == base::consts::kDefaultLoggerId) { + return false; + } + // get has internal lock + Logger* logger = base::utils::Registry::get(id); + if (logger != nullptr) { + // unregister has internal lock + unregister(logger); + } + return true; +} + +void +RegisteredLoggers::unsafeFlushAll(void) { + ELPP_INTERNAL_INFO(1, "Flushing all log files"); + for (base::LogStreamsReferenceMap::iterator it = m_logStreamsReference.begin(); it != m_logStreamsReference.end(); + ++it) { + if (it->second.get() == nullptr) + continue; + it->second->flush(); + } +} + +// VRegistry + +VRegistry::VRegistry(base::type::VerboseLevel level, base::type::EnumType* pFlags) : m_level(level), m_pFlags(pFlags) { +} + +/// @brief Sets verbose level. Accepted range is 0-9 +void +VRegistry::setLevel(base::type::VerboseLevel level) { + base::threading::ScopedLock scopedLock(lock()); + if (level > 9) + m_level = base::consts::kMaxVerboseLevel; + else + m_level = level; +} + +void +VRegistry::setModules(const char* modules) { + base::threading::ScopedLock scopedLock(lock()); + auto addSuffix = [](std::stringstream& ss, const char* sfx, const char* prev) { + if (prev != nullptr && base::utils::Str::endsWith(ss.str(), std::string(prev))) { + std::string chr(ss.str().substr(0, ss.str().size() - strlen(prev))); + ss.str(std::string("")); + ss << chr; + } + if (base::utils::Str::endsWith(ss.str(), std::string(sfx))) { + std::string chr(ss.str().substr(0, ss.str().size() - strlen(sfx))); + ss.str(std::string("")); + ss << chr; + } + ss << sfx; + }; + auto insert = [&](std::stringstream& ss, base::type::VerboseLevel level) { + if (!base::utils::hasFlag(LoggingFlag::DisableVModulesExtensions, *m_pFlags)) { + addSuffix(ss, ".h", nullptr); + m_modules.insert(std::make_pair(ss.str(), level)); + addSuffix(ss, ".c", ".h"); + m_modules.insert(std::make_pair(ss.str(), level)); + addSuffix(ss, ".cpp", ".c"); + m_modules.insert(std::make_pair(ss.str(), level)); + addSuffix(ss, ".cc", ".cpp"); + m_modules.insert(std::make_pair(ss.str(), level)); + addSuffix(ss, ".cxx", ".cc"); + m_modules.insert(std::make_pair(ss.str(), level)); + addSuffix(ss, ".-inl.h", ".cxx"); + m_modules.insert(std::make_pair(ss.str(), level)); + addSuffix(ss, ".hxx", ".-inl.h"); + m_modules.insert(std::make_pair(ss.str(), level)); + addSuffix(ss, ".hpp", ".hxx"); + m_modules.insert(std::make_pair(ss.str(), level)); + addSuffix(ss, ".hh", ".hpp"); + } + m_modules.insert(std::make_pair(ss.str(), level)); + }; + bool isMod = true; + bool isLevel = false; + std::stringstream ss; + int level = -1; + for (; *modules; ++modules) { + switch (*modules) { + case '=': + isLevel = true; + isMod = false; + break; + case ',': + isLevel = false; + isMod = true; + if (!ss.str().empty() && level != -1) { + insert(ss, static_cast(level)); + ss.str(std::string("")); + level = -1; + } + break; + default: + if (isMod) { + ss << *modules; + } else if (isLevel) { + if (isdigit(*modules)) { + level = static_cast(*modules) - 48; + } + } + break; + } + } + if (!ss.str().empty() && level != -1) { + insert(ss, static_cast(level)); + } +} + +bool +VRegistry::allowed(base::type::VerboseLevel vlevel, const char* file) { + base::threading::ScopedLock scopedLock(lock()); + if (m_modules.empty() || file == nullptr) { + return vlevel <= m_level; + } else { + char baseFilename[base::consts::kSourceFilenameMaxLength] = ""; + base::utils::File::buildBaseFilename(file, baseFilename); + std::unordered_map::iterator it = m_modules.begin(); + for (; it != m_modules.end(); ++it) { + if (base::utils::Str::wildCardMatch(baseFilename, it->first.c_str())) { + return vlevel <= it->second; + } + } + if (base::utils::hasFlag(LoggingFlag::AllowVerboseIfModuleNotSpecified, *m_pFlags)) { + return true; + } + return false; + } +} + +void +VRegistry::setFromArgs(const base::utils::CommandLineArgs* commandLineArgs) { + if (commandLineArgs->hasParam("-v") || commandLineArgs->hasParam("--verbose") || commandLineArgs->hasParam("-V") || + commandLineArgs->hasParam("--VERBOSE")) { + setLevel(base::consts::kMaxVerboseLevel); + } else if (commandLineArgs->hasParamWithValue("--v")) { + setLevel(static_cast(atoi(commandLineArgs->getParamValue("--v")))); + } else if (commandLineArgs->hasParamWithValue("--V")) { + setLevel(static_cast(atoi(commandLineArgs->getParamValue("--V")))); + } else if ((commandLineArgs->hasParamWithValue("-vmodule")) && vModulesEnabled()) { + setModules(commandLineArgs->getParamValue("-vmodule")); + } else if (commandLineArgs->hasParamWithValue("-VMODULE") && vModulesEnabled()) { + setModules(commandLineArgs->getParamValue("-VMODULE")); + } +} + +#if !defined(ELPP_DEFAULT_LOGGING_FLAGS) +#define ELPP_DEFAULT_LOGGING_FLAGS 0x0 +#endif // !defined(ELPP_DEFAULT_LOGGING_FLAGS) +// Storage +#if ELPP_ASYNC_LOGGING +Storage::Storage(const LogBuilderPtr& defaultLogBuilder, base::IWorker* asyncDispatchWorker) + : +#else +Storage::Storage(const LogBuilderPtr& defaultLogBuilder) + : +#endif // ELPP_ASYNC_LOGGING + m_registeredHitCounters(new base::RegisteredHitCounters()), + m_registeredLoggers(new base::RegisteredLoggers(defaultLogBuilder)), + m_flags(ELPP_DEFAULT_LOGGING_FLAGS), + m_vRegistry(new base::VRegistry(0, &m_flags)), + +#if ELPP_ASYNC_LOGGING + m_asyncLogQueue(new base::AsyncLogQueue()), + m_asyncDispatchWorker(asyncDispatchWorker), +#endif // ELPP_ASYNC_LOGGING + + m_preRollOutCallback(base::defaultPreRollOutCallback) { + // Register default logger + m_registeredLoggers->get(std::string(base::consts::kDefaultLoggerId)); + // We register default logger anyway (worse case it's not going to register) just in case + m_registeredLoggers->get("default"); + +#if defined(ELPP_FEATURE_ALL) || defined(ELPP_FEATURE_PERFORMANCE_TRACKING) + // Register performance logger and reconfigure format + Logger* performanceLogger = m_registeredLoggers->get(std::string(base::consts::kPerformanceLoggerId)); + m_registeredLoggers->get("performance"); + performanceLogger->configurations()->setGlobally(ConfigurationType::Format, std::string("%datetime %level %msg")); + performanceLogger->reconfigure(); +#endif // defined(ELPP_FEATURE_ALL) || defined(ELPP_FEATURE_PERFORMANCE_TRACKING) + +#if defined(ELPP_SYSLOG) + // Register syslog logger and reconfigure format + Logger* sysLogLogger = m_registeredLoggers->get(std::string(base::consts::kSysLogLoggerId)); + sysLogLogger->configurations()->setGlobally(ConfigurationType::Format, std::string("%level: %msg")); + sysLogLogger->reconfigure(); +#endif // defined(ELPP_SYSLOG) + addFlag(LoggingFlag::AllowVerboseIfModuleNotSpecified); +#if ELPP_ASYNC_LOGGING + installLogDispatchCallback(std::string("AsyncLogDispatchCallback")); +#else + installLogDispatchCallback(std::string("DefaultLogDispatchCallback")); +#endif // ELPP_ASYNC_LOGGING +#if defined(ELPP_FEATURE_ALL) || defined(ELPP_FEATURE_PERFORMANCE_TRACKING) + installPerformanceTrackingCallback( + std::string("DefaultPerformanceTrackingCallback")); +#endif // defined(ELPP_FEATURE_ALL) || defined(ELPP_FEATURE_PERFORMANCE_TRACKING) + ELPP_INTERNAL_INFO(1, "Easylogging++ has been initialized"); +#if ELPP_ASYNC_LOGGING + m_asyncDispatchWorker->start(); +#endif // ELPP_ASYNC_LOGGING +} + +Storage::~Storage(void) { + ELPP_INTERNAL_INFO(4, "Destroying storage"); +#if ELPP_ASYNC_LOGGING + ELPP_INTERNAL_INFO(5, "Replacing log dispatch callback to synchronous"); + uninstallLogDispatchCallback(std::string("AsyncLogDispatchCallback")); + installLogDispatchCallback(std::string("DefaultLogDispatchCallback")); + ELPP_INTERNAL_INFO(5, "Destroying asyncDispatchWorker"); + base::utils::safeDelete(m_asyncDispatchWorker); + ELPP_INTERNAL_INFO(5, "Destroying asyncLogQueue"); + base::utils::safeDelete(m_asyncLogQueue); +#endif // ELPP_ASYNC_LOGGING + ELPP_INTERNAL_INFO(5, "Destroying registeredHitCounters"); + base::utils::safeDelete(m_registeredHitCounters); + ELPP_INTERNAL_INFO(5, "Destroying registeredLoggers"); + base::utils::safeDelete(m_registeredLoggers); + ELPP_INTERNAL_INFO(5, "Destroying vRegistry"); + base::utils::safeDelete(m_vRegistry); +} + +bool +Storage::hasCustomFormatSpecifier(const char* formatSpecifier) { + base::threading::ScopedLock scopedLock(customFormatSpecifiersLock()); + return std::find(m_customFormatSpecifiers.begin(), m_customFormatSpecifiers.end(), formatSpecifier) != + m_customFormatSpecifiers.end(); +} + +void +Storage::installCustomFormatSpecifier(const CustomFormatSpecifier& customFormatSpecifier) { + if (hasCustomFormatSpecifier(customFormatSpecifier.formatSpecifier())) { + return; + } + base::threading::ScopedLock scopedLock(customFormatSpecifiersLock()); + m_customFormatSpecifiers.push_back(customFormatSpecifier); +} + +bool +Storage::uninstallCustomFormatSpecifier(const char* formatSpecifier) { + base::threading::ScopedLock scopedLock(customFormatSpecifiersLock()); + std::vector::iterator it = + std::find(m_customFormatSpecifiers.begin(), m_customFormatSpecifiers.end(), formatSpecifier); + if (it != m_customFormatSpecifiers.end() && strcmp(formatSpecifier, it->formatSpecifier()) == 0) { + m_customFormatSpecifiers.erase(it); + return true; + } + return false; +} + +void +Storage::setApplicationArguments(int argc, char** argv) { + m_commandLineArgs.setArgs(argc, argv); + m_vRegistry->setFromArgs(commandLineArgs()); + // default log file +#if !defined(ELPP_DISABLE_LOG_FILE_FROM_ARG) + if (m_commandLineArgs.hasParamWithValue(base::consts::kDefaultLogFileParam)) { + Configurations c; + c.setGlobally(ConfigurationType::Filename, + std::string(m_commandLineArgs.getParamValue(base::consts::kDefaultLogFileParam))); + registeredLoggers()->setDefaultConfigurations(c); + for (base::RegisteredLoggers::iterator it = registeredLoggers()->begin(); it != registeredLoggers()->end(); + ++it) { + it->second->configure(c); + } + } +#endif // !defined(ELPP_DISABLE_LOG_FILE_FROM_ARG) +#if defined(ELPP_LOGGING_FLAGS_FROM_ARG) + if (m_commandLineArgs.hasParamWithValue(base::consts::kLoggingFlagsParam)) { + int userInput = atoi(m_commandLineArgs.getParamValue(base::consts::kLoggingFlagsParam)); + if (ELPP_DEFAULT_LOGGING_FLAGS == 0x0) { + m_flags = userInput; + } else { + base::utils::addFlag(userInput, &m_flags); + } + } +#endif // defined(ELPP_LOGGING_FLAGS_FROM_ARG) +} + +} // namespace base + +// LogDispatchCallback +void +LogDispatchCallback::handle(const LogDispatchData* data) { +#if defined(ELPP_THREAD_SAFE) + base::threading::ScopedLock scopedLock(m_fileLocksMapLock); + std::string filename = data->logMessage()->logger()->typedConfigurations()->filename(data->logMessage()->level()); + auto lock = m_fileLocks.find(filename); + if (lock == m_fileLocks.end()) { + m_fileLocks.emplace( + std::make_pair(filename, std::unique_ptr(new base::threading::Mutex))); + } +#endif +} + +base::threading::Mutex& +LogDispatchCallback::fileHandle(const LogDispatchData* data) { + auto it = + m_fileLocks.find(data->logMessage()->logger()->typedConfigurations()->filename(data->logMessage()->level())); + return *(it->second.get()); +} + +namespace base { +// DefaultLogDispatchCallback + +void +DefaultLogDispatchCallback::handle(const LogDispatchData* data) { +#if defined(ELPP_THREAD_SAFE) + LogDispatchCallback::handle(data); + base::threading::ScopedLock scopedLock(fileHandle(data)); +#endif + m_data = data; + dispatch(m_data->logMessage()->logger()->logBuilder()->build( + m_data->logMessage(), m_data->dispatchAction() == base::DispatchAction::NormalLog)); +} + +void +DefaultLogDispatchCallback::dispatch(base::type::string_t&& logLine) { + if (m_data->dispatchAction() == base::DispatchAction::NormalLog) { + if (m_data->logMessage()->logger()->m_typedConfigurations->toFile(m_data->logMessage()->level())) { + base::type::fstream_t* fs = + m_data->logMessage()->logger()->m_typedConfigurations->fileStream(m_data->logMessage()->level()); + if (fs != nullptr) { + fs->write(logLine.c_str(), logLine.size()); + if (fs->fail()) { + ELPP_INTERNAL_ERROR("Unable to write log to file [" + << m_data->logMessage()->logger()->m_typedConfigurations->filename( + m_data->logMessage()->level()) + << "].\n" + << "Few possible reasons (could be something else):\n" + << " * Permission denied\n" + << " * Disk full\n" + << " * Disk is not writable", + true); + } else { + if (ELPP->hasFlag(LoggingFlag::ImmediateFlush) || + (m_data->logMessage()->logger()->isFlushNeeded(m_data->logMessage()->level()))) { + m_data->logMessage()->logger()->flush(m_data->logMessage()->level(), fs); + } + } + } else { + ELPP_INTERNAL_ERROR("Log file for [" + << LevelHelper::convertToString(m_data->logMessage()->level()) << "] " + << "has not been configured but [TO_FILE] is configured to TRUE. [Logger ID: " + << m_data->logMessage()->logger()->id() << "]", + false); + } + } + if (m_data->logMessage()->logger()->m_typedConfigurations->toStandardOutput(m_data->logMessage()->level())) { + if (ELPP->hasFlag(LoggingFlag::ColoredTerminalOutput)) + m_data->logMessage()->logger()->logBuilder()->convertToColoredOutput(&logLine, + m_data->logMessage()->level()); + ELPP_COUT << ELPP_COUT_LINE(logLine); + } + } +#if defined(ELPP_SYSLOG) + else if (m_data->dispatchAction() == base::DispatchAction::SysLog) { + // Determine syslog priority + int sysLogPriority = 0; + if (m_data->logMessage()->level() == Level::Fatal) + sysLogPriority = LOG_EMERG; + else if (m_data->logMessage()->level() == Level::Error) + sysLogPriority = LOG_ERR; + else if (m_data->logMessage()->level() == Level::Warning) + sysLogPriority = LOG_WARNING; + else if (m_data->logMessage()->level() == Level::Info) + sysLogPriority = LOG_INFO; + else if (m_data->logMessage()->level() == Level::Debug) + sysLogPriority = LOG_DEBUG; + else + sysLogPriority = LOG_NOTICE; +#if defined(ELPP_UNICODE) + char* line = base::utils::Str::wcharPtrToCharPtr(logLine.c_str()); + syslog(sysLogPriority, "%s", line); + free(line); +#else + syslog(sysLogPriority, "%s", logLine.c_str()); +#endif + } +#endif // defined(ELPP_SYSLOG) +} + +#if ELPP_ASYNC_LOGGING + +// AsyncLogDispatchCallback + +void +AsyncLogDispatchCallback::handle(const LogDispatchData* data) { + base::type::string_t logLine = data->logMessage()->logger()->logBuilder()->build( + data->logMessage(), data->dispatchAction() == base::DispatchAction::NormalLog); + if (data->dispatchAction() == base::DispatchAction::NormalLog && + data->logMessage()->logger()->typedConfigurations()->toStandardOutput(data->logMessage()->level())) { + if (ELPP->hasFlag(LoggingFlag::ColoredTerminalOutput)) + data->logMessage()->logger()->logBuilder()->convertToColoredOutput(&logLine, data->logMessage()->level()); + ELPP_COUT << ELPP_COUT_LINE(logLine); + } + // Save resources and only queue if we want to write to file otherwise just ignore handler + if (data->logMessage()->logger()->typedConfigurations()->toFile(data->logMessage()->level())) { + ELPP->asyncLogQueue()->push(AsyncLogItem(*(data->logMessage()), *data, logLine)); + } +} + +// AsyncDispatchWorker +AsyncDispatchWorker::AsyncDispatchWorker() { + setContinueRunning(false); +} + +AsyncDispatchWorker::~AsyncDispatchWorker() { + setContinueRunning(false); + ELPP_INTERNAL_INFO(6, "Stopping dispatch worker - Cleaning log queue"); + clean(); + ELPP_INTERNAL_INFO(6, "Log queue cleaned"); +} + +bool +AsyncDispatchWorker::clean(void) { + std::mutex m; + std::unique_lock lk(m); + cv.wait(lk, [] { return !ELPP->asyncLogQueue()->empty(); }); + emptyQueue(); + lk.unlock(); + cv.notify_one(); + return ELPP->asyncLogQueue()->empty(); +} + +void +AsyncDispatchWorker::emptyQueue(void) { + while (!ELPP->asyncLogQueue()->empty()) { + AsyncLogItem data = ELPP->asyncLogQueue()->next(); + handle(&data); + base::threading::msleep(100); + } +} + +void +AsyncDispatchWorker::start(void) { + base::threading::msleep(5000); // 5s (why?) + setContinueRunning(true); + std::thread t1(&AsyncDispatchWorker::run, this); + t1.join(); +} + +void +AsyncDispatchWorker::handle(AsyncLogItem* logItem) { + LogDispatchData* data = logItem->data(); + LogMessage* logMessage = logItem->logMessage(); + Logger* logger = logMessage->logger(); + base::TypedConfigurations* conf = logger->typedConfigurations(); + base::type::string_t logLine = logItem->logLine(); + if (data->dispatchAction() == base::DispatchAction::NormalLog) { + if (conf->toFile(logMessage->level())) { + base::type::fstream_t* fs = conf->fileStream(logMessage->level()); + if (fs != nullptr) { + fs->write(logLine.c_str(), logLine.size()); + if (fs->fail()) { + ELPP_INTERNAL_ERROR("Unable to write log to file [" + << conf->filename(logMessage->level()) << "].\n" + << "Few possible reasons (could be something else):\n" + << " * Permission denied\n" + << " * Disk full\n" + << " * Disk is not writable", + true); + } else { + if (ELPP->hasFlag(LoggingFlag::ImmediateFlush) || (logger->isFlushNeeded(logMessage->level()))) { + logger->flush(logMessage->level(), fs); + } + } + } else { + ELPP_INTERNAL_ERROR("Log file for [" + << LevelHelper::convertToString(logMessage->level()) << "] " + << "has not been configured but [TO_FILE] is configured to TRUE. [Logger ID: " + << logger->id() << "]", + false); + } + } + } +#if defined(ELPP_SYSLOG) + else if (data->dispatchAction() == base::DispatchAction::SysLog) { + // Determine syslog priority + int sysLogPriority = 0; + if (logMessage->level() == Level::Fatal) + sysLogPriority = LOG_EMERG; + else if (logMessage->level() == Level::Error) + sysLogPriority = LOG_ERR; + else if (logMessage->level() == Level::Warning) + sysLogPriority = LOG_WARNING; + else if (logMessage->level() == Level::Info) + sysLogPriority = LOG_INFO; + else if (logMessage->level() == Level::Debug) + sysLogPriority = LOG_DEBUG; + else + sysLogPriority = LOG_NOTICE; +#if defined(ELPP_UNICODE) + char* line = base::utils::Str::wcharPtrToCharPtr(logLine.c_str()); + syslog(sysLogPriority, "%s", line); + free(line); +#else + syslog(sysLogPriority, "%s", logLine.c_str()); +#endif + } +#endif // defined(ELPP_SYSLOG) +} + +void +AsyncDispatchWorker::run(void) { + while (continueRunning()) { + emptyQueue(); + base::threading::msleep(10); // 10ms + } +} +#endif // ELPP_ASYNC_LOGGING + +// DefaultLogBuilder + +base::type::string_t +DefaultLogBuilder::build(const LogMessage* logMessage, bool appendNewLine) const { + base::TypedConfigurations* tc = logMessage->logger()->typedConfigurations(); + const base::LogFormat* logFormat = &tc->logFormat(logMessage->level()); + base::type::string_t logLine = logFormat->format(); + char buff[base::consts::kSourceFilenameMaxLength + base::consts::kSourceLineMaxLength] = ""; + const char* bufLim = buff + sizeof(buff); + if (logFormat->hasFlag(base::FormatFlags::AppName)) { + // App name + base::utils::Str::replaceFirstWithEscape(logLine, base::consts::kAppNameFormatSpecifier, + logMessage->logger()->parentApplicationName()); + } + if (logFormat->hasFlag(base::FormatFlags::ThreadId)) { + // Thread ID + base::utils::Str::replaceFirstWithEscape(logLine, base::consts::kThreadIdFormatSpecifier, + ELPP->getThreadName(base::threading::getCurrentThreadId())); + } + if (logFormat->hasFlag(base::FormatFlags::DateTime)) { + // DateTime + base::utils::Str::replaceFirstWithEscape( + logLine, base::consts::kDateTimeFormatSpecifier, + base::utils::DateTime::getDateTime(logFormat->dateTimeFormat().c_str(), + &tc->subsecondPrecision(logMessage->level()))); + } + if (logFormat->hasFlag(base::FormatFlags::Function)) { + // Function + base::utils::Str::replaceFirstWithEscape(logLine, base::consts::kLogFunctionFormatSpecifier, + logMessage->func()); + } + if (logFormat->hasFlag(base::FormatFlags::File)) { + // File + base::utils::Str::clearBuff(buff, base::consts::kSourceFilenameMaxLength); + base::utils::File::buildStrippedFilename(logMessage->file().c_str(), buff); + base::utils::Str::replaceFirstWithEscape(logLine, base::consts::kLogFileFormatSpecifier, std::string(buff)); + } + if (logFormat->hasFlag(base::FormatFlags::FileBase)) { + // FileBase + base::utils::Str::clearBuff(buff, base::consts::kSourceFilenameMaxLength); + base::utils::File::buildBaseFilename(logMessage->file(), buff); + base::utils::Str::replaceFirstWithEscape(logLine, base::consts::kLogFileBaseFormatSpecifier, std::string(buff)); + } + if (logFormat->hasFlag(base::FormatFlags::Line)) { + // Line + char* buf = base::utils::Str::clearBuff(buff, base::consts::kSourceLineMaxLength); + buf = base::utils::Str::convertAndAddToBuff(logMessage->line(), base::consts::kSourceLineMaxLength, buf, bufLim, + false); + base::utils::Str::replaceFirstWithEscape(logLine, base::consts::kLogLineFormatSpecifier, std::string(buff)); + } + if (logFormat->hasFlag(base::FormatFlags::Location)) { + // Location + char* buf = base::utils::Str::clearBuff( + buff, base::consts::kSourceFilenameMaxLength + base::consts::kSourceLineMaxLength); + base::utils::File::buildStrippedFilename(logMessage->file().c_str(), buff); + buf = base::utils::Str::addToBuff(buff, buf, bufLim); + buf = base::utils::Str::addToBuff(":", buf, bufLim); + buf = base::utils::Str::convertAndAddToBuff(logMessage->line(), base::consts::kSourceLineMaxLength, buf, bufLim, + false); + base::utils::Str::replaceFirstWithEscape(logLine, base::consts::kLogLocationFormatSpecifier, std::string(buff)); + } + if (logMessage->level() == Level::Verbose && logFormat->hasFlag(base::FormatFlags::VerboseLevel)) { + // Verbose level + char* buf = base::utils::Str::clearBuff(buff, 1); + buf = base::utils::Str::convertAndAddToBuff(logMessage->verboseLevel(), 1, buf, bufLim, false); + base::utils::Str::replaceFirstWithEscape(logLine, base::consts::kVerboseLevelFormatSpecifier, + std::string(buff)); + } + if (logFormat->hasFlag(base::FormatFlags::LogMessage)) { + // Log message + base::utils::Str::replaceFirstWithEscape(logLine, base::consts::kMessageFormatSpecifier, logMessage->message()); + } +#if !defined(ELPP_DISABLE_CUSTOM_FORMAT_SPECIFIERS) + el::base::threading::ScopedLock lock_(ELPP->customFormatSpecifiersLock()); + ELPP_UNUSED(lock_); + for (std::vector::const_iterator it = ELPP->customFormatSpecifiers()->begin(); + it != ELPP->customFormatSpecifiers()->end(); ++it) { + std::string fs(it->formatSpecifier()); + base::type::string_t wcsFormatSpecifier(fs.begin(), fs.end()); + base::utils::Str::replaceFirstWithEscape(logLine, wcsFormatSpecifier, it->resolver()(logMessage)); + } +#endif // !defined(ELPP_DISABLE_CUSTOM_FORMAT_SPECIFIERS) + if (appendNewLine) + logLine += ELPP_LITERAL("\n"); + return logLine; +} + +// LogDispatcher + +void +LogDispatcher::dispatch(void) { + if (m_proceed && m_dispatchAction == base::DispatchAction::None) { + m_proceed = false; + } + if (!m_proceed) { + return; + } +#ifndef ELPP_NO_GLOBAL_LOCK + // see https://github.com/muflihun/easyloggingpp/issues/580 + // global lock is turned off by default unless + // ELPP_NO_GLOBAL_LOCK is defined + base::threading::ScopedLock scopedLock(ELPP->lock()); +#endif + base::TypedConfigurations* tc = m_logMessage->logger()->m_typedConfigurations; + if (ELPP->hasFlag(LoggingFlag::StrictLogFileSizeCheck)) { + tc->validateFileRolling(m_logMessage->level(), ELPP->preRollOutCallback()); + } + LogDispatchCallback* callback = nullptr; + LogDispatchData data; + for (const std::pair& h : ELPP->m_logDispatchCallbacks) { + callback = h.second.get(); + if (callback != nullptr && callback->enabled()) { + data.setLogMessage(m_logMessage); + data.setDispatchAction(m_dispatchAction); + callback->handle(&data); + } + } +} + +// MessageBuilder + +void +MessageBuilder::initialize(Logger* logger) { + m_logger = logger; + m_containerLogSeperator = + ELPP->hasFlag(LoggingFlag::NewLineForContainer) ? ELPP_LITERAL("\n ") : ELPP_LITERAL(", "); +} + +MessageBuilder& +MessageBuilder::operator<<(const wchar_t* msg) { + if (msg == nullptr) { + m_logger->stream() << base::consts::kNullPointer; + return *this; + } +#if defined(ELPP_UNICODE) + m_logger->stream() << msg; +#else + char* buff_ = base::utils::Str::wcharPtrToCharPtr(msg); + m_logger->stream() << buff_; + free(buff_); +#endif + if (ELPP->hasFlag(LoggingFlag::AutoSpacing)) { + m_logger->stream() << " "; + } + return *this; +} + +// Writer + +Writer& +Writer::construct(Logger* logger, bool needLock) { + m_logger = logger; + initializeLogger(logger->id(), false, needLock); + m_messageBuilder.initialize(m_logger); + return *this; +} + +Writer& +Writer::construct(int count, const char* loggerIds, ...) { + if (ELPP->hasFlag(LoggingFlag::MultiLoggerSupport)) { + va_list loggersList; + va_start(loggersList, loggerIds); + const char* id = loggerIds; + m_loggerIds.reserve(count); + for (int i = 0; i < count; ++i) { + m_loggerIds.push_back(std::string(id)); + id = va_arg(loggersList, const char*); + } + va_end(loggersList); + initializeLogger(m_loggerIds.at(0)); + } else { + initializeLogger(std::string(loggerIds)); + } + m_messageBuilder.initialize(m_logger); + return *this; +} + +void +Writer::initializeLogger(const std::string& loggerId, bool lookup, bool needLock) { + if (lookup) { + m_logger = ELPP->registeredLoggers()->get(loggerId, ELPP->hasFlag(LoggingFlag::CreateLoggerAutomatically)); + } + if (m_logger == nullptr) { + { + if (!ELPP->registeredLoggers()->has(std::string(base::consts::kDefaultLoggerId))) { + // Somehow default logger has been unregistered. Not good! Register again + ELPP->registeredLoggers()->get(std::string(base::consts::kDefaultLoggerId)); + } + } + Writer(Level::Debug, m_file, m_line, m_func).construct(1, base::consts::kDefaultLoggerId) + << "Logger [" << loggerId << "] is not registered yet!"; + m_proceed = false; + } else { + if (needLock) { + m_logger->acquireLock(); // This should not be unlocked by checking m_proceed because + // m_proceed can be changed by lines below + } + if (ELPP->hasFlag(LoggingFlag::HierarchicalLogging)) { + m_proceed = m_level == Level::Verbose + ? m_logger->enabled(m_level) + : LevelHelper::castToInt(m_level) >= LevelHelper::castToInt(ELPP->m_loggingLevel); + } else { + m_proceed = m_logger->enabled(m_level); + } + } +} + +void +Writer::processDispatch() { +#if ELPP_LOGGING_ENABLED + if (ELPP->hasFlag(LoggingFlag::MultiLoggerSupport)) { + bool firstDispatched = false; + base::type::string_t logMessage; + std::size_t i = 0; + do { + if (m_proceed) { + if (firstDispatched) { + m_logger->stream() << logMessage; + } else { + firstDispatched = true; + if (m_loggerIds.size() > 1) { + logMessage = m_logger->stream().str(); + } + } + triggerDispatch(); + } else if (m_logger != nullptr) { + m_logger->stream().str(ELPP_LITERAL("")); + m_logger->releaseLock(); + } + if (i + 1 < m_loggerIds.size()) { + initializeLogger(m_loggerIds.at(i + 1)); + } + } while (++i < m_loggerIds.size()); + } else { + if (m_proceed) { + triggerDispatch(); + } else if (m_logger != nullptr) { + m_logger->stream().str(ELPP_LITERAL("")); + m_logger->releaseLock(); + } + } +#else + if (m_logger != nullptr) { + m_logger->stream().str(ELPP_LITERAL("")); + m_logger->releaseLock(); + } +#endif // ELPP_LOGGING_ENABLED +} + +void +Writer::triggerDispatch(void) { + if (m_proceed) { + if (m_msg == nullptr) { + LogMessage msg(m_level, m_file, m_line, m_func, m_verboseLevel, m_logger); + base::LogDispatcher(m_proceed, &msg, m_dispatchAction).dispatch(); + } else { + base::LogDispatcher(m_proceed, m_msg, m_dispatchAction).dispatch(); + } + } + if (m_logger != nullptr) { + m_logger->stream().str(ELPP_LITERAL("")); + m_logger->releaseLock(); + } + if (m_proceed && m_level == Level::Fatal && !ELPP->hasFlag(LoggingFlag::DisableApplicationAbortOnFatalLog)) { + base::Writer(Level::Warning, m_file, m_line, m_func).construct(1, base::consts::kDefaultLoggerId) + << "Aborting application. Reason: Fatal log at [" << m_file << ":" << m_line << "]"; + std::stringstream reasonStream; + reasonStream << "Fatal log at [" << m_file << ":" << m_line << "]" + << " If you wish to disable 'abort on fatal log' please use " + << "el::Loggers::addFlag(el::LoggingFlag::DisableApplicationAbortOnFatalLog)"; + base::utils::abort(1, reasonStream.str()); + } + m_proceed = false; +} + +// PErrorWriter + +PErrorWriter::~PErrorWriter(void) { + if (m_proceed) { +#if ELPP_COMPILER_MSVC + char buff[256]; + strerror_s(buff, 256, errno); + m_logger->stream() << ": " << buff << " [" << errno << "]"; +#else + m_logger->stream() << ": " << strerror(errno) << " [" << errno << "]"; +#endif + } +} + +// PerformanceTracker + +#if defined(ELPP_FEATURE_ALL) || defined(ELPP_FEATURE_PERFORMANCE_TRACKING) + +PerformanceTracker::PerformanceTracker(const std::string& blockName, base::TimestampUnit timestampUnit, + const std::string& loggerId, bool scopedLog, Level level) + : m_blockName(blockName), + m_timestampUnit(timestampUnit), + m_loggerId(loggerId), + m_scopedLog(scopedLog), + m_level(level), + m_hasChecked(false), + m_lastCheckpointId(std::string()), + m_enabled(false) { +#if !defined(ELPP_DISABLE_PERFORMANCE_TRACKING) && ELPP_LOGGING_ENABLED + // We store it locally so that if user happen to change configuration by the end of scope + // or before calling checkpoint, we still depend on state of configuraton at time of construction + el::Logger* loggerPtr = ELPP->registeredLoggers()->get(loggerId, false); + m_enabled = loggerPtr != nullptr && loggerPtr->m_typedConfigurations->performanceTracking(m_level); + if (m_enabled) { + base::utils::DateTime::gettimeofday(&m_startTime); + } +#endif // !defined(ELPP_DISABLE_PERFORMANCE_TRACKING) && ELPP_LOGGING_ENABLED +} + +PerformanceTracker::~PerformanceTracker(void) { +#if !defined(ELPP_DISABLE_PERFORMANCE_TRACKING) && ELPP_LOGGING_ENABLED + if (m_enabled) { + base::threading::ScopedLock scopedLock(lock()); + if (m_scopedLog) { + base::utils::DateTime::gettimeofday(&m_endTime); + base::type::string_t formattedTime = getFormattedTimeTaken(); + PerformanceTrackingData data(PerformanceTrackingData::DataType::Complete); + data.init(this); + data.m_formattedTimeTaken = formattedTime; + PerformanceTrackingCallback* callback = nullptr; + for (const std::pair& h : + ELPP->m_performanceTrackingCallbacks) { + callback = h.second.get(); + if (callback != nullptr && callback->enabled()) { + callback->handle(&data); + } + } + } + } +#endif // !defined(ELPP_DISABLE_PERFORMANCE_TRACKING) +} + +void +PerformanceTracker::checkpoint(const std::string& id, const char* file, base::type::LineNumber line, const char* func) { +#if !defined(ELPP_DISABLE_PERFORMANCE_TRACKING) && ELPP_LOGGING_ENABLED + if (m_enabled) { + base::threading::ScopedLock scopedLock(lock()); + base::utils::DateTime::gettimeofday(&m_endTime); + base::type::string_t formattedTime = + m_hasChecked ? getFormattedTimeTaken(m_lastCheckpointTime) : ELPP_LITERAL(""); + PerformanceTrackingData data(PerformanceTrackingData::DataType::Checkpoint); + data.init(this); + data.m_checkpointId = id; + data.m_file = file; + data.m_line = line; + data.m_func = func; + data.m_formattedTimeTaken = formattedTime; + PerformanceTrackingCallback* callback = nullptr; + for (const std::pair& h : + ELPP->m_performanceTrackingCallbacks) { + callback = h.second.get(); + if (callback != nullptr && callback->enabled()) { + callback->handle(&data); + } + } + base::utils::DateTime::gettimeofday(&m_lastCheckpointTime); + m_hasChecked = true; + m_lastCheckpointId = id; + } +#endif // !defined(ELPP_DISABLE_PERFORMANCE_TRACKING) && ELPP_LOGGING_ENABLED + ELPP_UNUSED(id); + ELPP_UNUSED(file); + ELPP_UNUSED(line); + ELPP_UNUSED(func); +} + +const base::type::string_t +PerformanceTracker::getFormattedTimeTaken(struct timeval startTime) const { + if (ELPP->hasFlag(LoggingFlag::FixedTimeFormat)) { + base::type::stringstream_t ss; + ss << base::utils::DateTime::getTimeDifference(m_endTime, startTime, m_timestampUnit) << " " + << base::consts::kTimeFormats[static_cast(m_timestampUnit)].unit; + return ss.str(); + } + return base::utils::DateTime::formatTime( + base::utils::DateTime::getTimeDifference(m_endTime, startTime, m_timestampUnit), m_timestampUnit); +} + +#endif // defined(ELPP_FEATURE_ALL) || defined(ELPP_FEATURE_PERFORMANCE_TRACKING) + +namespace debug { +#if defined(ELPP_FEATURE_ALL) || defined(ELPP_FEATURE_CRASH_LOG) + +// StackTrace + +StackTrace::StackTraceEntry::StackTraceEntry(std::size_t index, const std::string& loc, const std::string& demang, + const std::string& hex, const std::string& addr) + : m_index(index), m_location(loc), m_demangled(demang), m_hex(hex), m_addr(addr) { +} + +std::ostream& +operator<<(std::ostream& ss, const StackTrace::StackTraceEntry& si) { + ss << "[" << si.m_index << "] " << si.m_location << (si.m_hex.empty() ? "" : "+") << si.m_hex << " " << si.m_addr + << (si.m_demangled.empty() ? "" : ":") << si.m_demangled; + return ss; +} + +std::ostream& +operator<<(std::ostream& os, const StackTrace& st) { + std::vector::const_iterator it = st.m_stack.begin(); + while (it != st.m_stack.end()) { + os << " " << *it++ << "\n"; + } + return os; +} + +void +StackTrace::generateNew(void) { +#if ELPP_STACKTRACE + m_stack.clear(); + void* stack[kMaxStack]; + unsigned int size = backtrace(stack, kMaxStack); + char** strings = backtrace_symbols(stack, size); + if (size > kStackStart) { // Skip StackTrace c'tor and generateNew + for (std::size_t i = kStackStart; i < size; ++i) { + std::string mangName; + std::string location; + std::string hex; + std::string addr; + + // entry: 2 crash.cpp.bin 0x0000000101552be5 _ZN2el4base5debug10StackTraceC1Ev + 21 + const std::string line(strings[i]); + auto p = line.find("_"); + if (p != std::string::npos) { + mangName = line.substr(p); + mangName = mangName.substr(0, mangName.find(" +")); + } + p = line.find("0x"); + if (p != std::string::npos) { + addr = line.substr(p); + addr = addr.substr(0, addr.find("_")); + } + // Perform demangling if parsed properly + if (!mangName.empty()) { + int status = 0; + char* demangName = abi::__cxa_demangle(mangName.data(), 0, 0, &status); + // if demangling is successful, output the demangled function name + if (status == 0) { + // Success (see http://gcc.gnu.org/onlinedocs/libstdc++/libstdc++-html-USERS-4.3/a01696.html) + StackTraceEntry entry(i - 1, location, demangName, hex, addr); + m_stack.push_back(entry); + } else { + // Not successful - we will use mangled name + StackTraceEntry entry(i - 1, location, mangName, hex, addr); + m_stack.push_back(entry); + } + free(demangName); + } else { + StackTraceEntry entry(i - 1, line); + m_stack.push_back(entry); + } + } + } + free(strings); +#else + ELPP_INTERNAL_INFO(1, "Stacktrace generation not supported for selected compiler"); +#endif // ELPP_STACKTRACE +} + +// Static helper functions + +static std::string +crashReason(int sig) { + std::stringstream ss; + bool foundReason = false; + for (int i = 0; i < base::consts::kCrashSignalsCount; ++i) { + if (base::consts::kCrashSignals[i].numb == sig) { + ss << "Application has crashed due to [" << base::consts::kCrashSignals[i].name << "] signal"; + if (ELPP->hasFlag(el::LoggingFlag::LogDetailedCrashReason)) { + ss << std::endl + << " " << base::consts::kCrashSignals[i].brief << std::endl + << " " << base::consts::kCrashSignals[i].detail; + } + foundReason = true; + } + } + if (!foundReason) { + ss << "Application has crashed due to unknown signal [" << sig << "]"; + } + return ss.str(); +} +/// @brief Logs reason of crash from sig +static void +logCrashReason(int sig, bool stackTraceIfAvailable, Level level, const char* logger) { + if (sig == SIGINT && ELPP->hasFlag(el::LoggingFlag::IgnoreSigInt)) { + return; + } + std::stringstream ss; + ss << "CRASH HANDLED; "; + ss << crashReason(sig); +#if ELPP_STACKTRACE + if (stackTraceIfAvailable) { + ss << std::endl << " ======= Backtrace: =========" << std::endl << base::debug::StackTrace(); + } +#else + ELPP_UNUSED(stackTraceIfAvailable); +#endif // ELPP_STACKTRACE + ELPP_WRITE_LOG(el::base::Writer, level, base::DispatchAction::NormalLog, logger) << ss.str(); +} + +static inline void +crashAbort(int sig) { + base::utils::abort(sig, std::string()); +} + +/// @brief Default application crash handler +/// +/// @detail This function writes log using 'default' logger, prints stack trace for GCC based compilers and aborts +/// program. +static inline void +defaultCrashHandler(int sig) { + base::debug::logCrashReason(sig, true, Level::Fatal, base::consts::kDefaultLoggerId); + base::debug::crashAbort(sig); +} + +// CrashHandler + +CrashHandler::CrashHandler(bool useDefault) { + if (useDefault) { + setHandler(defaultCrashHandler); + } +} + +void +CrashHandler::setHandler(const Handler& cHandler) { + m_handler = cHandler; +#if defined(ELPP_HANDLE_SIGABRT) + int i = 0; // SIGABRT is at base::consts::kCrashSignals[0] +#else + int i = 1; +#endif // defined(ELPP_HANDLE_SIGABRT) + for (; i < base::consts::kCrashSignalsCount; ++i) { + m_handler = signal(base::consts::kCrashSignals[i].numb, cHandler); + } +} + +#endif // defined(ELPP_FEATURE_ALL) || defined(ELPP_FEATURE_CRASH_LOG) +} // namespace debug +} // namespace base + +// el + +// Helpers + +#if defined(ELPP_FEATURE_ALL) || defined(ELPP_FEATURE_CRASH_LOG) + +void +Helpers::crashAbort(int sig, const char* sourceFile, unsigned int long line) { + std::stringstream ss; + ss << base::debug::crashReason(sig).c_str(); + ss << " - [Called el::Helpers::crashAbort(" << sig << ")]"; + if (sourceFile != nullptr && strlen(sourceFile) > 0) { + ss << " - Source: " << sourceFile; + if (line > 0) + ss << ":" << line; + else + ss << " (line number not specified)"; + } + base::utils::abort(sig, ss.str()); +} + +void +Helpers::logCrashReason(int sig, bool stackTraceIfAvailable, Level level, const char* logger) { + el::base::debug::logCrashReason(sig, stackTraceIfAvailable, level, logger); +} + +#endif // defined(ELPP_FEATURE_ALL) || defined(ELPP_FEATURE_CRASH_LOG) + +// Loggers + +Logger* +Loggers::getLogger(const std::string& identity, bool registerIfNotAvailable) { + return ELPP->registeredLoggers()->get(identity, registerIfNotAvailable); +} + +void +Loggers::setDefaultLogBuilder(el::LogBuilderPtr& logBuilderPtr) { + ELPP->registeredLoggers()->setDefaultLogBuilder(logBuilderPtr); +} + +bool +Loggers::unregisterLogger(const std::string& identity) { + return ELPP->registeredLoggers()->remove(identity); +} + +bool +Loggers::hasLogger(const std::string& identity) { + return ELPP->registeredLoggers()->has(identity); +} + +Logger* +Loggers::reconfigureLogger(Logger* logger, const Configurations& configurations) { + if (!logger) + return nullptr; + logger->configure(configurations); + return logger; +} + +Logger* +Loggers::reconfigureLogger(const std::string& identity, const Configurations& configurations) { + return Loggers::reconfigureLogger(Loggers::getLogger(identity), configurations); +} + +Logger* +Loggers::reconfigureLogger(const std::string& identity, ConfigurationType configurationType, const std::string& value) { + Logger* logger = Loggers::getLogger(identity); + if (logger == nullptr) { + return nullptr; + } + logger->configurations()->set(Level::Global, configurationType, value); + logger->reconfigure(); + return logger; +} + +void +Loggers::reconfigureAllLoggers(const Configurations& configurations) { + for (base::RegisteredLoggers::iterator it = ELPP->registeredLoggers()->begin(); + it != ELPP->registeredLoggers()->end(); ++it) { + Loggers::reconfigureLogger(it->second, configurations); + } +} + +void +Loggers::reconfigureAllLoggers(Level level, ConfigurationType configurationType, const std::string& value) { + for (base::RegisteredLoggers::iterator it = ELPP->registeredLoggers()->begin(); + it != ELPP->registeredLoggers()->end(); ++it) { + Logger* logger = it->second; + logger->configurations()->set(level, configurationType, value); + logger->reconfigure(); + } +} + +void +Loggers::setDefaultConfigurations(const Configurations& configurations, bool reconfigureExistingLoggers) { + ELPP->registeredLoggers()->setDefaultConfigurations(configurations); + if (reconfigureExistingLoggers) { + Loggers::reconfigureAllLoggers(configurations); + } +} + +const Configurations* +Loggers::defaultConfigurations(void) { + return ELPP->registeredLoggers()->defaultConfigurations(); +} + +const base::LogStreamsReferenceMap* +Loggers::logStreamsReference(void) { + return ELPP->registeredLoggers()->logStreamsReference(); +} + +base::TypedConfigurations +Loggers::defaultTypedConfigurations(void) { + return base::TypedConfigurations(ELPP->registeredLoggers()->defaultConfigurations(), + ELPP->registeredLoggers()->logStreamsReference()); +} + +std::vector* +Loggers::populateAllLoggerIds(std::vector* targetList) { + targetList->clear(); + for (base::RegisteredLoggers::iterator it = ELPP->registeredLoggers()->list().begin(); + it != ELPP->registeredLoggers()->list().end(); ++it) { + targetList->push_back(it->first); + } + return targetList; +} + +void +Loggers::configureFromGlobal(const char* globalConfigurationFilePath) { + std::ifstream gcfStream(globalConfigurationFilePath, std::ifstream::in); + ELPP_ASSERT(gcfStream.is_open(), + "Unable to open global configuration file [" << globalConfigurationFilePath << "] for parsing."); + std::string line = std::string(); + std::stringstream ss; + Logger* logger = nullptr; + auto configure = [&](void) { + ELPP_INTERNAL_INFO(8, "Configuring logger: '" << logger->id() << "' with configurations \n" + << ss.str() << "\n--------------"); + Configurations c; + c.parseFromText(ss.str()); + logger->configure(c); + }; + while (gcfStream.good()) { + std::getline(gcfStream, line); + ELPP_INTERNAL_INFO(1, "Parsing line: " << line); + base::utils::Str::trim(line); + if (Configurations::Parser::isComment(line)) + continue; + Configurations::Parser::ignoreComments(&line); + base::utils::Str::trim(line); + if (line.size() > 2 && base::utils::Str::startsWith(line, std::string(base::consts::kConfigurationLoggerId))) { + if (!ss.str().empty() && logger != nullptr) { + configure(); + } + ss.str(std::string("")); + line = line.substr(2); + base::utils::Str::trim(line); + if (line.size() > 1) { + ELPP_INTERNAL_INFO(1, "Getting logger: '" << line << "'"); + logger = getLogger(line); + } + } else { + ss << line << "\n"; + } + } + if (!ss.str().empty() && logger != nullptr) { + configure(); + } +} + +bool +Loggers::configureFromArg(const char* argKey) { +#if defined(ELPP_DISABLE_CONFIGURATION_FROM_PROGRAM_ARGS) + ELPP_UNUSED(argKey); +#else + if (!Helpers::commandLineArgs()->hasParamWithValue(argKey)) { + return false; + } + configureFromGlobal(Helpers::commandLineArgs()->getParamValue(argKey)); +#endif // defined(ELPP_DISABLE_CONFIGURATION_FROM_PROGRAM_ARGS) + return true; +} + +void +Loggers::flushAll(void) { + ELPP->registeredLoggers()->flushAll(); +} + +void +Loggers::setVerboseLevel(base::type::VerboseLevel level) { + ELPP->vRegistry()->setLevel(level); +} + +base::type::VerboseLevel +Loggers::verboseLevel(void) { + return ELPP->vRegistry()->level(); +} + +void +Loggers::setVModules(const char* modules) { + if (ELPP->vRegistry()->vModulesEnabled()) { + ELPP->vRegistry()->setModules(modules); + } +} + +void +Loggers::clearVModules(void) { + ELPP->vRegistry()->clearModules(); +} + +// VersionInfo + +const std::string +VersionInfo::version(void) { + return std::string("9.96.7"); +} +/// @brief Release date of current version +const std::string +VersionInfo::releaseDate(void) { + return std::string("24-11-2018 0728hrs"); +} + +} // namespace el diff --git a/core/thirdparty/easyloggingpp/easylogging++.h b/core/thirdparty/easyloggingpp/easylogging++.h new file mode 100644 index 0000000000..ce6c7ece48 --- /dev/null +++ b/core/thirdparty/easyloggingpp/easylogging++.h @@ -0,0 +1,5193 @@ +// +// Bismillah ar-Rahmaan ar-Raheem +// +// Easylogging++ v9.96.7 +// Single-header only, cross-platform logging library for C++ applications +// +// Copyright (c) 2012-2018 Zuhd Web Services +// Copyright (c) 2012-2018 @abumusamq +// +// This library is released under the MIT Licence. +// https://github.com/zuhd-org/easyloggingpp/blob/master/LICENSE +// +// https://zuhd.org +// http://muflihun.com +// + +#ifndef EASYLOGGINGPP_H +#define EASYLOGGINGPP_H +// Compilers and C++0x/C++11 Evaluation +#if __cplusplus >= 201103L +#define ELPP_CXX11 1 +#endif // __cplusplus >= 201103L +#if (defined(__GNUC__)) +#define ELPP_COMPILER_GCC 1 +#else +#define ELPP_COMPILER_GCC 0 +#endif +#if ELPP_COMPILER_GCC +#define ELPP_GCC_VERSION (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) +#if defined(__GXX_EXPERIMENTAL_CXX0X__) +#define ELPP_CXX0X 1 +#endif +#endif +// Visual C++ +#if defined(_MSC_VER) +#define ELPP_COMPILER_MSVC 1 +#else +#define ELPP_COMPILER_MSVC 0 +#endif +#define ELPP_CRT_DBG_WARNINGS ELPP_COMPILER_MSVC +#if ELPP_COMPILER_MSVC +#if (_MSC_VER == 1600) +#define ELPP_CXX0X 1 +#elif (_MSC_VER >= 1700) +#define ELPP_CXX11 1 +#endif +#endif +// Clang++ +#if (defined(__clang__) && (__clang__ == 1)) +#define ELPP_COMPILER_CLANG 1 +#else +#define ELPP_COMPILER_CLANG 0 +#endif +#if ELPP_COMPILER_CLANG +#if __has_include() +#include // Make __GLIBCXX__ defined when using libstdc++ +#if !defined(__GLIBCXX__) || __GLIBCXX__ >= 20150426 +#define ELPP_CLANG_SUPPORTS_THREAD +#endif // !defined(__GLIBCXX__) || __GLIBCXX__ >= 20150426 +#endif // __has_include() +#endif +#if (defined(__MINGW32__) || defined(__MINGW64__)) +#define ELPP_MINGW 1 +#else +#define ELPP_MINGW 0 +#endif +#if (defined(__CYGWIN__) && (__CYGWIN__ == 1)) +#define ELPP_CYGWIN 1 +#else +#define ELPP_CYGWIN 0 +#endif +#if (defined(__INTEL_COMPILER)) +#define ELPP_COMPILER_INTEL 1 +#else +#define ELPP_COMPILER_INTEL 0 +#endif +// Operating System Evaluation +// Windows +#if (defined(_WIN32) || defined(_WIN64)) +#define ELPP_OS_WINDOWS 1 +#else +#define ELPP_OS_WINDOWS 0 +#endif +// Linux +#if (defined(__linux) || defined(__linux__)) +#define ELPP_OS_LINUX 1 +#else +#define ELPP_OS_LINUX 0 +#endif +#if (defined(__APPLE__)) +#define ELPP_OS_MAC 1 +#else +#define ELPP_OS_MAC 0 +#endif +#if (defined(__FreeBSD__) || defined(__FreeBSD_kernel__)) +#define ELPP_OS_FREEBSD 1 +#else +#define ELPP_OS_FREEBSD 0 +#endif +#if (defined(__sun)) +#define ELPP_OS_SOLARIS 1 +#else +#define ELPP_OS_SOLARIS 0 +#endif +#if (defined(_AIX)) +#define ELPP_OS_AIX 1 +#else +#define ELPP_OS_AIX 0 +#endif +#if (defined(__NetBSD__)) +#define ELPP_OS_NETBSD 1 +#else +#define ELPP_OS_NETBSD 0 +#endif +#if defined(__EMSCRIPTEN__) +#define ELPP_OS_EMSCRIPTEN 1 +#else +#define ELPP_OS_EMSCRIPTEN 0 +#endif +// Unix +#if ((ELPP_OS_LINUX || ELPP_OS_MAC || ELPP_OS_FREEBSD || ELPP_OS_NETBSD || ELPP_OS_SOLARIS || ELPP_OS_AIX || \ + ELPP_OS_EMSCRIPTEN) && \ + (!ELPP_OS_WINDOWS)) +#define ELPP_OS_UNIX 1 +#else +#define ELPP_OS_UNIX 0 +#endif +#if (defined(__ANDROID__)) +#define ELPP_OS_ANDROID 1 +#else +#define ELPP_OS_ANDROID 0 +#endif +// Evaluating Cygwin as *nix OS +#if !ELPP_OS_UNIX && !ELPP_OS_WINDOWS && ELPP_CYGWIN +#undef ELPP_OS_UNIX +#undef ELPP_OS_LINUX +#define ELPP_OS_UNIX 1 +#define ELPP_OS_LINUX 1 +#endif // !ELPP_OS_UNIX && !ELPP_OS_WINDOWS && ELPP_CYGWIN +#if !defined(ELPP_INTERNAL_DEBUGGING_OUT_INFO) +#define ELPP_INTERNAL_DEBUGGING_OUT_INFO std::cout +#endif // !defined(ELPP_INTERNAL_DEBUGGING_OUT) +#if !defined(ELPP_INTERNAL_DEBUGGING_OUT_ERROR) +#define ELPP_INTERNAL_DEBUGGING_OUT_ERROR std::cerr +#endif // !defined(ELPP_INTERNAL_DEBUGGING_OUT) +#if !defined(ELPP_INTERNAL_DEBUGGING_ENDL) +#define ELPP_INTERNAL_DEBUGGING_ENDL std::endl +#endif // !defined(ELPP_INTERNAL_DEBUGGING_OUT) +#if !defined(ELPP_INTERNAL_DEBUGGING_MSG) +#define ELPP_INTERNAL_DEBUGGING_MSG(msg) msg +#endif // !defined(ELPP_INTERNAL_DEBUGGING_OUT) +// Internal Assertions and errors +#if !defined(ELPP_DISABLE_ASSERT) +#if (defined(ELPP_DEBUG_ASSERT_FAILURE)) +#define ELPP_ASSERT(expr, msg) \ + if (!(expr)) { \ + std::stringstream internalInfoStream; \ + internalInfoStream << msg; \ + ELPP_INTERNAL_DEBUGGING_OUT_ERROR \ + << "EASYLOGGING++ ASSERTION FAILED (LINE: " << __LINE__ << ") [" #expr << "] WITH MESSAGE \"" \ + << ELPP_INTERNAL_DEBUGGING_MSG(internalInfoStream.str()) << "\"" << ELPP_INTERNAL_DEBUGGING_ENDL; \ + base::utils::abort(1, "ELPP Assertion failure, please define ELPP_DEBUG_ASSERT_FAILURE"); \ + } +#else +#define ELPP_ASSERT(expr, msg) \ + if (!(expr)) { \ + std::stringstream internalInfoStream; \ + internalInfoStream << msg; \ + ELPP_INTERNAL_DEBUGGING_OUT_ERROR \ + << "ASSERTION FAILURE FROM EASYLOGGING++ (LINE: " << __LINE__ << ") [" #expr << "] WITH MESSAGE \"" \ + << ELPP_INTERNAL_DEBUGGING_MSG(internalInfoStream.str()) << "\"" << ELPP_INTERNAL_DEBUGGING_ENDL; \ + } +#endif // (defined(ELPP_DEBUG_ASSERT_FAILURE)) +#else +#define ELPP_ASSERT(x, y) +#endif //(!defined(ELPP_DISABLE_ASSERT) +#if ELPP_COMPILER_MSVC +#define ELPP_INTERNAL_DEBUGGING_WRITE_PERROR \ + { \ + char buff[256]; \ + strerror_s(buff, 256, errno); \ + ELPP_INTERNAL_DEBUGGING_OUT_ERROR << ": " << buff << " [" << errno << "]"; \ + } \ + (void)0 +#else +#define ELPP_INTERNAL_DEBUGGING_WRITE_PERROR \ + ELPP_INTERNAL_DEBUGGING_OUT_ERROR << ": " << strerror(errno) << " [" << errno << "]"; \ + (void)0 +#endif // ELPP_COMPILER_MSVC +#if defined(ELPP_DEBUG_ERRORS) +#if !defined(ELPP_INTERNAL_ERROR) +#define ELPP_INTERNAL_ERROR(msg, pe) \ + { \ + std::stringstream internalInfoStream; \ + internalInfoStream << " " << msg; \ + ELPP_INTERNAL_DEBUGGING_OUT_ERROR << "ERROR FROM EASYLOGGING++ (LINE: " << __LINE__ << ") " \ + << ELPP_INTERNAL_DEBUGGING_MSG(internalInfoStream.str()) \ + << ELPP_INTERNAL_DEBUGGING_ENDL; \ + if (pe) { \ + ELPP_INTERNAL_DEBUGGING_OUT_ERROR << " "; \ + ELPP_INTERNAL_DEBUGGING_WRITE_PERROR; \ + } \ + } \ + (void)0 +#endif +#else +#undef ELPP_INTERNAL_INFO +#define ELPP_INTERNAL_ERROR(msg, pe) +#endif // defined(ELPP_DEBUG_ERRORS) +#if (defined(ELPP_DEBUG_INFO)) +#if !(defined(ELPP_INTERNAL_INFO_LEVEL)) +#define ELPP_INTERNAL_INFO_LEVEL 9 +#endif // !(defined(ELPP_INTERNAL_INFO_LEVEL)) +#if !defined(ELPP_INTERNAL_INFO) +#define ELPP_INTERNAL_INFO(lvl, msg) \ + { \ + if (lvl <= ELPP_INTERNAL_INFO_LEVEL) { \ + std::stringstream internalInfoStream; \ + internalInfoStream << " " << msg; \ + ELPP_INTERNAL_DEBUGGING_OUT_INFO << ELPP_INTERNAL_DEBUGGING_MSG(internalInfoStream.str()) \ + << ELPP_INTERNAL_DEBUGGING_ENDL; \ + } \ + } +#endif +#else +#undef ELPP_INTERNAL_INFO +#define ELPP_INTERNAL_INFO(lvl, msg) +#endif // (defined(ELPP_DEBUG_INFO)) +#if (defined(ELPP_FEATURE_ALL)) || (defined(ELPP_FEATURE_CRASH_LOG)) +#if (ELPP_COMPILER_GCC && !ELPP_MINGW && !ELPP_OS_ANDROID && !ELPP_OS_EMSCRIPTEN) +#define ELPP_STACKTRACE 1 +#else +#if ELPP_COMPILER_MSVC +#pragma message("Stack trace not available for this compiler") +#else +#warning "Stack trace not available for this compiler"; +#endif // ELPP_COMPILER_MSVC +#define ELPP_STACKTRACE 0 +#endif // ELPP_COMPILER_GCC +#else +#define ELPP_STACKTRACE 0 +#endif // (defined(ELPP_FEATURE_ALL)) || (defined(ELPP_FEATURE_CRASH_LOG)) +// Miscellaneous macros +#define ELPP_UNUSED(x) (void)x +#if ELPP_OS_UNIX +// Log file permissions for unix-based systems +#define ELPP_LOG_PERMS S_IRUSR | S_IWUSR | S_IXUSR | S_IWGRP | S_IRGRP | S_IXGRP | S_IWOTH | S_IXOTH +#endif // ELPP_OS_UNIX +#if defined(ELPP_AS_DLL) && ELPP_COMPILER_MSVC +#if defined(ELPP_EXPORT_SYMBOLS) +#define ELPP_EXPORT __declspec(dllexport) +#else +#define ELPP_EXPORT __declspec(dllimport) +#endif // defined(ELPP_EXPORT_SYMBOLS) +#else +#define ELPP_EXPORT +#endif // defined(ELPP_AS_DLL) && ELPP_COMPILER_MSVC +// Some special functions that are VC++ specific +#undef STRTOK +#undef STRERROR +#undef STRCAT +#undef STRCPY +#if ELPP_CRT_DBG_WARNINGS +#define STRTOK(a, b, c) strtok_s(a, b, c) +#define STRERROR(a, b, c) strerror_s(a, b, c) +#define STRCAT(a, b, len) strcat_s(a, len, b) +#define STRCPY(a, b, len) strcpy_s(a, len, b) +#else +#define STRTOK(a, b, c) strtok(a, b) +#define STRERROR(a, b, c) strerror(c) +#define STRCAT(a, b, len) strcat(a, b) +#define STRCPY(a, b, len) strcpy(a, b) +#endif +// Compiler specific support evaluations +#if (ELPP_MINGW && !defined(ELPP_FORCE_USE_STD_THREAD)) +#define ELPP_USE_STD_THREADING 0 +#else +#if ((ELPP_COMPILER_CLANG && defined(ELPP_CLANG_SUPPORTS_THREAD)) || (!ELPP_COMPILER_CLANG && defined(ELPP_CXX11)) || \ + defined(ELPP_FORCE_USE_STD_THREAD)) +#define ELPP_USE_STD_THREADING 1 +#else +#define ELPP_USE_STD_THREADING 0 +#endif +#endif +#undef ELPP_FINAL +#if ELPP_COMPILER_INTEL || (ELPP_GCC_VERSION < 40702) +#define ELPP_FINAL +#else +#define ELPP_FINAL final +#endif // ELPP_COMPILER_INTEL || (ELPP_GCC_VERSION < 40702) +#if defined(ELPP_EXPERIMENTAL_ASYNC) +#define ELPP_ASYNC_LOGGING 1 +#else +#define ELPP_ASYNC_LOGGING 0 +#endif // defined(ELPP_EXPERIMENTAL_ASYNC) +#if defined(ELPP_THREAD_SAFE) || ELPP_ASYNC_LOGGING +#define ELPP_THREADING_ENABLED 1 +#else +#define ELPP_THREADING_ENABLED 0 +#endif // defined(ELPP_THREAD_SAFE) || ELPP_ASYNC_LOGGING +// Function macro ELPP_FUNC +#undef ELPP_FUNC +#if ELPP_COMPILER_MSVC // Visual C++ +#define ELPP_FUNC __FUNCSIG__ +#elif ELPP_COMPILER_GCC // GCC +#define ELPP_FUNC __PRETTY_FUNCTION__ +#elif ELPP_COMPILER_INTEL // Intel C++ +#define ELPP_FUNC __PRETTY_FUNCTION__ +#elif ELPP_COMPILER_CLANG // Clang++ +#define ELPP_FUNC __PRETTY_FUNCTION__ +#else +#if defined(__func__) +#define ELPP_FUNC __func__ +#else +#define ELPP_FUNC "" +#endif // defined(__func__) +#endif // defined(_MSC_VER) +#undef ELPP_VARIADIC_TEMPLATES_SUPPORTED +// Keep following line commented until features are fixed +#define ELPP_VARIADIC_TEMPLATES_SUPPORTED \ + (ELPP_COMPILER_GCC || ELPP_COMPILER_CLANG || ELPP_COMPILER_INTEL || (ELPP_COMPILER_MSVC && _MSC_VER >= 1800)) +// Logging Enable/Disable macros +#if defined(ELPP_DISABLE_LOGS) +#define ELPP_LOGGING_ENABLED 0 +#else +#define ELPP_LOGGING_ENABLED 1 +#endif +#if (!defined(ELPP_DISABLE_DEBUG_LOGS) && (ELPP_LOGGING_ENABLED)) +#define ELPP_DEBUG_LOG 1 +#else +#define ELPP_DEBUG_LOG 0 +#endif // (!defined(ELPP_DISABLE_DEBUG_LOGS) && (ELPP_LOGGING_ENABLED)) +#if (!defined(ELPP_DISABLE_INFO_LOGS) && (ELPP_LOGGING_ENABLED)) +#define ELPP_INFO_LOG 1 +#else +#define ELPP_INFO_LOG 0 +#endif // (!defined(ELPP_DISABLE_INFO_LOGS) && (ELPP_LOGGING_ENABLED)) +#if (!defined(ELPP_DISABLE_WARNING_LOGS) && (ELPP_LOGGING_ENABLED)) +#define ELPP_WARNING_LOG 1 +#else +#define ELPP_WARNING_LOG 0 +#endif // (!defined(ELPP_DISABLE_WARNING_LOGS) && (ELPP_LOGGING_ENABLED)) +#if (!defined(ELPP_DISABLE_ERROR_LOGS) && (ELPP_LOGGING_ENABLED)) +#define ELPP_ERROR_LOG 1 +#else +#define ELPP_ERROR_LOG 0 +#endif // (!defined(ELPP_DISABLE_ERROR_LOGS) && (ELPP_LOGGING_ENABLED)) +#if (!defined(ELPP_DISABLE_FATAL_LOGS) && (ELPP_LOGGING_ENABLED)) +#define ELPP_FATAL_LOG 1 +#else +#define ELPP_FATAL_LOG 0 +#endif // (!defined(ELPP_DISABLE_FATAL_LOGS) && (ELPP_LOGGING_ENABLED)) +#if (!defined(ELPP_DISABLE_TRACE_LOGS) && (ELPP_LOGGING_ENABLED)) +#define ELPP_TRACE_LOG 1 +#else +#define ELPP_TRACE_LOG 0 +#endif // (!defined(ELPP_DISABLE_TRACE_LOGS) && (ELPP_LOGGING_ENABLED)) +#if (!defined(ELPP_DISABLE_VERBOSE_LOGS) && (ELPP_LOGGING_ENABLED)) +#define ELPP_VERBOSE_LOG 1 +#else +#define ELPP_VERBOSE_LOG 0 +#endif // (!defined(ELPP_DISABLE_VERBOSE_LOGS) && (ELPP_LOGGING_ENABLED)) +#if (!(ELPP_CXX0X || ELPP_CXX11)) +#error "C++0x (or higher) support not detected! (Is `-std=c++11' missing?)" +#endif // (!(ELPP_CXX0X || ELPP_CXX11)) +// Headers +#if defined(ELPP_SYSLOG) +#include +#endif // defined(ELPP_SYSLOG) +#include +#include +#include +#include +#include +#include +#include +#include +#if defined(ELPP_UNICODE) +#include +#if ELPP_OS_WINDOWS +#include +#endif // ELPP_OS_WINDOWS +#endif // defined(ELPP_UNICODE) +#if ELPP_STACKTRACE +#include +#include +#endif // ELPP_STACKTRACE +#if ELPP_OS_ANDROID +#include +#endif // ELPP_OS_ANDROID +#if ELPP_OS_UNIX +#include +#include +#elif ELPP_OS_WINDOWS +#include +#include +#if defined(WIN32_LEAN_AND_MEAN) +#if defined(ELPP_WINSOCK2) +#include +#else +#include +#endif // defined(ELPP_WINSOCK2) +#endif // defined(WIN32_LEAN_AND_MEAN) +#endif // ELPP_OS_UNIX +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#if ELPP_THREADING_ENABLED +#if ELPP_USE_STD_THREADING +#include +#include +#else +#if ELPP_OS_UNIX +#include +#endif // ELPP_OS_UNIX +#endif // ELPP_USE_STD_THREADING +#endif // ELPP_THREADING_ENABLED +#if ELPP_ASYNC_LOGGING +#if defined(ELPP_NO_SLEEP_FOR) +#include +#endif // defined(ELPP_NO_SLEEP_FOR) +#include +#include +#include +#endif // ELPP_ASYNC_LOGGING +#if defined(ELPP_STL_LOGGING) +// For logging STL based templates +#include +#include +#include +#include +#include +#include +#if defined(ELPP_LOG_STD_ARRAY) +#include +#endif // defined(ELPP_LOG_STD_ARRAY) +#if defined(ELPP_LOG_UNORDERED_SET) +#include +#endif // defined(ELPP_UNORDERED_SET) +#endif // defined(ELPP_STL_LOGGING) +#if defined(ELPP_QT_LOGGING) +// For logging Qt based classes & templates +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#endif // defined(ELPP_QT_LOGGING) +#if defined(ELPP_BOOST_LOGGING) +// For logging boost based classes & templates +#include +#include +#include +#include +#include +#include +#include +#include +#endif // defined(ELPP_BOOST_LOGGING) +#if defined(ELPP_WXWIDGETS_LOGGING) +// For logging wxWidgets based classes & templates +#include +#endif // defined(ELPP_WXWIDGETS_LOGGING) +#if defined(ELPP_UTC_DATETIME) +#define elpptime_r gmtime_r +#define elpptime_s gmtime_s +#define elpptime gmtime +#else +#define elpptime_r localtime_r +#define elpptime_s localtime_s +#define elpptime localtime +#endif // defined(ELPP_UTC_DATETIME) +// Forward declarations +namespace el { +class Logger; +class LogMessage; +class PerformanceTrackingData; +class Loggers; +class Helpers; +template +class Callback; +class LogDispatchCallback; +class PerformanceTrackingCallback; +class LoggerRegistrationCallback; +class LogDispatchData; +namespace base { +class Storage; +class RegisteredLoggers; +class PerformanceTracker; +class MessageBuilder; +class Writer; +class PErrorWriter; +class LogDispatcher; +class DefaultLogBuilder; +class DefaultLogDispatchCallback; +#if ELPP_ASYNC_LOGGING +class AsyncLogDispatchCallback; +class AsyncDispatchWorker; +#endif // ELPP_ASYNC_LOGGING +class DefaultPerformanceTrackingCallback; +} // namespace base +} // namespace el +/// @brief Easylogging++ entry namespace +namespace el { +/// @brief Namespace containing base/internal functionality used by Easylogging++ +namespace base { +/// @brief Data types used by Easylogging++ +namespace type { +#undef ELPP_LITERAL +#undef ELPP_STRLEN +#undef ELPP_COUT +#if defined(ELPP_UNICODE) +#define ELPP_LITERAL(txt) L##txt +#define ELPP_STRLEN wcslen +#if defined ELPP_CUSTOM_COUT +#define ELPP_COUT ELPP_CUSTOM_COUT +#else +#define ELPP_COUT std::wcout +#endif // defined ELPP_CUSTOM_COUT +typedef wchar_t char_t; +typedef std::wstring string_t; +typedef std::wstringstream stringstream_t; +typedef std::wfstream fstream_t; +typedef std::wostream ostream_t; +#else +#define ELPP_LITERAL(txt) txt +#define ELPP_STRLEN strlen +#if defined ELPP_CUSTOM_COUT +#define ELPP_COUT ELPP_CUSTOM_COUT +#else +#define ELPP_COUT std::cout +#endif // defined ELPP_CUSTOM_COUT +typedef char char_t; +typedef std::string string_t; +typedef std::stringstream stringstream_t; +typedef std::fstream fstream_t; +typedef std::ostream ostream_t; +#endif // defined(ELPP_UNICODE) +#if defined(ELPP_CUSTOM_COUT_LINE) +#define ELPP_COUT_LINE(logLine) ELPP_CUSTOM_COUT_LINE(logLine) +#else +#define ELPP_COUT_LINE(logLine) logLine << std::flush +#endif // defined(ELPP_CUSTOM_COUT_LINE) +typedef unsigned int EnumType; +typedef unsigned short VerboseLevel; +typedef unsigned long int LineNumber; +typedef std::shared_ptr StoragePointer; +typedef std::shared_ptr LogDispatchCallbackPtr; +typedef std::shared_ptr PerformanceTrackingCallbackPtr; +typedef std::shared_ptr LoggerRegistrationCallbackPtr; +typedef std::unique_ptr PerformanceTrackerPtr; +} // namespace type +/// @brief Internal helper class that prevent copy constructor for class +/// +/// @detail When using this class simply inherit it privately +class NoCopy { + protected: + NoCopy(void) { + } + + private: + NoCopy(const NoCopy&); + NoCopy& + operator=(const NoCopy&); +}; +/// @brief Internal helper class that makes all default constructors private. +/// +/// @detail This prevents initializing class making it static unless an explicit constructor is declared. +/// When using this class simply inherit it privately +class StaticClass { + private: + StaticClass(void); + StaticClass(const StaticClass&); + StaticClass& + operator=(const StaticClass&); +}; +} // namespace base +/// @brief Represents enumeration for severity level used to determine level of logging +/// +/// @detail With Easylogging++, developers may disable or enable any level regardless of +/// what the severity is. Or they can choose to log using hierarchical logging flag +enum class Level : base::type::EnumType { + /// @brief Generic level that represents all the levels. Useful when setting global configuration for all levels + Global = 1, + /// @brief Information that can be useful to back-trace certain events - mostly useful than debug logs. + Trace = 2, + /// @brief Informational events most useful for developers to debug application + Debug = 4, + /// @brief Severe error information that will presumably abort application + Fatal = 8, + /// @brief Information representing errors in application but application will keep running + Error = 16, + /// @brief Useful when application has potentially harmful situtaions + Warning = 32, + /// @brief Information that can be highly useful and vary with verbose logging level. + Verbose = 64, + /// @brief Mainly useful to represent current progress of application + Info = 128, + /// @brief Represents unknown level + Unknown = 1010 +}; +} // namespace el +namespace std { +template <> +struct hash { + public: + std::size_t + operator()(const el::Level& l) const { + return hash{}(static_cast(l)); + } +}; +} // namespace std +namespace el { +/// @brief Static class that contains helper functions for el::Level +class LevelHelper : base::StaticClass { + public: + /// @brief Represents minimum valid level. Useful when iterating through enum. + static const base::type::EnumType kMinValid = static_cast(Level::Trace); + /// @brief Represents maximum valid level. This is used internally and you should not need it. + static const base::type::EnumType kMaxValid = static_cast(Level::Info); + /// @brief Casts level to int, useful for iterating through enum. + static base::type::EnumType + castToInt(Level level) { + return static_cast(level); + } + /// @brief Casts int(ushort) to level, useful for iterating through enum. + static Level + castFromInt(base::type::EnumType l) { + return static_cast(l); + } + /// @brief Converts level to associated const char* + /// @return Upper case string based level. + static const char* + convertToString(Level level); + /// @brief Converts from levelStr to Level + /// @param levelStr Upper case string based level. + /// Lower case is also valid but providing upper case is recommended. + static Level + convertFromString(const char* levelStr); + /// @brief Applies specified function to each level starting from startIndex + /// @param startIndex initial value to start the iteration from. This is passed as pointer and + /// is left-shifted so this can be used inside function (fn) to represent current level. + /// @param fn function to apply with each level. This bool represent whether or not to stop iterating through + /// levels. + static void + forEachLevel(base::type::EnumType* startIndex, const std::function& fn); +}; +/// @brief Represents enumeration of ConfigurationType used to configure or access certain aspect +/// of logging +enum class ConfigurationType : base::type::EnumType { + /// @brief Determines whether or not corresponding level and logger of logging is enabled + /// You may disable all logs by using el::Level::Global + Enabled = 1, + /// @brief Whether or not to write corresponding log to log file + ToFile = 2, + /// @brief Whether or not to write corresponding level and logger log to standard output. + /// By standard output meaning termnal, command prompt etc + ToStandardOutput = 4, + /// @brief Determines format of logging corresponding level and logger. + Format = 8, + /// @brief Determines log file (full path) to write logs to for correponding level and logger + Filename = 16, + /// @brief Specifies precision of the subsecond part. It should be within range (1-6). + SubsecondPrecision = 32, + /// @brief Alias of SubsecondPrecision (for backward compatibility) + MillisecondsWidth = SubsecondPrecision, + /// @brief Determines whether or not performance tracking is enabled. + /// + /// @detail This does not depend on logger or level. Performance tracking always uses 'performance' logger + PerformanceTracking = 64, + /// @brief Specifies log file max size. + /// + /// @detail If file size of corresponding log file (for corresponding level) is >= specified size, log file will + /// be truncated and re-initiated. + MaxLogFileSize = 128, + /// @brief Specifies number of log entries to hold until we flush pending log data + LogFlushThreshold = 256, + /// @brief Represents unknown configuration + Unknown = 1010 +}; +/// @brief Static class that contains helper functions for el::ConfigurationType +class ConfigurationTypeHelper : base::StaticClass { + public: + /// @brief Represents minimum valid configuration type. Useful when iterating through enum. + static const base::type::EnumType kMinValid = static_cast(ConfigurationType::Enabled); + /// @brief Represents maximum valid configuration type. This is used internally and you should not need it. + static const base::type::EnumType kMaxValid = static_cast(ConfigurationType::MaxLogFileSize); + /// @brief Casts configuration type to int, useful for iterating through enum. + static base::type::EnumType + castToInt(ConfigurationType configurationType) { + return static_cast(configurationType); + } + /// @brief Casts int(ushort) to configurationt type, useful for iterating through enum. + static ConfigurationType + castFromInt(base::type::EnumType c) { + return static_cast(c); + } + /// @brief Converts configuration type to associated const char* + /// @returns Upper case string based configuration type. + static const char* + convertToString(ConfigurationType configurationType); + /// @brief Converts from configStr to ConfigurationType + /// @param configStr Upper case string based configuration type. + /// Lower case is also valid but providing upper case is recommended. + static ConfigurationType + convertFromString(const char* configStr); + /// @brief Applies specified function to each configuration type starting from startIndex + /// @param startIndex initial value to start the iteration from. This is passed by pointer and is left-shifted + /// so this can be used inside function (fn) to represent current configuration type. + /// @param fn function to apply with each configuration type. + /// This bool represent whether or not to stop iterating through configurations. + static inline void + forEachConfigType(base::type::EnumType* startIndex, const std::function& fn); +}; +/// @brief Flags used while writing logs. This flags are set by user +enum class LoggingFlag : base::type::EnumType { + /// @brief Makes sure we have new line for each container log entry + NewLineForContainer = 1, + /// @brief Makes sure if -vmodule is used and does not specifies a module, then verbose + /// logging is allowed via that module. + AllowVerboseIfModuleNotSpecified = 2, + /// @brief When handling crashes by default, detailed crash reason will be logged as well + LogDetailedCrashReason = 4, + /// @brief Allows to disable application abortion when logged using FATAL level + DisableApplicationAbortOnFatalLog = 8, + /// @brief Flushes log with every log-entry (performance sensative) - Disabled by default + ImmediateFlush = 16, + /// @brief Enables strict file rolling + StrictLogFileSizeCheck = 32, + /// @brief Make terminal output colorful for supported terminals + ColoredTerminalOutput = 64, + /// @brief Supports use of multiple logging in same macro, e.g, CLOG(INFO, "default", "network") + MultiLoggerSupport = 128, + /// @brief Disables comparing performance tracker's checkpoints + DisablePerformanceTrackingCheckpointComparison = 256, + /// @brief Disable VModules + DisableVModules = 512, + /// @brief Disable VModules extensions + DisableVModulesExtensions = 1024, + /// @brief Enables hierarchical logging + HierarchicalLogging = 2048, + /// @brief Creates logger automatically when not available + CreateLoggerAutomatically = 4096, + /// @brief Adds spaces b/w logs that separated by left-shift operator + AutoSpacing = 8192, + /// @brief Preserves time format and does not convert it to sec, hour etc (performance tracking only) + FixedTimeFormat = 16384, + // @brief Ignore SIGINT or crash + IgnoreSigInt = 32768, +}; +namespace base { +/// @brief Namespace containing constants used internally. +namespace consts { +static const char kFormatSpecifierCharValue = 'v'; +static const char kFormatSpecifierChar = '%'; +static const unsigned int kMaxLogPerCounter = 100000; +static const unsigned int kMaxLogPerContainer = 100; +static const unsigned int kDefaultSubsecondPrecision = 3; + +#ifdef ELPP_DEFAULT_LOGGER +static const char* kDefaultLoggerId = ELPP_DEFAULT_LOGGER; +#else +static const char* kDefaultLoggerId = "default"; +#endif + +#if defined(ELPP_FEATURE_ALL) || defined(ELPP_FEATURE_PERFORMANCE_TRACKING) +#ifdef ELPP_DEFAULT_PERFORMANCE_LOGGER +static const char* kPerformanceLoggerId = ELPP_DEFAULT_PERFORMANCE_LOGGER; +#else +static const char* kPerformanceLoggerId = "performance"; +#endif // ELPP_DEFAULT_PERFORMANCE_LOGGER +#endif + +#if defined(ELPP_SYSLOG) +static const char* kSysLogLoggerId = "syslog"; +#endif // defined(ELPP_SYSLOG) + +#if ELPP_OS_WINDOWS +static const char* kFilePathSeperator = "\\"; +#else +static const char* kFilePathSeperator = "/"; +#endif // ELPP_OS_WINDOWS + +static const std::size_t kSourceFilenameMaxLength = 100; +static const std::size_t kSourceLineMaxLength = 10; +static const Level kPerformanceTrackerDefaultLevel = Level::Info; +const struct { + double value; + const base::type::char_t* unit; +} kTimeFormats[] = {{1000.0f, ELPP_LITERAL("us")}, {1000.0f, ELPP_LITERAL("ms")}, {60.0f, ELPP_LITERAL("seconds")}, + {60.0f, ELPP_LITERAL("minutes")}, {24.0f, ELPP_LITERAL("hours")}, {7.0f, ELPP_LITERAL("days")}}; +static const int kTimeFormatsCount = sizeof(kTimeFormats) / sizeof(kTimeFormats[0]); +const struct { + int numb; + const char* name; + const char* brief; + const char* detail; +} kCrashSignals[] = { + // NOTE: Do not re-order, if you do please check CrashHandler(bool) constructor and CrashHandler::setHandler(..) + {SIGABRT, "SIGABRT", "Abnormal termination", "Program was abnormally terminated."}, + {SIGFPE, "SIGFPE", "Erroneous arithmetic operation", + "Arithemetic operation issue such as division by zero or operation resulting in overflow."}, + {SIGILL, "SIGILL", "Illegal instruction", + "Generally due to a corruption in the code or to an attempt to execute data."}, + {SIGSEGV, "SIGSEGV", "Invalid access to memory", + "Program is trying to read an invalid (unallocated, deleted or corrupted) or inaccessible memory."}, + {SIGINT, "SIGINT", "Interactive attention signal", + "Interruption generated (generally) by user or operating system."}, +}; +static const int kCrashSignalsCount = sizeof(kCrashSignals) / sizeof(kCrashSignals[0]); +} // namespace consts +} // namespace base +typedef std::function PreRollOutCallback; +namespace base { +static inline void +defaultPreRollOutCallback(const char*, std::size_t, Level level) { +} +/// @brief Enum to represent timestamp unit +enum class TimestampUnit : base::type::EnumType { + Microsecond = 0, + Millisecond = 1, + Second = 2, + Minute = 3, + Hour = 4, + Day = 5 +}; +/// @brief Format flags used to determine specifiers that are active for performance improvements. +enum class FormatFlags : base::type::EnumType { + DateTime = 1 << 1, + LoggerId = 1 << 2, + File = 1 << 3, + Line = 1 << 4, + Location = 1 << 5, + Function = 1 << 6, + User = 1 << 7, + Host = 1 << 8, + LogMessage = 1 << 9, + VerboseLevel = 1 << 10, + AppName = 1 << 11, + ThreadId = 1 << 12, + Level = 1 << 13, + FileBase = 1 << 14, + LevelShort = 1 << 15 +}; +/// @brief A subsecond precision class containing actual width and offset of the subsecond part +class SubsecondPrecision { + public: + SubsecondPrecision(void) { + init(base::consts::kDefaultSubsecondPrecision); + } + explicit SubsecondPrecision(int width) { + init(width); + } + bool + operator==(const SubsecondPrecision& ssPrec) { + return m_width == ssPrec.m_width && m_offset == ssPrec.m_offset; + } + int m_width; + unsigned int m_offset; + + private: + void + init(int width); +}; +/// @brief Type alias of SubsecondPrecision +typedef SubsecondPrecision MillisecondsWidth; +/// @brief Namespace containing utility functions/static classes used internally +namespace utils { +/// @brief Deletes memory safely and points to null +template +static typename std::enable_if::value, void>::type +safeDelete(T*& pointer) { + if (pointer == nullptr) + return; + delete pointer; + pointer = nullptr; +} +/// @brief Bitwise operations for C++11 strong enum class. This casts e into Flag_T and returns value after bitwise +/// operation Use these function as 
flag = bitwise::Or(MyEnum::val1, flag);
+namespace bitwise { +template +static inline base::type::EnumType +And(Enum e, base::type::EnumType flag) { + return static_cast(flag) & static_cast(e); +} +template +static inline base::type::EnumType +Not(Enum e, base::type::EnumType flag) { + return static_cast(flag) & ~(static_cast(e)); +} +template +static inline base::type::EnumType +Or(Enum e, base::type::EnumType flag) { + return static_cast(flag) | static_cast(e); +} +} // namespace bitwise +template +static inline void +addFlag(Enum e, base::type::EnumType* flag) { + *flag = base::utils::bitwise::Or(e, *flag); +} +template +static inline void +removeFlag(Enum e, base::type::EnumType* flag) { + *flag = base::utils::bitwise::Not(e, *flag); +} +template +static inline bool +hasFlag(Enum e, base::type::EnumType flag) { + return base::utils::bitwise::And(e, flag) > 0x0; +} +} // namespace utils +namespace threading { +#if ELPP_THREADING_ENABLED +#if !ELPP_USE_STD_THREADING +namespace internal { +/// @brief A mutex wrapper for compiler that dont yet support std::recursive_mutex +class Mutex : base::NoCopy { + public: + Mutex(void) { +#if ELPP_OS_UNIX + pthread_mutexattr_t attr; + pthread_mutexattr_init(&attr); + pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_RECURSIVE); + pthread_mutex_init(&m_underlyingMutex, &attr); + pthread_mutexattr_destroy(&attr); +#elif ELPP_OS_WINDOWS + InitializeCriticalSection(&m_underlyingMutex); +#endif // ELPP_OS_UNIX + } + + virtual ~Mutex(void) { +#if ELPP_OS_UNIX + pthread_mutex_destroy(&m_underlyingMutex); +#elif ELPP_OS_WINDOWS + DeleteCriticalSection(&m_underlyingMutex); +#endif // ELPP_OS_UNIX + } + + inline void + lock(void) { +#if ELPP_OS_UNIX + pthread_mutex_lock(&m_underlyingMutex); +#elif ELPP_OS_WINDOWS + EnterCriticalSection(&m_underlyingMutex); +#endif // ELPP_OS_UNIX + } + + inline bool + try_lock(void) { +#if ELPP_OS_UNIX + return (pthread_mutex_trylock(&m_underlyingMutex) == 0); +#elif ELPP_OS_WINDOWS + return TryEnterCriticalSection(&m_underlyingMutex); +#endif // ELPP_OS_UNIX + } + + inline void + unlock(void) { +#if ELPP_OS_UNIX + pthread_mutex_unlock(&m_underlyingMutex); +#elif ELPP_OS_WINDOWS + LeaveCriticalSection(&m_underlyingMutex); +#endif // ELPP_OS_UNIX + } + + private: +#if ELPP_OS_UNIX + pthread_mutex_t m_underlyingMutex; +#elif ELPP_OS_WINDOWS + CRITICAL_SECTION m_underlyingMutex; +#endif // ELPP_OS_UNIX +}; +/// @brief Scoped lock for compiler that dont yet support std::lock_guard +template +class ScopedLock : base::NoCopy { + public: + explicit ScopedLock(M& mutex) { + m_mutex = &mutex; + m_mutex->lock(); + } + + virtual ~ScopedLock(void) { + m_mutex->unlock(); + } + + private: + M* m_mutex; + ScopedLock(void); +}; +} // namespace internal +typedef base::threading::internal::Mutex Mutex; +typedef base::threading::internal::ScopedLock ScopedLock; +#else +typedef std::recursive_mutex Mutex; +typedef std::lock_guard ScopedLock; +#endif // !ELPP_USE_STD_THREADING +#else +namespace internal { +/// @brief Mutex wrapper used when multi-threading is disabled. +class NoMutex : base::NoCopy { + public: + NoMutex(void) { + } + inline void + lock(void) { + } + inline bool + try_lock(void) { + return true; + } + inline void + unlock(void) { + } +}; +/// @brief Lock guard wrapper used when multi-threading is disabled. +template +class NoScopedLock : base::NoCopy { + public: + explicit NoScopedLock(Mutex&) { + } + virtual ~NoScopedLock(void) { + } + + private: + NoScopedLock(void); +}; +} // namespace internal +typedef base::threading::internal::NoMutex Mutex; +typedef base::threading::internal::NoScopedLock ScopedLock; +#endif // ELPP_THREADING_ENABLED +/// @brief Base of thread safe class, this class is inheritable-only +class ThreadSafe { + public: + virtual inline void + acquireLock(void) ELPP_FINAL { + m_mutex.lock(); + } + virtual inline void + releaseLock(void) ELPP_FINAL { + m_mutex.unlock(); + } + virtual inline base::threading::Mutex& + lock(void) ELPP_FINAL { + return m_mutex; + } + + protected: + ThreadSafe(void) { + } + virtual ~ThreadSafe(void) { + } + + private: + base::threading::Mutex m_mutex; +}; + +#if ELPP_THREADING_ENABLED +#if !ELPP_USE_STD_THREADING +/// @brief Gets ID of currently running threading in windows systems. On unix, nothing is returned. +static std::string +getCurrentThreadId(void) { + std::stringstream ss; +#if (ELPP_OS_WINDOWS) + ss << GetCurrentThreadId(); +#endif // (ELPP_OS_WINDOWS) + return ss.str(); +} +#else +/// @brief Gets ID of currently running threading using std::this_thread::get_id() +static std::string +getCurrentThreadId(void) { + std::stringstream ss; + ss << std::this_thread::get_id(); + return ss.str(); +} +#endif // !ELPP_USE_STD_THREADING +#else +static inline std::string +getCurrentThreadId(void) { + return std::string(); +} +#endif // ELPP_THREADING_ENABLED +} // namespace threading +namespace utils { +class File : base::StaticClass { + public: + /// @brief Creates new out file stream for specified filename. + /// @return Pointer to newly created fstream or nullptr + static base::type::fstream_t* + newFileStream(const std::string& filename); + + /// @brief Gets size of file provided in stream + static std::size_t + getSizeOfFile(base::type::fstream_t* fs); + + /// @brief Determines whether or not provided path exist in current file system + static bool + pathExists(const char* path, bool considerFile = false); + + /// @brief Creates specified path on file system + /// @param path Path to create. + static bool + createPath(const std::string& path); + /// @brief Extracts path of filename with leading slash + static std::string + extractPathFromFilename(const std::string& fullPath, const char* seperator = base::consts::kFilePathSeperator); + /// @brief builds stripped filename and puts it in buff + static void + buildStrippedFilename(const char* filename, char buff[], + std::size_t limit = base::consts::kSourceFilenameMaxLength); + /// @brief builds base filename and puts it in buff + static void + buildBaseFilename(const std::string& fullPath, char buff[], + std::size_t limit = base::consts::kSourceFilenameMaxLength, + const char* seperator = base::consts::kFilePathSeperator); +}; +/// @brief String utilities helper class used internally. You should not use it. +class Str : base::StaticClass { + public: + /// @brief Checks if character is digit. Dont use libc implementation of it to prevent locale issues. + static inline bool + isDigit(char c) { + return c >= '0' && c <= '9'; + } + + /// @brief Matches wildcards, '*' and '?' only supported. + static bool + wildCardMatch(const char* str, const char* pattern); + + static std::string& + ltrim(std::string& str); + static std::string& + rtrim(std::string& str); + static std::string& + trim(std::string& str); + + /// @brief Determines whether or not str starts with specified string + /// @param str String to check + /// @param start String to check against + /// @return Returns true if starts with specified string, false otherwise + static bool + startsWith(const std::string& str, const std::string& start); + + /// @brief Determines whether or not str ends with specified string + /// @param str String to check + /// @param end String to check against + /// @return Returns true if ends with specified string, false otherwise + static bool + endsWith(const std::string& str, const std::string& end); + + /// @brief Replaces all instances of replaceWhat with 'replaceWith'. Original variable is changed for performance. + /// @param [in,out] str String to replace from + /// @param replaceWhat Character to replace + /// @param replaceWith Character to replace with + /// @return Modified version of str + static std::string& + replaceAll(std::string& str, char replaceWhat, char replaceWith); + + /// @brief Replaces all instances of 'replaceWhat' with 'replaceWith'. (String version) Replaces in place + /// @param str String to replace from + /// @param replaceWhat Character to replace + /// @param replaceWith Character to replace with + /// @return Modified (original) str + static std::string& + replaceAll(std::string& str, const std::string& replaceWhat, const std::string& replaceWith); + + static void + replaceFirstWithEscape(base::type::string_t& str, const base::type::string_t& replaceWhat, + const base::type::string_t& replaceWith); +#if defined(ELPP_UNICODE) + static void + replaceFirstWithEscape(base::type::string_t& str, const base::type::string_t& replaceWhat, + const std::string& replaceWith); +#endif // defined(ELPP_UNICODE) + /// @brief Converts string to uppercase + /// @param str String to convert + /// @return Uppercase string + static std::string& + toUpper(std::string& str); + + /// @brief Compares cstring equality - uses strcmp + static bool + cStringEq(const char* s1, const char* s2); + + /// @brief Compares cstring equality (case-insensitive) - uses toupper(char) + /// Dont use strcasecmp because of CRT (VC++) + static bool + cStringCaseEq(const char* s1, const char* s2); + + /// @brief Returns true if c exist in str + static bool + contains(const char* str, char c); + + static char* + convertAndAddToBuff(std::size_t n, int len, char* buf, const char* bufLim, bool zeroPadded = true); + static char* + addToBuff(const char* str, char* buf, const char* bufLim); + static char* + clearBuff(char buff[], std::size_t lim); + + /// @brief Converst wchar* to char* + /// NOTE: Need to free return value after use! + static char* + wcharPtrToCharPtr(const wchar_t* line); +}; +/// @brief Operating System helper static class used internally. You should not use it. +class OS : base::StaticClass { + public: +#if ELPP_OS_WINDOWS + /// @brief Gets environment variables for Windows based OS. + /// We are not using getenv(const char*) because of CRT deprecation + /// @param varname Variable name to get environment variable value for + /// @return If variable exist the value of it otherwise nullptr + static const char* + getWindowsEnvironmentVariable(const char* varname); +#endif // ELPP_OS_WINDOWS +#if ELPP_OS_ANDROID + /// @brief Reads android property value + static std::string + getProperty(const char* prop); + + /// @brief Reads android device name + static std::string + getDeviceName(void); +#endif // ELPP_OS_ANDROID + + /// @brief Runs command on terminal and returns the output. + /// + /// @detail This is applicable only on unix based systems, for all other OS, an empty string is returned. + /// @param command Bash command + /// @return Result of bash output or empty string if no result found. + static const std::string + getBashOutput(const char* command); + + /// @brief Gets environment variable. This is cross-platform and CRT safe (for VC++) + /// @param variableName Environment variable name + /// @param defaultVal If no environment variable or value found the value to return by default + /// @param alternativeBashCommand If environment variable not found what would be alternative bash command + /// in order to look for value user is looking for. E.g, for 'user' alternative command will 'whoami' + static std::string + getEnvironmentVariable(const char* variableName, const char* defaultVal, + const char* alternativeBashCommand = nullptr); + /// @brief Gets current username. + static std::string + currentUser(void); + + /// @brief Gets current host name or computer name. + /// + /// @detail For android systems this is device name with its manufacturer and model seperated by hyphen + static std::string + currentHost(void); + /// @brief Whether or not terminal supports colors + static bool + termSupportsColor(void); +}; +/// @brief Contains utilities for cross-platform date/time. This class make use of el::base::utils::Str +class DateTime : base::StaticClass { + public: + /// @brief Cross platform gettimeofday for Windows and unix platform. This can be used to determine current + /// microsecond. + /// + /// @detail For unix system it uses gettimeofday(timeval*, timezone*) and for Windows, a seperate implementation is + /// provided + /// @param [in,out] tv Pointer that gets updated + static void + gettimeofday(struct timeval* tv); + + /// @brief Gets current date and time with a subsecond part. + /// @param format User provided date/time format + /// @param ssPrec A pointer to base::SubsecondPrecision from configuration (non-null) + /// @returns string based date time in specified format. + static std::string + getDateTime(const char* format, const base::SubsecondPrecision* ssPrec); + + /// @brief Converts timeval (struct from ctime) to string using specified format and subsecond precision + static std::string + timevalToString(struct timeval tval, const char* format, const el::base::SubsecondPrecision* ssPrec); + + /// @brief Formats time to get unit accordingly, units like second if > 1000 or minutes if > 60000 etc + static base::type::string_t + formatTime(unsigned long long time, base::TimestampUnit timestampUnit); + + /// @brief Gets time difference in milli/micro second depending on timestampUnit + static unsigned long long + getTimeDifference(const struct timeval& endTime, const struct timeval& startTime, + base::TimestampUnit timestampUnit); + + static struct ::tm* + buildTimeInfo(struct timeval* currTime, struct ::tm* timeInfo); + + private: + static char* + parseFormat(char* buf, std::size_t bufSz, const char* format, const struct tm* tInfo, std::size_t msec, + const base::SubsecondPrecision* ssPrec); +}; +/// @brief Command line arguments for application if specified using el::Helpers::setArgs(..) or START_EASYLOGGINGPP(..) +class CommandLineArgs { + public: + CommandLineArgs(void) { + setArgs(0, static_cast(nullptr)); + } + CommandLineArgs(int argc, const char** argv) { + setArgs(argc, argv); + } + CommandLineArgs(int argc, char** argv) { + setArgs(argc, argv); + } + virtual ~CommandLineArgs(void) { + } + /// @brief Sets arguments and parses them + inline void + setArgs(int argc, const char** argv) { + setArgs(argc, const_cast(argv)); + } + /// @brief Sets arguments and parses them + void + setArgs(int argc, char** argv); + /// @brief Returns true if arguments contain paramKey with a value (seperated by '=') + bool + hasParamWithValue(const char* paramKey) const; + /// @brief Returns value of arguments + /// @see hasParamWithValue(const char*) + const char* + getParamValue(const char* paramKey) const; + /// @brief Return true if arguments has a param (not having a value) i,e without '=' + bool + hasParam(const char* paramKey) const; + /// @brief Returns true if no params available. This exclude argv[0] + bool + empty(void) const; + /// @brief Returns total number of arguments. This exclude argv[0] + std::size_t + size(void) const; + friend base::type::ostream_t& + operator<<(base::type::ostream_t& os, const CommandLineArgs& c); + + private: + int m_argc; + char** m_argv; + std::unordered_map m_paramsWithValue; + std::vector m_params; +}; +/// @brief Abstract registry (aka repository) that provides basic interface for pointer repository specified by T_Ptr +/// type. +/// +/// @detail Most of the functions are virtual final methods but anything implementing this abstract class should +/// implement unregisterAll() and deepCopy(const AbstractRegistry&) and write registerNew() method +/// according to container and few more methods; get() to find element, unregister() to unregister single entry. Please +/// note that this is thread-unsafe and should also implement thread-safety mechanisms in implementation. +template +class AbstractRegistry : public base::threading::ThreadSafe { + public: + typedef typename Container::iterator iterator; + typedef typename Container::const_iterator const_iterator; + + /// @brief Default constructor + AbstractRegistry(void) { + } + + /// @brief Move constructor that is useful for base classes + AbstractRegistry(AbstractRegistry&& sr) { + if (this == &sr) { + return; + } + unregisterAll(); + m_list = std::move(sr.m_list); + } + + bool + operator==(const AbstractRegistry& other) { + if (size() != other.size()) { + return false; + } + for (std::size_t i = 0; i < m_list.size(); ++i) { + if (m_list.at(i) != other.m_list.at(i)) { + return false; + } + } + return true; + } + + bool + operator!=(const AbstractRegistry& other) { + if (size() != other.size()) { + return true; + } + for (std::size_t i = 0; i < m_list.size(); ++i) { + if (m_list.at(i) != other.m_list.at(i)) { + return true; + } + } + return false; + } + + /// @brief Assignment move operator + AbstractRegistry& + operator=(AbstractRegistry&& sr) { + if (this == &sr) { + return *this; + } + unregisterAll(); + m_list = std::move(sr.m_list); + return *this; + } + + virtual ~AbstractRegistry(void) { + } + + /// @return Iterator pointer from start of repository + virtual inline iterator + begin(void) ELPP_FINAL { + return m_list.begin(); + } + + /// @return Iterator pointer from end of repository + virtual inline iterator + end(void) ELPP_FINAL { + return m_list.end(); + } + + /// @return Constant iterator pointer from start of repository + virtual inline const_iterator + cbegin(void) const ELPP_FINAL { + return m_list.cbegin(); + } + + /// @return End of repository + virtual inline const_iterator + cend(void) const ELPP_FINAL { + return m_list.cend(); + } + + /// @return Whether or not repository is empty + virtual inline bool + empty(void) const ELPP_FINAL { + return m_list.empty(); + } + + /// @return Size of repository + virtual inline std::size_t + size(void) const ELPP_FINAL { + return m_list.size(); + } + + /// @brief Returns underlying container by reference + virtual inline Container& + list(void) ELPP_FINAL { + return m_list; + } + + /// @brief Returns underlying container by constant reference. + virtual inline const Container& + list(void) const ELPP_FINAL { + return m_list; + } + + /// @brief Unregisters all the pointers from current repository. + virtual void + unregisterAll(void) = 0; + + protected: + virtual void + deepCopy(const AbstractRegistry&) = 0; + void + reinitDeepCopy(const AbstractRegistry& sr) { + unregisterAll(); + deepCopy(sr); + } + + private: + Container m_list; +}; + +/// @brief A pointer registry mechanism to manage memory and provide search functionalities. (non-predicate version) +/// +/// @detail NOTE: This is thread-unsafe implementation (although it contains lock function, it does not use these +/// functions) +/// of AbstractRegistry. Any implementation of this class should be +/// explicitly (by using lock functions) +template +class Registry : public AbstractRegistry> { + public: + typedef typename Registry::iterator iterator; + typedef typename Registry::const_iterator const_iterator; + + Registry(void) { + } + + /// @brief Copy constructor that is useful for base classes. Try to avoid this constructor, use move constructor. + Registry(const Registry& sr) : AbstractRegistry>() { + if (this == &sr) { + return; + } + this->reinitDeepCopy(sr); + } + + /// @brief Assignment operator that unregisters all the existing registeries and deeply copies each of repo element + /// @see unregisterAll() + /// @see deepCopy(const AbstractRegistry&) + Registry& + operator=(const Registry& sr) { + if (this == &sr) { + return *this; + } + this->reinitDeepCopy(sr); + return *this; + } + + virtual ~Registry(void) { + unregisterAll(); + } + + protected: + virtual void + unregisterAll(void) ELPP_FINAL { + if (!this->empty()) { + for (auto&& curr : this->list()) { + base::utils::safeDelete(curr.second); + } + this->list().clear(); + } + } + + /// @brief Registers new registry to repository. + virtual void + registerNew(const T_Key& uniqKey, T_Ptr* ptr) ELPP_FINAL { + unregister(uniqKey); + this->list().insert(std::make_pair(uniqKey, ptr)); + } + + /// @brief Unregisters single entry mapped to specified unique key + void + unregister(const T_Key& uniqKey) { + T_Ptr* existing = get(uniqKey); + if (existing != nullptr) { + this->list().erase(uniqKey); + base::utils::safeDelete(existing); + } + } + + /// @brief Gets pointer from repository. If none found, nullptr is returned. + T_Ptr* + get(const T_Key& uniqKey) { + iterator it = this->list().find(uniqKey); + return it == this->list().end() ? nullptr : it->second; + } + + private: + virtual void + deepCopy(const AbstractRegistry>& sr) ELPP_FINAL { + for (const_iterator it = sr.cbegin(); it != sr.cend(); ++it) { + registerNew(it->first, new T_Ptr(*it->second)); + } + } +}; + +/// @brief A pointer registry mechanism to manage memory and provide search functionalities. (predicate version) +/// +/// @detail NOTE: This is thread-unsafe implementation of AbstractRegistry. Any implementation of this +/// class should be made thread-safe explicitly +template +class RegistryWithPred : public AbstractRegistry> { + public: + typedef typename RegistryWithPred::iterator iterator; + typedef typename RegistryWithPred::const_iterator const_iterator; + + RegistryWithPred(void) { + } + + virtual ~RegistryWithPred(void) { + unregisterAll(); + } + + /// @brief Copy constructor that is useful for base classes. Try to avoid this constructor, use move constructor. + RegistryWithPred(const RegistryWithPred& sr) : AbstractRegistry>() { + if (this == &sr) { + return; + } + this->reinitDeepCopy(sr); + } + + /// @brief Assignment operator that unregisters all the existing registeries and deeply copies each of repo element + /// @see unregisterAll() + /// @see deepCopy(const AbstractRegistry&) + RegistryWithPred& + operator=(const RegistryWithPred& sr) { + if (this == &sr) { + return *this; + } + this->reinitDeepCopy(sr); + return *this; + } + + friend base::type::ostream_t& + operator<<(base::type::ostream_t& os, const RegistryWithPred& sr) { + for (const_iterator it = sr.list().begin(); it != sr.list().end(); ++it) { + os << ELPP_LITERAL(" ") << **it << ELPP_LITERAL("\n"); + } + return os; + } + + protected: + virtual void + unregisterAll(void) ELPP_FINAL { + if (!this->empty()) { + for (auto&& curr : this->list()) { + base::utils::safeDelete(curr); + } + this->list().clear(); + } + } + + virtual void + unregister(T_Ptr*& ptr) ELPP_FINAL { + if (ptr) { + iterator iter = this->begin(); + for (; iter != this->end(); ++iter) { + if (ptr == *iter) { + break; + } + } + if (iter != this->end() && *iter != nullptr) { + this->list().erase(iter); + base::utils::safeDelete(*iter); + } + } + } + + virtual inline void + registerNew(T_Ptr* ptr) ELPP_FINAL { + this->list().push_back(ptr); + } + + /// @brief Gets pointer from repository with speicifed arguments. Arguments are passed to predicate + /// in order to validate pointer. + template + T_Ptr* + get(const T& arg1, const T2 arg2) { + iterator iter = std::find_if(this->list().begin(), this->list().end(), Pred(arg1, arg2)); + if (iter != this->list().end() && *iter != nullptr) { + return *iter; + } + return nullptr; + } + + private: + virtual void + deepCopy(const AbstractRegistry>& sr) { + for (const_iterator it = sr.list().begin(); it != sr.list().end(); ++it) { + registerNew(new T_Ptr(**it)); + } + } +}; +class Utils { + public: + template + static bool + installCallback(const std::string& id, std::unordered_map* mapT) { + if (mapT->find(id) == mapT->end()) { + mapT->insert(std::make_pair(id, TPtr(new T()))); + return true; + } + return false; + } + + template + static void + uninstallCallback(const std::string& id, std::unordered_map* mapT) { + if (mapT->find(id) != mapT->end()) { + mapT->erase(id); + } + } + + template + static T* + callback(const std::string& id, std::unordered_map* mapT) { + typename std::unordered_map::iterator iter = mapT->find(id); + if (iter != mapT->end()) { + return static_cast(iter->second.get()); + } + return nullptr; + } +}; +} // namespace utils +} // namespace base +/// @brief Base of Easylogging++ friendly class +/// +/// @detail After inheriting this class publicly, implement pure-virtual function `void log(std::ostream&) const` +class Loggable { + public: + virtual ~Loggable(void) { + } + virtual void + log(el::base::type::ostream_t&) const = 0; + + private: + friend inline el::base::type::ostream_t& + operator<<(el::base::type::ostream_t& os, const Loggable& loggable) { + loggable.log(os); + return os; + } +}; +namespace base { +/// @brief Represents log format containing flags and date format. This is used internally to start initial log +class LogFormat : public Loggable { + public: + LogFormat(void); + LogFormat(Level level, const base::type::string_t& format); + LogFormat(const LogFormat& logFormat); + LogFormat(LogFormat&& logFormat); + LogFormat& + operator=(const LogFormat& logFormat); + virtual ~LogFormat(void) { + } + bool + operator==(const LogFormat& other); + + /// @brief Updates format to be used while logging. + /// @param userFormat User provided format + void + parseFromFormat(const base::type::string_t& userFormat); + + inline Level + level(void) const { + return m_level; + } + + inline const base::type::string_t& + userFormat(void) const { + return m_userFormat; + } + + inline const base::type::string_t& + format(void) const { + return m_format; + } + + inline const std::string& + dateTimeFormat(void) const { + return m_dateTimeFormat; + } + + inline base::type::EnumType + flags(void) const { + return m_flags; + } + + inline bool + hasFlag(base::FormatFlags flag) const { + return base::utils::hasFlag(flag, m_flags); + } + + virtual void + log(el::base::type::ostream_t& os) const { + os << m_format; + } + + protected: + /// @brief Updates date time format if available in currFormat. + /// @param index Index where %datetime, %date or %time was found + /// @param [in,out] currFormat current format that is being used to format + virtual void + updateDateFormat(std::size_t index, base::type::string_t& currFormat) ELPP_FINAL; + + /// @brief Updates %level from format. This is so that we dont have to do it at log-writing-time. It uses m_format + /// and m_level + virtual void + updateFormatSpec(void) ELPP_FINAL; + + inline void + addFlag(base::FormatFlags flag) { + base::utils::addFlag(flag, &m_flags); + } + + private: + Level m_level; + base::type::string_t m_userFormat; + base::type::string_t m_format; + std::string m_dateTimeFormat; + base::type::EnumType m_flags; + std::string m_currentUser; + std::string m_currentHost; + friend class el::Logger; // To resolve loggerId format specifier easily +}; +} // namespace base +/// @brief Resolving function for format specifier +typedef std::function FormatSpecifierValueResolver; +/// @brief User-provided custom format specifier +/// @see el::Helpers::installCustomFormatSpecifier +/// @see FormatSpecifierValueResolver +class CustomFormatSpecifier { + public: + CustomFormatSpecifier(const char* formatSpecifier, const FormatSpecifierValueResolver& resolver) + : m_formatSpecifier(formatSpecifier), m_resolver(resolver) { + } + inline const char* + formatSpecifier(void) const { + return m_formatSpecifier; + } + inline const FormatSpecifierValueResolver& + resolver(void) const { + return m_resolver; + } + inline bool + operator==(const char* formatSpecifier) { + return strcmp(m_formatSpecifier, formatSpecifier) == 0; + } + + private: + const char* m_formatSpecifier; + FormatSpecifierValueResolver m_resolver; +}; +/// @brief Represents single configuration that has representing level, configuration type and a string based value. +/// +/// @detail String based value means any value either its boolean, integer or string itself, it will be embedded inside +/// quotes and will be parsed later. +/// +/// Consider some examples below: +/// * el::Configuration confEnabledInfo(el::Level::Info, el::ConfigurationType::Enabled, "true"); +/// * el::Configuration confMaxLogFileSizeInfo(el::Level::Info, el::ConfigurationType::MaxLogFileSize, "2048"); +/// * el::Configuration confFilenameInfo(el::Level::Info, el::ConfigurationType::Filename, "/var/log/my.log"); +class Configuration : public Loggable { + public: + Configuration(const Configuration& c); + Configuration& + operator=(const Configuration& c); + + virtual ~Configuration(void) { + } + + /// @brief Full constructor used to sets value of configuration + Configuration(Level level, ConfigurationType configurationType, const std::string& value); + + /// @brief Gets level of current configuration + inline Level + level(void) const { + return m_level; + } + + /// @brief Gets configuration type of current configuration + inline ConfigurationType + configurationType(void) const { + return m_configurationType; + } + + /// @brief Gets string based configuration value + inline const std::string& + value(void) const { + return m_value; + } + + /// @brief Set string based configuration value + /// @param value Value to set. Values have to be std::string; For boolean values use "true", "false", for any + /// integral values + /// use them in quotes. They will be parsed when configuring + inline void + setValue(const std::string& value) { + m_value = value; + } + + virtual void + log(el::base::type::ostream_t& os) const; + + /// @brief Used to find configuration from configuration (pointers) repository. Avoid using it. + class Predicate { + public: + Predicate(Level level, ConfigurationType configurationType); + + bool + operator()(const Configuration* conf) const; + + private: + Level m_level; + ConfigurationType m_configurationType; + }; + + private: + Level m_level; + ConfigurationType m_configurationType; + std::string m_value; +}; + +/// @brief Thread-safe Configuration repository +/// +/// @detail This repository represents configurations for all the levels and configuration type mapped to a value. +class Configurations : public base::utils::RegistryWithPred { + public: + /// @brief Default constructor with empty repository + Configurations(void); + + /// @brief Constructor used to set configurations using configuration file. + /// @param configurationFile Full path to configuration file + /// @param useDefaultsForRemaining Lets you set the remaining configurations to default. + /// @param base If provided, this configuration will be based off existing repository that this argument is pointing + /// to. + /// @see parseFromFile(const std::string&, Configurations* base) + /// @see setRemainingToDefault() + Configurations(const std::string& configurationFile, bool useDefaultsForRemaining = true, + Configurations* base = nullptr); + + virtual ~Configurations(void) { + } + + /// @brief Parses configuration from file. + /// @param configurationFile Full path to configuration file + /// @param base Configurations to base new configuration repository off. This value is used when you want to use + /// existing Configurations to base all the values and then set rest of configuration via configuration file. + /// @return True if successfully parsed, false otherwise. You may define 'ELPP_DEBUG_ASSERT_FAILURE' to make sure + /// you + /// do not proceed without successful parse. + bool + parseFromFile(const std::string& configurationFile, Configurations* base = nullptr); + + /// @brief Parse configurations from configuration string. + /// + /// @detail This configuration string has same syntax as configuration file contents. Make sure all the necessary + /// new line characters are provided. + /// @param base Configurations to base new configuration repository off. This value is used when you want to use + /// existing Configurations to base all the values and then set rest of configuration via configuration text. + /// @return True if successfully parsed, false otherwise. You may define 'ELPP_DEBUG_ASSERT_FAILURE' to make sure + /// you + /// do not proceed without successful parse. + bool + parseFromText(const std::string& configurationsString, Configurations* base = nullptr); + + /// @brief Sets configuration based-off an existing configurations. + /// @param base Pointer to existing configurations. + void + setFromBase(Configurations* base); + + /// @brief Determines whether or not specified configuration type exists in the repository. + /// + /// @detail Returns as soon as first level is found. + /// @param configurationType Type of configuration to check existence for. + bool + hasConfiguration(ConfigurationType configurationType); + + /// @brief Determines whether or not specified configuration type exists for specified level + /// @param level Level to check + /// @param configurationType Type of configuration to check existence for. + bool + hasConfiguration(Level level, ConfigurationType configurationType); + + /// @brief Sets value of configuration for specified level. + /// + /// @detail Any existing configuration for specified level will be replaced. Also note that configuration types + /// ConfigurationType::SubsecondPrecision and ConfigurationType::PerformanceTracking will be ignored if not set for + /// Level::Global because these configurations are not dependant on level. + /// @param level Level to set configuration for (el::Level). + /// @param configurationType Type of configuration (el::ConfigurationType) + /// @param value A string based value. Regardless of what the data type of configuration is, it will always be + /// string from users' point of view. This is then parsed later to be used internally. + /// @see Configuration::setValue(const std::string& value) + /// @see el::Level + /// @see el::ConfigurationType + void + set(Level level, ConfigurationType configurationType, const std::string& value); + + /// @brief Sets single configuration based on other single configuration. + /// @see set(Level level, ConfigurationType configurationType, const std::string& value) + void + set(Configuration* conf); + + inline Configuration* + get(Level level, ConfigurationType configurationType) { + base::threading::ScopedLock scopedLock(lock()); + return RegistryWithPred::get(level, configurationType); + } + + /// @brief Sets configuration for all levels. + /// @param configurationType Type of configuration + /// @param value String based value + /// @see Configurations::set(Level level, ConfigurationType configurationType, const std::string& value) + inline void + setGlobally(ConfigurationType configurationType, const std::string& value) { + setGlobally(configurationType, value, false); + } + + /// @brief Clears repository so that all the configurations are unset + inline void + clear(void) { + base::threading::ScopedLock scopedLock(lock()); + unregisterAll(); + } + + /// @brief Gets configuration file used in parsing this configurations. + /// + /// @detail If this repository was set manually or by text this returns empty string. + inline const std::string& + configurationFile(void) const { + return m_configurationFile; + } + + /// @brief Sets configurations to "factory based" configurations. + void + setToDefault(void); + + /// @brief Lets you set the remaining configurations to default. + /// + /// @detail By remaining, it means that the level/type a configuration does not exist for. + /// This function is useful when you want to minimize chances of failures, e.g, if you have a configuration file + /// that sets configuration for all the configurations except for Enabled or not, we use this so that ENABLED is set + /// to default i.e, true. If you dont do this explicitly (either by calling this function or by using second param + /// in Constructor and try to access a value, an error is thrown + void + setRemainingToDefault(void); + + /// @brief Parser used internally to parse configurations from file or text. + /// + /// @detail This class makes use of base::utils::Str. + /// You should not need this unless you are working on some tool for Easylogging++ + class Parser : base::StaticClass { + public: + /// @brief Parses configuration from file. + /// @param configurationFile Full path to configuration file + /// @param sender Sender configurations pointer. Usually 'this' is used from calling class + /// @param base Configurations to base new configuration repository off. This value is used when you want to use + /// existing Configurations to base all the values and then set rest of configuration via configuration + /// file. + /// @return True if successfully parsed, false otherwise. You may define '_STOP_ON_FIRSTELPP_ASSERTION' to make + /// sure you + /// do not proceed without successful parse. + static bool + parseFromFile(const std::string& configurationFile, Configurations* sender, Configurations* base = nullptr); + + /// @brief Parse configurations from configuration string. + /// + /// @detail This configuration string has same syntax as configuration file contents. Make sure all the + /// necessary new line characters are provided. You may define '_STOP_ON_FIRSTELPP_ASSERTION' to make sure you + /// do not proceed without successful parse (This is recommended) + /// @param configurationsString the configuration in plain text format + /// @param sender Sender configurations pointer. Usually 'this' is used from calling class + /// @param base Configurations to base new configuration repository off. This value is used when you want to use + /// existing Configurations to base all the values and then set rest of configuration via configuration + /// text. + /// @return True if successfully parsed, false otherwise. + static bool + parseFromText(const std::string& configurationsString, Configurations* sender, Configurations* base = nullptr); + + private: + friend class el::Loggers; + static void + ignoreComments(std::string* line); + static bool + isLevel(const std::string& line); + static bool + isComment(const std::string& line); + static inline bool + isConfig(const std::string& line); + static bool + parseLine(std::string* line, std::string* currConfigStr, std::string* currLevelStr, Level* currLevel, + Configurations* conf); + }; + + private: + std::string m_configurationFile; + bool m_isFromFile; + friend class el::Loggers; + + /// @brief Unsafely sets configuration if does not already exist + void + unsafeSetIfNotExist(Level level, ConfigurationType configurationType, const std::string& value); + + /// @brief Thread unsafe set + void + unsafeSet(Level level, ConfigurationType configurationType, const std::string& value); + + /// @brief Sets configurations for all levels including Level::Global if includeGlobalLevel is true + /// @see Configurations::setGlobally(ConfigurationType configurationType, const std::string& value) + void + setGlobally(ConfigurationType configurationType, const std::string& value, bool includeGlobalLevel); + + /// @brief Sets configurations (Unsafely) for all levels including Level::Global if includeGlobalLevel is true + /// @see Configurations::setGlobally(ConfigurationType configurationType, const std::string& value) + void + unsafeSetGlobally(ConfigurationType configurationType, const std::string& value, bool includeGlobalLevel); +}; + +namespace base { +typedef std::shared_ptr FileStreamPtr; +typedef std::unordered_map LogStreamsReferenceMap; +/// @brief Configurations with data types. +/// +/// @detail el::Configurations have string based values. This is whats used internally in order to read correct +/// configurations. This is to perform faster while writing logs using correct configurations. +/// +/// This is thread safe and final class containing non-virtual destructor (means nothing should inherit this class) +class TypedConfigurations : public base::threading::ThreadSafe { + public: + /// @brief Constructor to initialize (construct) the object off el::Configurations + /// @param configurations Configurations pointer/reference to base this typed configurations off. + /// @param logStreamsReference Use ELPP->registeredLoggers()->logStreamsReference() + TypedConfigurations(Configurations* configurations, base::LogStreamsReferenceMap* logStreamsReference); + + TypedConfigurations(const TypedConfigurations& other); + + virtual ~TypedConfigurations(void) { + } + + const Configurations* + configurations(void) const { + return m_configurations; + } + + bool + enabled(Level level); + bool + toFile(Level level); + const std::string& + filename(Level level); + bool + toStandardOutput(Level level); + const base::LogFormat& + logFormat(Level level); + const base::SubsecondPrecision& + subsecondPrecision(Level level = Level::Global); + const base::MillisecondsWidth& + millisecondsWidth(Level level = Level::Global); + bool + performanceTracking(Level level = Level::Global); + base::type::fstream_t* + fileStream(Level level); + std::size_t + maxLogFileSize(Level level); + std::size_t + logFlushThreshold(Level level); + + private: + Configurations* m_configurations; + std::unordered_map m_enabledMap; + std::unordered_map m_toFileMap; + std::unordered_map m_filenameMap; + std::unordered_map m_toStandardOutputMap; + std::unordered_map m_logFormatMap; + std::unordered_map m_subsecondPrecisionMap; + std::unordered_map m_performanceTrackingMap; + std::unordered_map m_fileStreamMap; + std::unordered_map m_maxLogFileSizeMap; + std::unordered_map m_logFlushThresholdMap; + base::LogStreamsReferenceMap* m_logStreamsReference; + + friend class el::Helpers; + friend class el::base::MessageBuilder; + friend class el::base::Writer; + friend class el::base::DefaultLogDispatchCallback; + friend class el::base::LogDispatcher; + + template + inline Conf_T + getConfigByVal(Level level, const std::unordered_map* confMap, const char* confName) { + base::threading::ScopedLock scopedLock(lock()); + return unsafeGetConfigByVal(level, confMap, confName); // This is not unsafe anymore - mutex locked in scope + } + + template + inline Conf_T& + getConfigByRef(Level level, std::unordered_map* confMap, const char* confName) { + base::threading::ScopedLock scopedLock(lock()); + return unsafeGetConfigByRef(level, confMap, confName); // This is not unsafe anymore - mutex locked in scope + } + + template + Conf_T + unsafeGetConfigByVal(Level level, const std::unordered_map* confMap, const char* confName) { + ELPP_UNUSED(confName); + typename std::unordered_map::const_iterator it = confMap->find(level); + if (it == confMap->end()) { + try { + return confMap->at(Level::Global); + } catch (...) { + ELPP_INTERNAL_ERROR("Unable to get configuration [" + << confName << "] for level [" << LevelHelper::convertToString(level) << "]" + << std::endl + << "Please ensure you have properly configured logger.", + false); + return Conf_T(); + } + } + return it->second; + } + + template + Conf_T& + unsafeGetConfigByRef(Level level, std::unordered_map* confMap, const char* confName) { + ELPP_UNUSED(confName); + typename std::unordered_map::iterator it = confMap->find(level); + if (it == confMap->end()) { + try { + return confMap->at(Level::Global); + } catch (...) { + ELPP_INTERNAL_ERROR("Unable to get configuration [" + << confName << "] for level [" << LevelHelper::convertToString(level) << "]" + << std::endl + << "Please ensure you have properly configured logger.", + false); + } + } + return it->second; + } + + template + void + setValue(Level level, const Conf_T& value, std::unordered_map* confMap, + bool includeGlobalLevel = true) { + // If map is empty and we are allowed to add into generic level (Level::Global), do it! + if (confMap->empty() && includeGlobalLevel) { + confMap->insert(std::make_pair(Level::Global, value)); + return; + } + // If same value exist in generic level already, dont add it to explicit level + typename std::unordered_map::iterator it = confMap->find(Level::Global); + if (it != confMap->end() && it->second == value) { + return; + } + // Now make sure we dont double up values if we really need to add it to explicit level + it = confMap->find(level); + if (it == confMap->end()) { + // Value not found for level, add new + confMap->insert(std::make_pair(level, value)); + } else { + // Value found, just update value + confMap->at(level) = value; + } + } + + void + build(Configurations* configurations); + unsigned long + getULong(std::string confVal); + std::string + resolveFilename(const std::string& filename); + void + insertFile(Level level, const std::string& fullFilename); + bool + unsafeValidateFileRolling(Level level, const PreRollOutCallback& preRollOutCallback); + + inline bool + validateFileRolling(Level level, const PreRollOutCallback& preRollOutCallback) { + base::threading::ScopedLock scopedLock(lock()); + return unsafeValidateFileRolling(level, preRollOutCallback); + } +}; +/// @brief Class that keeps record of current line hit for occasional logging +class HitCounter { + public: + HitCounter(void) : m_filename(""), m_lineNumber(0), m_hitCounts(0) { + } + + HitCounter(const char* filename, base::type::LineNumber lineNumber) + : m_filename(filename), m_lineNumber(lineNumber), m_hitCounts(0) { + } + + HitCounter(const HitCounter& hitCounter) + : m_filename(hitCounter.m_filename), + m_lineNumber(hitCounter.m_lineNumber), + m_hitCounts(hitCounter.m_hitCounts) { + } + + HitCounter& + operator=(const HitCounter& hitCounter) { + if (&hitCounter != this) { + m_filename = hitCounter.m_filename; + m_lineNumber = hitCounter.m_lineNumber; + m_hitCounts = hitCounter.m_hitCounts; + } + return *this; + } + + virtual ~HitCounter(void) { + } + + /// @brief Resets location of current hit counter + inline void + resetLocation(const char* filename, base::type::LineNumber lineNumber) { + m_filename = filename; + m_lineNumber = lineNumber; + } + + /// @brief Validates hit counts and resets it if necessary + inline void + validateHitCounts(std::size_t n) { + if (m_hitCounts >= base::consts::kMaxLogPerCounter) { + m_hitCounts = (n >= 1 ? base::consts::kMaxLogPerCounter % n : 0); + } + ++m_hitCounts; + } + + inline const char* + filename(void) const { + return m_filename; + } + + inline base::type::LineNumber + lineNumber(void) const { + return m_lineNumber; + } + + inline std::size_t + hitCounts(void) const { + return m_hitCounts; + } + + inline void + increment(void) { + ++m_hitCounts; + } + + class Predicate { + public: + Predicate(const char* filename, base::type::LineNumber lineNumber) + : m_filename(filename), m_lineNumber(lineNumber) { + } + inline bool + operator()(const HitCounter* counter) { + return ((counter != nullptr) && (strcmp(counter->m_filename, m_filename) == 0) && + (counter->m_lineNumber == m_lineNumber)); + } + + private: + const char* m_filename; + base::type::LineNumber m_lineNumber; + }; + + private: + const char* m_filename; + base::type::LineNumber m_lineNumber; + std::size_t m_hitCounts; +}; +/// @brief Repository for hit counters used across the application +class RegisteredHitCounters : public base::utils::RegistryWithPred { + public: + /// @brief Validates counter for every N, i.e, registers new if does not exist otherwise updates original one + /// @return True if validation resulted in triggering hit. Meaning logs should be written everytime true is returned + bool + validateEveryN(const char* filename, base::type::LineNumber lineNumber, std::size_t n); + + /// @brief Validates counter for hits >= N, i.e, registers new if does not exist otherwise updates original one + /// @return True if validation resulted in triggering hit. Meaning logs should be written everytime true is returned + bool + validateAfterN(const char* filename, base::type::LineNumber lineNumber, std::size_t n); + + /// @brief Validates counter for hits are <= n, i.e, registers new if does not exist otherwise updates original one + /// @return True if validation resulted in triggering hit. Meaning logs should be written everytime true is returned + bool + validateNTimes(const char* filename, base::type::LineNumber lineNumber, std::size_t n); + + /// @brief Gets hit counter registered at specified position + inline const base::HitCounter* + getCounter(const char* filename, base::type::LineNumber lineNumber) { + base::threading::ScopedLock scopedLock(lock()); + return get(filename, lineNumber); + } +}; +/// @brief Action to be taken for dispatching +enum class DispatchAction : base::type::EnumType { None = 1, NormalLog = 2, SysLog = 4 }; +} // namespace base +template +class Callback : protected base::threading::ThreadSafe { + public: + Callback(void) : m_enabled(true) { + } + inline bool + enabled(void) const { + return m_enabled; + } + inline void + setEnabled(bool enabled) { + base::threading::ScopedLock scopedLock(lock()); + m_enabled = enabled; + } + + protected: + virtual void + handle(const T* handlePtr) = 0; + + private: + bool m_enabled; +}; +class LogDispatchData { + public: + LogDispatchData() : m_logMessage(nullptr), m_dispatchAction(base::DispatchAction::None) { + } + inline const LogMessage* + logMessage(void) const { + return m_logMessage; + } + inline base::DispatchAction + dispatchAction(void) const { + return m_dispatchAction; + } + inline void + setLogMessage(LogMessage* logMessage) { + m_logMessage = logMessage; + } + inline void + setDispatchAction(base::DispatchAction dispatchAction) { + m_dispatchAction = dispatchAction; + } + + private: + LogMessage* m_logMessage; + base::DispatchAction m_dispatchAction; + friend class base::LogDispatcher; +}; +class LogDispatchCallback : public Callback { + protected: + virtual void + handle(const LogDispatchData* data); + base::threading::Mutex& + fileHandle(const LogDispatchData* data); + + private: + friend class base::LogDispatcher; + std::unordered_map> m_fileLocks; + base::threading::Mutex m_fileLocksMapLock; +}; +class PerformanceTrackingCallback : public Callback { + private: + friend class base::PerformanceTracker; +}; +class LoggerRegistrationCallback : public Callback { + private: + friend class base::RegisteredLoggers; +}; +class LogBuilder : base::NoCopy { + public: + LogBuilder() : m_termSupportsColor(base::utils::OS::termSupportsColor()) { + } + virtual ~LogBuilder(void) { + ELPP_INTERNAL_INFO(3, "Destroying log builder...") + } + virtual base::type::string_t + build(const LogMessage* logMessage, bool appendNewLine) const = 0; + void + convertToColoredOutput(base::type::string_t* logLine, Level level); + + private: + bool m_termSupportsColor; + friend class el::base::DefaultLogDispatchCallback; +}; +typedef std::shared_ptr LogBuilderPtr; +/// @brief Represents a logger holding ID and configurations we need to write logs +/// +/// @detail This class does not write logs itself instead its used by writer to read configuations from. +class Logger : public base::threading::ThreadSafe, public Loggable { + public: + Logger(const std::string& id, base::LogStreamsReferenceMap* logStreamsReference); + Logger(const std::string& id, const Configurations& configurations, + base::LogStreamsReferenceMap* logStreamsReference); + Logger(const Logger& logger); + Logger& + operator=(const Logger& logger); + + virtual ~Logger(void) { + base::utils::safeDelete(m_typedConfigurations); + } + + virtual inline void + log(el::base::type::ostream_t& os) const { + os << m_id.c_str(); + } + + /// @brief Configures the logger using specified configurations. + void + configure(const Configurations& configurations); + + /// @brief Reconfigures logger using existing configurations + void + reconfigure(void); + + inline const std::string& + id(void) const { + return m_id; + } + + inline const std::string& + parentApplicationName(void) const { + return m_parentApplicationName; + } + + inline void + setParentApplicationName(const std::string& parentApplicationName) { + m_parentApplicationName = parentApplicationName; + } + + inline Configurations* + configurations(void) { + return &m_configurations; + } + + inline base::TypedConfigurations* + typedConfigurations(void) { + return m_typedConfigurations; + } + + static bool + isValidId(const std::string& id); + + /// @brief Flushes logger to sync all log files for all levels + void + flush(void); + + void + flush(Level level, base::type::fstream_t* fs); + + inline bool + isFlushNeeded(Level level) { + return ++m_unflushedCount.find(level)->second >= m_typedConfigurations->logFlushThreshold(level); + } + + inline LogBuilder* + logBuilder(void) const { + return m_logBuilder.get(); + } + + inline void + setLogBuilder(const LogBuilderPtr& logBuilder) { + m_logBuilder = logBuilder; + } + + inline bool + enabled(Level level) const { + return m_typedConfigurations->enabled(level); + } + +#if ELPP_VARIADIC_TEMPLATES_SUPPORTED +#define LOGGER_LEVEL_WRITERS_SIGNATURES(FUNCTION_NAME) \ + template \ + inline void FUNCTION_NAME(const char*, const T&, const Args&...); \ + template \ + inline void FUNCTION_NAME(const T&); + + template + inline void + verbose(int, const char*, const T&, const Args&...); + + template + inline void + verbose(int, const T&); + + LOGGER_LEVEL_WRITERS_SIGNATURES(info) + LOGGER_LEVEL_WRITERS_SIGNATURES(debug) + LOGGER_LEVEL_WRITERS_SIGNATURES(warn) + LOGGER_LEVEL_WRITERS_SIGNATURES(error) + LOGGER_LEVEL_WRITERS_SIGNATURES(fatal) + LOGGER_LEVEL_WRITERS_SIGNATURES(trace) +#undef LOGGER_LEVEL_WRITERS_SIGNATURES +#endif // ELPP_VARIADIC_TEMPLATES_SUPPORTED + private: + std::string m_id; + base::TypedConfigurations* m_typedConfigurations; + base::type::stringstream_t m_stream; + std::string m_parentApplicationName; + bool m_isConfigured; + Configurations m_configurations; + std::unordered_map m_unflushedCount; + base::LogStreamsReferenceMap* m_logStreamsReference; + LogBuilderPtr m_logBuilder; + + friend class el::LogMessage; + friend class el::Loggers; + friend class el::Helpers; + friend class el::base::RegisteredLoggers; + friend class el::base::DefaultLogDispatchCallback; + friend class el::base::MessageBuilder; + friend class el::base::Writer; + friend class el::base::PErrorWriter; + friend class el::base::Storage; + friend class el::base::PerformanceTracker; + friend class el::base::LogDispatcher; + + Logger(void); + +#if ELPP_VARIADIC_TEMPLATES_SUPPORTED + template + void + log_(Level, int, const char*, const T&, const Args&...); + + template + inline void + log_(Level, int, const T&); + + template + void + log(Level, const char*, const T&, const Args&...); + + template + inline void + log(Level, const T&); +#endif // ELPP_VARIADIC_TEMPLATES_SUPPORTED + + void + initUnflushedCount(void); + + inline base::type::stringstream_t& + stream(void) { + return m_stream; + } + + void + resolveLoggerFormatSpec(void) const; +}; +namespace base { +/// @brief Loggers repository +class RegisteredLoggers : public base::utils::Registry { + public: + explicit RegisteredLoggers(const LogBuilderPtr& defaultLogBuilder); + + virtual ~RegisteredLoggers(void) { + unsafeFlushAll(); + } + + inline void + setDefaultConfigurations(const Configurations& configurations) { + base::threading::ScopedLock scopedLock(lock()); + m_defaultConfigurations.setFromBase(const_cast(&configurations)); + } + + inline Configurations* + defaultConfigurations(void) { + return &m_defaultConfigurations; + } + + Logger* + get(const std::string& id, bool forceCreation = true); + + template + inline bool + installLoggerRegistrationCallback(const std::string& id) { + return base::utils::Utils::installCallback( + id, &m_loggerRegistrationCallbacks); + } + + template + inline void + uninstallLoggerRegistrationCallback(const std::string& id) { + base::utils::Utils::uninstallCallback( + id, &m_loggerRegistrationCallbacks); + } + + template + inline T* + loggerRegistrationCallback(const std::string& id) { + return base::utils::Utils::callback( + id, &m_loggerRegistrationCallbacks); + } + + bool + remove(const std::string& id); + + inline bool + has(const std::string& id) { + return get(id, false) != nullptr; + } + + inline void + unregister(Logger*& logger) { + base::threading::ScopedLock scopedLock(lock()); + base::utils::Registry::unregister(logger->id()); + } + + inline base::LogStreamsReferenceMap* + logStreamsReference(void) { + return &m_logStreamsReference; + } + + inline void + flushAll(void) { + base::threading::ScopedLock scopedLock(lock()); + unsafeFlushAll(); + } + + inline void + setDefaultLogBuilder(LogBuilderPtr& logBuilderPtr) { + base::threading::ScopedLock scopedLock(lock()); + m_defaultLogBuilder = logBuilderPtr; + } + + private: + LogBuilderPtr m_defaultLogBuilder; + Configurations m_defaultConfigurations; + base::LogStreamsReferenceMap m_logStreamsReference; + std::unordered_map m_loggerRegistrationCallbacks; + friend class el::base::Storage; + + void + unsafeFlushAll(void); +}; +/// @brief Represents registries for verbose logging +class VRegistry : base::NoCopy, public base::threading::ThreadSafe { + public: + explicit VRegistry(base::type::VerboseLevel level, base::type::EnumType* pFlags); + + /// @brief Sets verbose level. Accepted range is 0-9 + void + setLevel(base::type::VerboseLevel level); + + inline base::type::VerboseLevel + level(void) const { + return m_level; + } + + inline void + clearModules(void) { + base::threading::ScopedLock scopedLock(lock()); + m_modules.clear(); + } + + void + setModules(const char* modules); + + bool + allowed(base::type::VerboseLevel vlevel, const char* file); + + inline const std::unordered_map& + modules(void) const { + return m_modules; + } + + void + setFromArgs(const base::utils::CommandLineArgs* commandLineArgs); + + /// @brief Whether or not vModules enabled + inline bool + vModulesEnabled(void) { + return !base::utils::hasFlag(LoggingFlag::DisableVModules, *m_pFlags); + } + + private: + base::type::VerboseLevel m_level; + base::type::EnumType* m_pFlags; + std::unordered_map m_modules; +}; +} // namespace base +class LogMessage { + public: + LogMessage(Level level, const std::string& file, base::type::LineNumber line, const std::string& func, + base::type::VerboseLevel verboseLevel, Logger* logger) + : m_level(level), + m_file(file), + m_line(line), + m_func(func), + m_verboseLevel(verboseLevel), + m_logger(logger), + m_message(logger->stream().str()) { + } + inline Level + level(void) const { + return m_level; + } + inline const std::string& + file(void) const { + return m_file; + } + inline base::type::LineNumber + line(void) const { + return m_line; + } + inline const std::string& + func(void) const { + return m_func; + } + inline base::type::VerboseLevel + verboseLevel(void) const { + return m_verboseLevel; + } + inline Logger* + logger(void) const { + return m_logger; + } + inline const base::type::string_t& + message(void) const { + return m_message; + } + + private: + Level m_level; + std::string m_file; + base::type::LineNumber m_line; + std::string m_func; + base::type::VerboseLevel m_verboseLevel; + Logger* m_logger; + base::type::string_t m_message; +}; +namespace base { +#if ELPP_ASYNC_LOGGING +class AsyncLogItem { + public: + explicit AsyncLogItem(const LogMessage& logMessage, const LogDispatchData& data, + const base::type::string_t& logLine) + : m_logMessage(logMessage), m_dispatchData(data), m_logLine(logLine) { + } + virtual ~AsyncLogItem() { + } + inline LogMessage* + logMessage(void) { + return &m_logMessage; + } + inline LogDispatchData* + data(void) { + return &m_dispatchData; + } + inline base::type::string_t + logLine(void) { + return m_logLine; + } + + private: + LogMessage m_logMessage; + LogDispatchData m_dispatchData; + base::type::string_t m_logLine; +}; +class AsyncLogQueue : public base::threading::ThreadSafe { + public: + virtual ~AsyncLogQueue() { + ELPP_INTERNAL_INFO(6, "~AsyncLogQueue"); + } + + inline AsyncLogItem + next(void) { + base::threading::ScopedLock scopedLock(lock()); + AsyncLogItem result = m_queue.front(); + m_queue.pop(); + return result; + } + + inline void + push(const AsyncLogItem& item) { + base::threading::ScopedLock scopedLock(lock()); + m_queue.push(item); + } + inline void + pop(void) { + base::threading::ScopedLock scopedLock(lock()); + m_queue.pop(); + } + inline AsyncLogItem + front(void) { + base::threading::ScopedLock scopedLock(lock()); + return m_queue.front(); + } + inline bool + empty(void) { + base::threading::ScopedLock scopedLock(lock()); + return m_queue.empty(); + } + + private: + std::queue m_queue; +}; +class IWorker { + public: + virtual ~IWorker() { + } + virtual void + start() = 0; +}; +#endif // ELPP_ASYNC_LOGGING +/// @brief Easylogging++ management storage +class Storage : base::NoCopy, public base::threading::ThreadSafe { + public: +#if ELPP_ASYNC_LOGGING + Storage(const LogBuilderPtr& defaultLogBuilder, base::IWorker* asyncDispatchWorker); +#else + explicit Storage(const LogBuilderPtr& defaultLogBuilder); +#endif // ELPP_ASYNC_LOGGING + + virtual ~Storage(void); + + inline bool + validateEveryNCounter(const char* filename, base::type::LineNumber lineNumber, std::size_t occasion) { + return hitCounters()->validateEveryN(filename, lineNumber, occasion); + } + + inline bool + validateAfterNCounter(const char* filename, base::type::LineNumber lineNumber, std::size_t n) { + return hitCounters()->validateAfterN(filename, lineNumber, n); + } + + inline bool + validateNTimesCounter(const char* filename, base::type::LineNumber lineNumber, std::size_t n) { + return hitCounters()->validateNTimes(filename, lineNumber, n); + } + + inline base::RegisteredHitCounters* + hitCounters(void) const { + return m_registeredHitCounters; + } + + inline base::RegisteredLoggers* + registeredLoggers(void) const { + return m_registeredLoggers; + } + + inline base::VRegistry* + vRegistry(void) const { + return m_vRegistry; + } + +#if ELPP_ASYNC_LOGGING + inline base::AsyncLogQueue* + asyncLogQueue(void) const { + return m_asyncLogQueue; + } +#endif // ELPP_ASYNC_LOGGING + + inline const base::utils::CommandLineArgs* + commandLineArgs(void) const { + return &m_commandLineArgs; + } + + inline void + addFlag(LoggingFlag flag) { + base::utils::addFlag(flag, &m_flags); + } + + inline void + removeFlag(LoggingFlag flag) { + base::utils::removeFlag(flag, &m_flags); + } + + inline bool + hasFlag(LoggingFlag flag) const { + return base::utils::hasFlag(flag, m_flags); + } + + inline base::type::EnumType + flags(void) const { + return m_flags; + } + + inline void + setFlags(base::type::EnumType flags) { + m_flags = flags; + } + + inline void + setPreRollOutCallback(const PreRollOutCallback& callback) { + m_preRollOutCallback = callback; + } + + inline void + unsetPreRollOutCallback(void) { + m_preRollOutCallback = base::defaultPreRollOutCallback; + } + + inline PreRollOutCallback& + preRollOutCallback(void) { + return m_preRollOutCallback; + } + + bool + hasCustomFormatSpecifier(const char* formatSpecifier); + void + installCustomFormatSpecifier(const CustomFormatSpecifier& customFormatSpecifier); + bool + uninstallCustomFormatSpecifier(const char* formatSpecifier); + + const std::vector* + customFormatSpecifiers(void) const { + return &m_customFormatSpecifiers; + } + + base::threading::Mutex& + customFormatSpecifiersLock() { + return m_customFormatSpecifiersLock; + } + + inline void + setLoggingLevel(Level level) { + m_loggingLevel = level; + } + + template + inline bool + installLogDispatchCallback(const std::string& id) { + return base::utils::Utils::installCallback(id, &m_logDispatchCallbacks); + } + + template + inline void + uninstallLogDispatchCallback(const std::string& id) { + base::utils::Utils::uninstallCallback(id, &m_logDispatchCallbacks); + } + template + inline T* + logDispatchCallback(const std::string& id) { + return base::utils::Utils::callback(id, &m_logDispatchCallbacks); + } + +#if defined(ELPP_FEATURE_ALL) || defined(ELPP_FEATURE_PERFORMANCE_TRACKING) + template + inline bool + installPerformanceTrackingCallback(const std::string& id) { + return base::utils::Utils::installCallback( + id, &m_performanceTrackingCallbacks); + } + + template + inline void + uninstallPerformanceTrackingCallback(const std::string& id) { + base::utils::Utils::uninstallCallback( + id, &m_performanceTrackingCallbacks); + } + + template + inline T* + performanceTrackingCallback(const std::string& id) { + return base::utils::Utils::callback( + id, &m_performanceTrackingCallbacks); + } +#endif // defined(ELPP_FEATURE_ALL) || defined(ELPP_FEATURE_PERFORMANCE_TRACKING) + + /// @brief Sets thread name for current thread. Requires std::thread + inline void + setThreadName(const std::string& name) { + if (name.empty()) + return; + base::threading::ScopedLock scopedLock(m_threadNamesLock); + m_threadNames[base::threading::getCurrentThreadId()] = name; + } + + inline std::string + getThreadName(const std::string& threadId) { + base::threading::ScopedLock scopedLock(m_threadNamesLock); + std::unordered_map::const_iterator it = m_threadNames.find(threadId); + if (it == m_threadNames.end()) { + return threadId; + } + return it->second; + } + + private: + base::RegisteredHitCounters* m_registeredHitCounters; + base::RegisteredLoggers* m_registeredLoggers; + base::type::EnumType m_flags; + base::VRegistry* m_vRegistry; +#if ELPP_ASYNC_LOGGING + base::AsyncLogQueue* m_asyncLogQueue; + base::IWorker* m_asyncDispatchWorker; +#endif // ELPP_ASYNC_LOGGING + base::utils::CommandLineArgs m_commandLineArgs; + PreRollOutCallback m_preRollOutCallback; + std::unordered_map m_logDispatchCallbacks; + std::unordered_map m_performanceTrackingCallbacks; + std::unordered_map m_threadNames; + std::vector m_customFormatSpecifiers; + base::threading::Mutex m_customFormatSpecifiersLock; + base::threading::Mutex m_threadNamesLock; + Level m_loggingLevel; + + friend class el::Helpers; + friend class el::base::DefaultLogDispatchCallback; + friend class el::LogBuilder; + friend class el::base::MessageBuilder; + friend class el::base::Writer; + friend class el::base::PerformanceTracker; + friend class el::base::LogDispatcher; + + void + setApplicationArguments(int argc, char** argv); + + inline void + setApplicationArguments(int argc, const char** argv) { + setApplicationArguments(argc, const_cast(argv)); + } +}; +extern ELPP_EXPORT base::type::StoragePointer elStorage; +#define ELPP el::base::elStorage +class DefaultLogDispatchCallback : public LogDispatchCallback { + protected: + void + handle(const LogDispatchData* data); + + private: + const LogDispatchData* m_data; + void + dispatch(base::type::string_t&& logLine); +}; +#if ELPP_ASYNC_LOGGING +class AsyncLogDispatchCallback : public LogDispatchCallback { + protected: + void + handle(const LogDispatchData* data); +}; +class AsyncDispatchWorker : public base::IWorker, public base::threading::ThreadSafe { + public: + AsyncDispatchWorker(); + virtual ~AsyncDispatchWorker(); + + bool + clean(void); + void + emptyQueue(void); + virtual void + start(void); + void + handle(AsyncLogItem* logItem); + void + run(void); + + void + setContinueRunning(bool value) { + base::threading::ScopedLock scopedLock(m_continueRunningLock); + m_continueRunning = value; + } + + bool + continueRunning(void) const { + return m_continueRunning; + } + + private: + std::condition_variable cv; + bool m_continueRunning; + base::threading::Mutex m_continueRunningLock; +}; +#endif // ELPP_ASYNC_LOGGING +} // namespace base +namespace base { +class DefaultLogBuilder : public LogBuilder { + public: + base::type::string_t + build(const LogMessage* logMessage, bool appendNewLine) const; +}; +/// @brief Dispatches log messages +class LogDispatcher : base::NoCopy { + public: + LogDispatcher(bool proceed, LogMessage* logMessage, base::DispatchAction dispatchAction) + : m_proceed(proceed), m_logMessage(logMessage), m_dispatchAction(std::move(dispatchAction)) { + } + + void + dispatch(void); + + private: + bool m_proceed; + LogMessage* m_logMessage; + base::DispatchAction m_dispatchAction; +}; +#if defined(ELPP_STL_LOGGING) +/// @brief Workarounds to write some STL logs +/// +/// @detail There is workaround needed to loop through some stl containers. In order to do that, we need iterable +/// containers of same type and provide iterator interface and pass it on to writeIterator(). Remember, this is passed +/// by value in constructor so that we dont change original containers. This operation is as expensive as +/// Big-O(std::min(class_.size(), base::consts::kMaxLogPerContainer)) +namespace workarounds { +/// @brief Abstract IterableContainer template that provides interface for iterable classes of type T +template +class IterableContainer { + public: + typedef typename Container::iterator iterator; + typedef typename Container::const_iterator const_iterator; + IterableContainer(void) { + } + virtual ~IterableContainer(void) { + } + iterator + begin(void) { + return getContainer().begin(); + } + iterator + end(void) { + return getContainer().end(); + } + + private: + virtual Container& + getContainer(void) = 0; +}; +/// @brief Implements IterableContainer and provides iterable std::priority_queue class +template , + typename Comparator = std::less> +class IterablePriorityQueue : public IterableContainer, + public std::priority_queue { + public: + IterablePriorityQueue(std::priority_queue queue_) { + std::size_t count_ = 0; + while (++count_ < base::consts::kMaxLogPerContainer && !queue_.empty()) { + this->push(queue_.top()); + queue_.pop(); + } + } + + private: + inline Container& + getContainer(void) { + return this->c; + } +}; +/// @brief Implements IterableContainer and provides iterable std::queue class +template > +class IterableQueue : public IterableContainer, public std::queue { + public: + IterableQueue(std::queue queue_) { + std::size_t count_ = 0; + while (++count_ < base::consts::kMaxLogPerContainer && !queue_.empty()) { + this->push(queue_.front()); + queue_.pop(); + } + } + + private: + inline Container& + getContainer(void) { + return this->c; + } +}; +/// @brief Implements IterableContainer and provides iterable std::stack class +template > +class IterableStack : public IterableContainer, public std::stack { + public: + IterableStack(std::stack stack_) { + std::size_t count_ = 0; + while (++count_ < base::consts::kMaxLogPerContainer && !stack_.empty()) { + this->push(stack_.top()); + stack_.pop(); + } + } + + private: + inline Container& + getContainer(void) { + return this->c; + } +}; +} // namespace workarounds +#endif // defined(ELPP_STL_LOGGING) +// Log message builder +class MessageBuilder { + public: + MessageBuilder(void) : m_logger(nullptr), m_containerLogSeperator(ELPP_LITERAL("")) { + } + void + initialize(Logger* logger); + +#define ELPP_SIMPLE_LOG(LOG_TYPE) \ + MessageBuilder& operator<<(LOG_TYPE msg) { \ + m_logger->stream() << msg; \ + if (ELPP->hasFlag(LoggingFlag::AutoSpacing)) { \ + m_logger->stream() << " "; \ + } \ + return *this; \ + } + + inline MessageBuilder& + operator<<(const std::string& msg) { + return operator<<(msg.c_str()); + } + ELPP_SIMPLE_LOG(char) + ELPP_SIMPLE_LOG(bool) + ELPP_SIMPLE_LOG(signed short) + ELPP_SIMPLE_LOG(unsigned short) + ELPP_SIMPLE_LOG(signed int) + ELPP_SIMPLE_LOG(unsigned int) + ELPP_SIMPLE_LOG(signed long) + ELPP_SIMPLE_LOG(unsigned long) + ELPP_SIMPLE_LOG(float) + ELPP_SIMPLE_LOG(double) + ELPP_SIMPLE_LOG(char*) + ELPP_SIMPLE_LOG(const char*) + ELPP_SIMPLE_LOG(const void*) + ELPP_SIMPLE_LOG(long double) + inline MessageBuilder& + operator<<(const std::wstring& msg) { + return operator<<(msg.c_str()); + } + MessageBuilder& + operator<<(const wchar_t* msg); + // ostream manipulators + inline MessageBuilder& + operator<<(std::ostream& (*OStreamMani)(std::ostream&)) { + m_logger->stream() << OStreamMani; + return *this; + } +#define ELPP_ITERATOR_CONTAINER_LOG_ONE_ARG(temp) \ + template \ + inline MessageBuilder& operator<<(const temp& template_inst) { \ + return writeIterator(template_inst.begin(), template_inst.end(), template_inst.size()); \ + } +#define ELPP_ITERATOR_CONTAINER_LOG_TWO_ARG(temp) \ + template \ + inline MessageBuilder& operator<<(const temp& template_inst) { \ + return writeIterator(template_inst.begin(), template_inst.end(), template_inst.size()); \ + } +#define ELPP_ITERATOR_CONTAINER_LOG_THREE_ARG(temp) \ + template \ + inline MessageBuilder& operator<<(const temp& template_inst) { \ + return writeIterator(template_inst.begin(), template_inst.end(), template_inst.size()); \ + } +#define ELPP_ITERATOR_CONTAINER_LOG_FOUR_ARG(temp) \ + template \ + inline MessageBuilder& operator<<(const temp& template_inst) { \ + return writeIterator(template_inst.begin(), template_inst.end(), template_inst.size()); \ + } +#define ELPP_ITERATOR_CONTAINER_LOG_FIVE_ARG(temp) \ + template \ + inline MessageBuilder& operator<<(const temp& template_inst) { \ + return writeIterator(template_inst.begin(), template_inst.end(), template_inst.size()); \ + } + +#if defined(ELPP_STL_LOGGING) + ELPP_ITERATOR_CONTAINER_LOG_TWO_ARG(std::vector) + ELPP_ITERATOR_CONTAINER_LOG_TWO_ARG(std::list) + ELPP_ITERATOR_CONTAINER_LOG_TWO_ARG(std::deque) + ELPP_ITERATOR_CONTAINER_LOG_THREE_ARG(std::set) + ELPP_ITERATOR_CONTAINER_LOG_THREE_ARG(std::multiset) + ELPP_ITERATOR_CONTAINER_LOG_FOUR_ARG(std::map) + ELPP_ITERATOR_CONTAINER_LOG_FOUR_ARG(std::multimap) + template + inline MessageBuilder& + operator<<(const std::queue& queue_) { + base::workarounds::IterableQueue iterableQueue_ = + static_cast>(queue_); + return writeIterator(iterableQueue_.begin(), iterableQueue_.end(), iterableQueue_.size()); + } + template + inline MessageBuilder& + operator<<(const std::stack& stack_) { + base::workarounds::IterableStack iterableStack_ = + static_cast>(stack_); + return writeIterator(iterableStack_.begin(), iterableStack_.end(), iterableStack_.size()); + } + template + inline MessageBuilder& + operator<<(const std::priority_queue& priorityQueue_) { + base::workarounds::IterablePriorityQueue iterablePriorityQueue_ = + static_cast>(priorityQueue_); + return writeIterator(iterablePriorityQueue_.begin(), iterablePriorityQueue_.end(), + iterablePriorityQueue_.size()); + } + template + MessageBuilder& + operator<<(const std::pair& pair_) { + m_logger->stream() << ELPP_LITERAL("("); + operator<<(static_cast(pair_.first)); + m_logger->stream() << ELPP_LITERAL(", "); + operator<<(static_cast(pair_.second)); + m_logger->stream() << ELPP_LITERAL(")"); + return *this; + } + template + MessageBuilder& + operator<<(const std::bitset& bitset_) { + m_logger->stream() << ELPP_LITERAL("["); + operator<<(bitset_.to_string()); + m_logger->stream() << ELPP_LITERAL("]"); + return *this; + } +#if defined(ELPP_LOG_STD_ARRAY) + template + inline MessageBuilder& + operator<<(const std::array& array) { + return writeIterator(array.begin(), array.end(), array.size()); + } +#endif // defined(ELPP_LOG_STD_ARRAY) +#if defined(ELPP_LOG_UNORDERED_MAP) + ELPP_ITERATOR_CONTAINER_LOG_FIVE_ARG(std::unordered_map) + ELPP_ITERATOR_CONTAINER_LOG_FIVE_ARG(std::unordered_multimap) +#endif // defined(ELPP_LOG_UNORDERED_MAP) +#if defined(ELPP_LOG_UNORDERED_SET) + ELPP_ITERATOR_CONTAINER_LOG_FOUR_ARG(std::unordered_set) + ELPP_ITERATOR_CONTAINER_LOG_FOUR_ARG(std::unordered_multiset) +#endif // defined(ELPP_LOG_UNORDERED_SET) +#endif // defined(ELPP_STL_LOGGING) +#if defined(ELPP_QT_LOGGING) + inline MessageBuilder& + operator<<(const QString& msg) { +#if defined(ELPP_UNICODE) + m_logger->stream() << msg.toStdWString(); +#else + m_logger->stream() << msg.toStdString(); +#endif // defined(ELPP_UNICODE) + return *this; + } + inline MessageBuilder& + operator<<(const QByteArray& msg) { + return operator<<(QString(msg)); + } + inline MessageBuilder& + operator<<(const QStringRef& msg) { + return operator<<(msg.toString()); + } + inline MessageBuilder& + operator<<(qint64 msg) { +#if defined(ELPP_UNICODE) + m_logger->stream() << QString::number(msg).toStdWString(); +#else + m_logger->stream() << QString::number(msg).toStdString(); +#endif // defined(ELPP_UNICODE) + return *this; + } + inline MessageBuilder& + operator<<(quint64 msg) { +#if defined(ELPP_UNICODE) + m_logger->stream() << QString::number(msg).toStdWString(); +#else + m_logger->stream() << QString::number(msg).toStdString(); +#endif // defined(ELPP_UNICODE) + return *this; + } + inline MessageBuilder& + operator<<(QChar msg) { + m_logger->stream() << msg.toLatin1(); + return *this; + } + inline MessageBuilder& + operator<<(const QLatin1String& msg) { + m_logger->stream() << msg.latin1(); + return *this; + } + ELPP_ITERATOR_CONTAINER_LOG_ONE_ARG(QList) + ELPP_ITERATOR_CONTAINER_LOG_ONE_ARG(QVector) + ELPP_ITERATOR_CONTAINER_LOG_ONE_ARG(QQueue) + ELPP_ITERATOR_CONTAINER_LOG_ONE_ARG(QSet) + ELPP_ITERATOR_CONTAINER_LOG_ONE_ARG(QLinkedList) + ELPP_ITERATOR_CONTAINER_LOG_ONE_ARG(QStack) + template + MessageBuilder& + operator<<(const QPair& pair_) { + m_logger->stream() << ELPP_LITERAL("("); + operator<<(static_cast(pair_.first)); + m_logger->stream() << ELPP_LITERAL(", "); + operator<<(static_cast(pair_.second)); + m_logger->stream() << ELPP_LITERAL(")"); + return *this; + } + template + MessageBuilder& + operator<<(const QMap& map_) { + m_logger->stream() << ELPP_LITERAL("["); + QList keys = map_.keys(); + typename QList::const_iterator begin = keys.begin(); + typename QList::const_iterator end = keys.end(); + int max_ = static_cast(base::consts::kMaxLogPerContainer); // to prevent warning + for (int index_ = 0; begin != end && index_ < max_; ++index_, ++begin) { + m_logger->stream() << ELPP_LITERAL("("); + operator<<(static_cast(*begin)); + m_logger->stream() << ELPP_LITERAL(", "); + operator<<(static_cast(map_.value(*begin))); + m_logger->stream() << ELPP_LITERAL(")"); + m_logger->stream() << ((index_ < keys.size() - 1) ? m_containerLogSeperator : ELPP_LITERAL("")); + } + if (begin != end) { + m_logger->stream() << ELPP_LITERAL("..."); + } + m_logger->stream() << ELPP_LITERAL("]"); + return *this; + } + template + inline MessageBuilder& + operator<<(const QMultiMap& map_) { + operator<<(static_cast>(map_)); + return *this; + } + template + MessageBuilder& + operator<<(const QHash& hash_) { + m_logger->stream() << ELPP_LITERAL("["); + QList keys = hash_.keys(); + typename QList::const_iterator begin = keys.begin(); + typename QList::const_iterator end = keys.end(); + int max_ = static_cast(base::consts::kMaxLogPerContainer); // prevent type warning + for (int index_ = 0; begin != end && index_ < max_; ++index_, ++begin) { + m_logger->stream() << ELPP_LITERAL("("); + operator<<(static_cast(*begin)); + m_logger->stream() << ELPP_LITERAL(", "); + operator<<(static_cast(hash_.value(*begin))); + m_logger->stream() << ELPP_LITERAL(")"); + m_logger->stream() << ((index_ < keys.size() - 1) ? m_containerLogSeperator : ELPP_LITERAL("")); + } + if (begin != end) { + m_logger->stream() << ELPP_LITERAL("..."); + } + m_logger->stream() << ELPP_LITERAL("]"); + return *this; + } + template + inline MessageBuilder& + operator<<(const QMultiHash& multiHash_) { + operator<<(static_cast>(multiHash_)); + return *this; + } +#endif // defined(ELPP_QT_LOGGING) +#if defined(ELPP_BOOST_LOGGING) + ELPP_ITERATOR_CONTAINER_LOG_TWO_ARG(boost::container::vector) + ELPP_ITERATOR_CONTAINER_LOG_TWO_ARG(boost::container::stable_vector) + ELPP_ITERATOR_CONTAINER_LOG_TWO_ARG(boost::container::list) + ELPP_ITERATOR_CONTAINER_LOG_TWO_ARG(boost::container::deque) + ELPP_ITERATOR_CONTAINER_LOG_FOUR_ARG(boost::container::map) + ELPP_ITERATOR_CONTAINER_LOG_FOUR_ARG(boost::container::flat_map) + ELPP_ITERATOR_CONTAINER_LOG_THREE_ARG(boost::container::set) + ELPP_ITERATOR_CONTAINER_LOG_THREE_ARG(boost::container::flat_set) +#endif // defined(ELPP_BOOST_LOGGING) + + /// @brief Macro used internally that can be used externally to make containers easylogging++ friendly + /// + /// @detail This macro expands to write an ostream& operator<< for container. This container is expected to + /// have begin() and end() methods that return respective iterators + /// @param ContainerType Type of container e.g, MyList from WX_DECLARE_LIST(int, MyList); in wxwidgets + /// @param SizeMethod Method used to get size of container. + /// @param ElementInstance Instance of element to be fed out. Insance name is "elem". See WXELPP_ENABLED macro + /// for an example usage +#define MAKE_CONTAINERELPP_FRIENDLY(ContainerType, SizeMethod, ElementInstance) \ + el::base::type::ostream_t& operator<<(el::base::type::ostream_t& ss, const ContainerType& container) { \ + const el::base::type::char_t* sep = \ + ELPP->hasFlag(el::LoggingFlag::NewLineForContainer) ? ELPP_LITERAL("\n ") : ELPP_LITERAL(", "); \ + ContainerType::const_iterator elem = container.begin(); \ + ContainerType::const_iterator endElem = container.end(); \ + std::size_t size_ = container.SizeMethod; \ + ss << ELPP_LITERAL("["); \ + for (std::size_t i = 0; elem != endElem && i < el::base::consts::kMaxLogPerContainer; ++i, ++elem) { \ + ss << ElementInstance; \ + ss << ((i < size_ - 1) ? sep : ELPP_LITERAL("")); \ + } \ + if (elem != endElem) { \ + ss << ELPP_LITERAL("..."); \ + } \ + ss << ELPP_LITERAL("]"); \ + return ss; \ + } +#if defined(ELPP_WXWIDGETS_LOGGING) + ELPP_ITERATOR_CONTAINER_LOG_ONE_ARG(wxVector) +#define ELPP_WX_PTR_ENABLED(ContainerType) MAKE_CONTAINERELPP_FRIENDLY(ContainerType, size(), *(*elem)) +#define ELPP_WX_ENABLED(ContainerType) MAKE_CONTAINERELPP_FRIENDLY(ContainerType, size(), (*elem)) +#define ELPP_WX_HASH_MAP_ENABLED(ContainerType) MAKE_CONTAINERELPP_FRIENDLY(ContainerType, size(), \ +ELPP_LITERAL("(") << elem->first << ELPP_LITERAL(", ") << elem->second << ELPP_LITERAL(")") +#else +#define ELPP_WX_PTR_ENABLED(ContainerType) +#define ELPP_WX_ENABLED(ContainerType) +#define ELPP_WX_HASH_MAP_ENABLED(ContainerType) +#endif // defined(ELPP_WXWIDGETS_LOGGING) + // Other classes + template + ELPP_SIMPLE_LOG(const Class&) +#undef ELPP_SIMPLE_LOG +#undef ELPP_ITERATOR_CONTAINER_LOG_ONE_ARG +#undef ELPP_ITERATOR_CONTAINER_LOG_TWO_ARG +#undef ELPP_ITERATOR_CONTAINER_LOG_THREE_ARG +#undef ELPP_ITERATOR_CONTAINER_LOG_FOUR_ARG +#undef ELPP_ITERATOR_CONTAINER_LOG_FIVE_ARG + private : Logger* m_logger; + const base::type::char_t* m_containerLogSeperator; + + template + MessageBuilder& + writeIterator(Iterator begin_, Iterator end_, std::size_t size_) { + m_logger->stream() << ELPP_LITERAL("["); + for (std::size_t i = 0; begin_ != end_ && i < base::consts::kMaxLogPerContainer; ++i, ++begin_) { + operator<<(*begin_); + m_logger->stream() << ((i < size_ - 1) ? m_containerLogSeperator : ELPP_LITERAL("")); + } + if (begin_ != end_) { + m_logger->stream() << ELPP_LITERAL("..."); + } + m_logger->stream() << ELPP_LITERAL("]"); + if (ELPP->hasFlag(LoggingFlag::AutoSpacing)) { + m_logger->stream() << " "; + } + return *this; + } +}; +/// @brief Writes nothing - Used when certain log is disabled +class NullWriter : base::NoCopy { + public: + NullWriter(void) { + } + + // Null manipulator + inline NullWriter& + operator<<(std::ostream& (*)(std::ostream&)) { + return *this; + } + + template + inline NullWriter& + operator<<(const T&) { + return *this; + } + + inline operator bool() { + return true; + } +}; +/// @brief Main entry point of each logging +class Writer : base::NoCopy { + public: + Writer(Level level, const char* file, base::type::LineNumber line, const char* func, + base::DispatchAction dispatchAction = base::DispatchAction::NormalLog, + base::type::VerboseLevel verboseLevel = 0) + : m_msg(nullptr), + m_level(level), + m_file(file), + m_line(line), + m_func(func), + m_verboseLevel(verboseLevel), + m_logger(nullptr), + m_proceed(false), + m_dispatchAction(dispatchAction) { + } + + Writer(LogMessage* msg, base::DispatchAction dispatchAction = base::DispatchAction::NormalLog) + : m_msg(msg), + m_level(msg != nullptr ? msg->level() : Level::Unknown), + m_line(0), + m_logger(nullptr), + m_proceed(false), + m_dispatchAction(dispatchAction) { + } + + virtual ~Writer(void) { + processDispatch(); + } + + template + inline Writer& + operator<<(const T& log) { +#if ELPP_LOGGING_ENABLED + if (m_proceed) { + m_messageBuilder << log; + } +#endif // ELPP_LOGGING_ENABLED + return *this; + } + + inline Writer& + operator<<(std::ostream& (*log)(std::ostream&)) { +#if ELPP_LOGGING_ENABLED + if (m_proceed) { + m_messageBuilder << log; + } +#endif // ELPP_LOGGING_ENABLED + return *this; + } + + inline operator bool() { + return true; + } + + Writer& + construct(Logger* logger, bool needLock = true); + Writer& + construct(int count, const char* loggerIds, ...); + + protected: + LogMessage* m_msg; + Level m_level; + const char* m_file; + const base::type::LineNumber m_line; + const char* m_func; + base::type::VerboseLevel m_verboseLevel; + Logger* m_logger; + bool m_proceed; + base::MessageBuilder m_messageBuilder; + base::DispatchAction m_dispatchAction; + std::vector m_loggerIds; + friend class el::Helpers; + + void + initializeLogger(const std::string& loggerId, bool lookup = true, bool needLock = true); + void + processDispatch(); + void + triggerDispatch(void); +}; +class PErrorWriter : public base::Writer { + public: + PErrorWriter(Level level, const char* file, base::type::LineNumber line, const char* func, + base::DispatchAction dispatchAction = base::DispatchAction::NormalLog, + base::type::VerboseLevel verboseLevel = 0) + : base::Writer(level, file, line, func, dispatchAction, verboseLevel) { + } + + virtual ~PErrorWriter(void); +}; +} // namespace base +// Logging from Logger class. Why this is here? Because we have Storage and Writer class available +#if ELPP_VARIADIC_TEMPLATES_SUPPORTED +template +void +Logger::log_(Level level, int vlevel, const char* s, const T& value, const Args&... args) { + base::MessageBuilder b; + b.initialize(this); + while (*s) { + if (*s == base::consts::kFormatSpecifierChar) { + if (*(s + 1) == base::consts::kFormatSpecifierChar) { + ++s; + } else { + if (*(s + 1) == base::consts::kFormatSpecifierCharValue) { + ++s; + b << value; + log_(level, vlevel, ++s, args...); + return; + } + } + } + b << *s++; + } + ELPP_INTERNAL_ERROR("Too many arguments provided. Unable to handle. Please provide more format specifiers", false); +} +template +void +Logger::log_(Level level, int vlevel, const T& log) { + if (level == Level::Verbose) { + if (ELPP->vRegistry()->allowed(vlevel, __FILE__)) { + base::Writer(Level::Verbose, "FILE", 0, "FUNCTION", base::DispatchAction::NormalLog, vlevel) + .construct(this, false) + << log; + } else { + stream().str(ELPP_LITERAL("")); + releaseLock(); + } + } else { + base::Writer(level, "FILE", 0, "FUNCTION").construct(this, false) << log; + } +} +template +inline void +Logger::log(Level level, const char* s, const T& value, const Args&... args) { + acquireLock(); // released in Writer! + log_(level, 0, s, value, args...); +} +template +inline void +Logger::log(Level level, const T& log) { + acquireLock(); // released in Writer! + log_(level, 0, log); +} +#if ELPP_VERBOSE_LOG +template +inline void +Logger::verbose(int vlevel, const char* s, const T& value, const Args&... args) { + acquireLock(); // released in Writer! + log_(el::Level::Verbose, vlevel, s, value, args...); +} +template +inline void +Logger::verbose(int vlevel, const T& log) { + acquireLock(); // released in Writer! + log_(el::Level::Verbose, vlevel, log); +} +#else +template +inline void +Logger::verbose(int, const char*, const T&, const Args&...) { + return; +} +template +inline void +Logger::verbose(int, const T&) { + return; +} +#endif // ELPP_VERBOSE_LOG +#define LOGGER_LEVEL_WRITERS(FUNCTION_NAME, LOG_LEVEL) \ + template \ + inline void Logger::FUNCTION_NAME(const char* s, const T& value, const Args&... args) { \ + log(LOG_LEVEL, s, value, args...); \ + } \ + template \ + inline void Logger::FUNCTION_NAME(const T& value) { \ + log(LOG_LEVEL, value); \ + } +#define LOGGER_LEVEL_WRITERS_DISABLED(FUNCTION_NAME, LOG_LEVEL) \ + template \ + inline void Logger::FUNCTION_NAME(const char*, const T&, const Args&...) { \ + return; \ + } \ + template \ + inline void Logger::FUNCTION_NAME(const T&) { \ + return; \ + } + +#if ELPP_INFO_LOG +LOGGER_LEVEL_WRITERS(info, Level::Info) +#else +LOGGER_LEVEL_WRITERS_DISABLED(info, Level::Info) +#endif // ELPP_INFO_LOG +#if ELPP_DEBUG_LOG +LOGGER_LEVEL_WRITERS(debug, Level::Debug) +#else +LOGGER_LEVEL_WRITERS_DISABLED(debug, Level::Debug) +#endif // ELPP_DEBUG_LOG +#if ELPP_WARNING_LOG +LOGGER_LEVEL_WRITERS(warn, Level::Warning) +#else +LOGGER_LEVEL_WRITERS_DISABLED(warn, Level::Warning) +#endif // ELPP_WARNING_LOG +#if ELPP_ERROR_LOG +LOGGER_LEVEL_WRITERS(error, Level::Error) +#else +LOGGER_LEVEL_WRITERS_DISABLED(error, Level::Error) +#endif // ELPP_ERROR_LOG +#if ELPP_FATAL_LOG +LOGGER_LEVEL_WRITERS(fatal, Level::Fatal) +#else +LOGGER_LEVEL_WRITERS_DISABLED(fatal, Level::Fatal) +#endif // ELPP_FATAL_LOG +#if ELPP_TRACE_LOG +LOGGER_LEVEL_WRITERS(trace, Level::Trace) +#else +LOGGER_LEVEL_WRITERS_DISABLED(trace, Level::Trace) +#endif // ELPP_TRACE_LOG +#undef LOGGER_LEVEL_WRITERS +#undef LOGGER_LEVEL_WRITERS_DISABLED +#endif // ELPP_VARIADIC_TEMPLATES_SUPPORTED +#if ELPP_COMPILER_MSVC +#define ELPP_VARIADIC_FUNC_MSVC(variadicFunction, variadicArgs) variadicFunction variadicArgs +#define ELPP_VARIADIC_FUNC_MSVC_RUN(variadicFunction, ...) ELPP_VARIADIC_FUNC_MSVC(variadicFunction, (__VA_ARGS__)) +#define el_getVALength(...) \ + ELPP_VARIADIC_FUNC_MSVC_RUN(el_resolveVALength, 0, ##__VA_ARGS__, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0) +#else +#if ELPP_COMPILER_CLANG +#define el_getVALength(...) el_resolveVALength(0, __VA_ARGS__, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0) +#else +#define el_getVALength(...) el_resolveVALength(0, ##__VA_ARGS__, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0) +#endif // ELPP_COMPILER_CLANG +#endif // ELPP_COMPILER_MSVC +#define el_resolveVALength(_0, _1, _2, _3, _4, _5, _6, _7, _8, _9, _10, N, ...) N +#define ELPP_WRITE_LOG(writer, level, dispatchAction, ...) \ + writer(level, __FILE__, __LINE__, ELPP_FUNC, dispatchAction).construct(el_getVALength(__VA_ARGS__), __VA_ARGS__) +#define ELPP_WRITE_LOG_IF(writer, condition, level, dispatchAction, ...) \ + if (condition) \ + writer(level, __FILE__, __LINE__, ELPP_FUNC, dispatchAction).construct(el_getVALength(__VA_ARGS__), __VA_ARGS__) +#define ELPP_WRITE_LOG_EVERY_N(writer, occasion, level, dispatchAction, ...) \ + ELPP->validateEveryNCounter(__FILE__, __LINE__, occasion) && \ + writer(level, __FILE__, __LINE__, ELPP_FUNC, dispatchAction) \ + .construct(el_getVALength(__VA_ARGS__), __VA_ARGS__) +#define ELPP_WRITE_LOG_AFTER_N(writer, n, level, dispatchAction, ...) \ + ELPP->validateAfterNCounter(__FILE__, __LINE__, n) && writer(level, __FILE__, __LINE__, ELPP_FUNC, dispatchAction) \ + .construct(el_getVALength(__VA_ARGS__), __VA_ARGS__) +#define ELPP_WRITE_LOG_N_TIMES(writer, n, level, dispatchAction, ...) \ + ELPP->validateNTimesCounter(__FILE__, __LINE__, n) && writer(level, __FILE__, __LINE__, ELPP_FUNC, dispatchAction) \ + .construct(el_getVALength(__VA_ARGS__), __VA_ARGS__) +#if defined(ELPP_FEATURE_ALL) || defined(ELPP_FEATURE_PERFORMANCE_TRACKING) +class PerformanceTrackingData { + public: + enum class DataType : base::type::EnumType { Checkpoint = 1, Complete = 2 }; + // Do not use constructor, will run into multiple definition error, use init(PerformanceTracker*) + explicit PerformanceTrackingData(DataType dataType) + : m_performanceTracker(nullptr), + m_dataType(dataType), + m_firstCheckpoint(false), + m_file(""), + m_line(0), + m_func("") { + } + inline const std::string* + blockName(void) const; + inline const struct timeval* + startTime(void) const; + inline const struct timeval* + endTime(void) const; + inline const struct timeval* + lastCheckpointTime(void) const; + inline const base::PerformanceTracker* + performanceTracker(void) const { + return m_performanceTracker; + } + inline PerformanceTrackingData::DataType + dataType(void) const { + return m_dataType; + } + inline bool + firstCheckpoint(void) const { + return m_firstCheckpoint; + } + inline std::string + checkpointId(void) const { + return m_checkpointId; + } + inline const char* + file(void) const { + return m_file; + } + inline base::type::LineNumber + line(void) const { + return m_line; + } + inline const char* + func(void) const { + return m_func; + } + inline const base::type::string_t* + formattedTimeTaken() const { + return &m_formattedTimeTaken; + } + inline const std::string& + loggerId(void) const; + + private: + base::PerformanceTracker* m_performanceTracker; + base::type::string_t m_formattedTimeTaken; + PerformanceTrackingData::DataType m_dataType; + bool m_firstCheckpoint; + std::string m_checkpointId; + const char* m_file; + base::type::LineNumber m_line; + const char* m_func; + inline void + init(base::PerformanceTracker* performanceTracker, bool firstCheckpoint = false) { + m_performanceTracker = performanceTracker; + m_firstCheckpoint = firstCheckpoint; + } + + friend class el::base::PerformanceTracker; +}; +namespace base { +/// @brief Represents performanceTracker block of code that conditionally adds performance status to log +/// either when goes outside the scope of when checkpoint() is called +class PerformanceTracker : public base::threading::ThreadSafe, public Loggable { + public: + PerformanceTracker(const std::string& blockName, + base::TimestampUnit timestampUnit = base::TimestampUnit::Millisecond, + const std::string& loggerId = std::string(el::base::consts::kPerformanceLoggerId), + bool scopedLog = true, Level level = base::consts::kPerformanceTrackerDefaultLevel); + /// @brief Copy constructor + PerformanceTracker(const PerformanceTracker& t) + : m_blockName(t.m_blockName), + m_timestampUnit(t.m_timestampUnit), + m_loggerId(t.m_loggerId), + m_scopedLog(t.m_scopedLog), + m_level(t.m_level), + m_hasChecked(t.m_hasChecked), + m_lastCheckpointId(t.m_lastCheckpointId), + m_enabled(t.m_enabled), + m_startTime(t.m_startTime), + m_endTime(t.m_endTime), + m_lastCheckpointTime(t.m_lastCheckpointTime) { + } + virtual ~PerformanceTracker(void); + /// @brief A checkpoint for current performanceTracker block. + void + checkpoint(const std::string& id = std::string(), const char* file = __FILE__, + base::type::LineNumber line = __LINE__, const char* func = ""); + inline Level + level(void) const { + return m_level; + } + + private: + std::string m_blockName; + base::TimestampUnit m_timestampUnit; + std::string m_loggerId; + bool m_scopedLog; + Level m_level; + bool m_hasChecked; + std::string m_lastCheckpointId; + bool m_enabled; + struct timeval m_startTime, m_endTime, m_lastCheckpointTime; + + PerformanceTracker(void); + + friend class el::PerformanceTrackingData; + friend class base::DefaultPerformanceTrackingCallback; + + const inline base::type::string_t + getFormattedTimeTaken() const { + return getFormattedTimeTaken(m_startTime); + } + + const base::type::string_t + getFormattedTimeTaken(struct timeval startTime) const; + + virtual inline void + log(el::base::type::ostream_t& os) const { + os << getFormattedTimeTaken(); + } +}; +class DefaultPerformanceTrackingCallback : public PerformanceTrackingCallback { + protected: + void + handle(const PerformanceTrackingData* data) { + m_data = data; + base::type::stringstream_t ss; + if (m_data->dataType() == PerformanceTrackingData::DataType::Complete) { + ss << ELPP_LITERAL("Executed [") << m_data->blockName()->c_str() << ELPP_LITERAL("] in [") + << *m_data->formattedTimeTaken() << ELPP_LITERAL("]"); + } else { + ss << ELPP_LITERAL("Performance checkpoint"); + if (!m_data->checkpointId().empty()) { + ss << ELPP_LITERAL(" [") << m_data->checkpointId().c_str() << ELPP_LITERAL("]"); + } + ss << ELPP_LITERAL(" for block [") << m_data->blockName()->c_str() << ELPP_LITERAL("] : [") + << *m_data->performanceTracker(); + if (!ELPP->hasFlag(LoggingFlag::DisablePerformanceTrackingCheckpointComparison) && + m_data->performanceTracker()->m_hasChecked) { + ss << ELPP_LITERAL(" ([") << *m_data->formattedTimeTaken() << ELPP_LITERAL("] from "); + if (m_data->performanceTracker()->m_lastCheckpointId.empty()) { + ss << ELPP_LITERAL("last checkpoint"); + } else { + ss << ELPP_LITERAL("checkpoint '") << m_data->performanceTracker()->m_lastCheckpointId.c_str() + << ELPP_LITERAL("'"); + } + ss << ELPP_LITERAL(")]"); + } else { + ss << ELPP_LITERAL("]"); + } + } + el::base::Writer(m_data->performanceTracker()->level(), m_data->file(), m_data->line(), m_data->func()) + .construct(1, m_data->loggerId().c_str()) + << ss.str(); + } + + private: + const PerformanceTrackingData* m_data; +}; +} // namespace base +inline const std::string* +PerformanceTrackingData::blockName() const { + return const_cast(&m_performanceTracker->m_blockName); +} +inline const struct timeval* +PerformanceTrackingData::startTime() const { + return const_cast(&m_performanceTracker->m_startTime); +} +inline const struct timeval* +PerformanceTrackingData::endTime() const { + return const_cast(&m_performanceTracker->m_endTime); +} +inline const struct timeval* +PerformanceTrackingData::lastCheckpointTime() const { + return const_cast(&m_performanceTracker->m_lastCheckpointTime); +} +inline const std::string& +PerformanceTrackingData::loggerId(void) const { + return m_performanceTracker->m_loggerId; +} +#endif // defined(ELPP_FEATURE_ALL) || defined(ELPP_FEATURE_PERFORMANCE_TRACKING) +namespace base { +/// @brief Contains some internal debugging tools like crash handler and stack tracer +namespace debug { +#if defined(ELPP_FEATURE_ALL) || defined(ELPP_FEATURE_CRASH_LOG) +class StackTrace : base::NoCopy { + public: + static const unsigned int kMaxStack = 64; + static const unsigned int kStackStart = 2; // We want to skip c'tor and StackTrace::generateNew() + class StackTraceEntry { + public: + StackTraceEntry(std::size_t index, const std::string& loc, const std::string& demang, const std::string& hex, + const std::string& addr); + StackTraceEntry(std::size_t index, const std::string& loc) : m_index(index), m_location(loc) { + } + std::size_t m_index; + std::string m_location; + std::string m_demangled; + std::string m_hex; + std::string m_addr; + friend std::ostream& + operator<<(std::ostream& ss, const StackTraceEntry& si); + + private: + StackTraceEntry(void); + }; + + StackTrace(void) { + generateNew(); + } + + virtual ~StackTrace(void) { + } + + inline std::vector& + getLatestStack(void) { + return m_stack; + } + + friend std::ostream& + operator<<(std::ostream& os, const StackTrace& st); + + private: + std::vector m_stack; + + void + generateNew(void); +}; +/// @brief Handles unexpected crashes +class CrashHandler : base::NoCopy { + public: + typedef void (*Handler)(int); + + explicit CrashHandler(bool useDefault); + explicit CrashHandler(const Handler& cHandler) { + setHandler(cHandler); + } + void + setHandler(const Handler& cHandler); + + private: + Handler m_handler; +}; +#else +class CrashHandler { + public: + explicit CrashHandler(bool) { + } +}; +#endif // defined(ELPP_FEATURE_ALL) || defined(ELPP_FEATURE_CRASH_LOG) +} // namespace debug +} // namespace base +extern base::debug::CrashHandler elCrashHandler; +#define MAKE_LOGGABLE(ClassType, ClassInstance, OutputStreamInstance) \ + el::base::type::ostream_t& operator<<(el::base::type::ostream_t& OutputStreamInstance, \ + const ClassType& ClassInstance) +/// @brief Initializes syslog with process ID, options and facility. calls closelog() on d'tor +class SysLogInitializer { + public: + SysLogInitializer(const char* processIdent, int options = 0, int facility = 0) { +#if defined(ELPP_SYSLOG) + openlog(processIdent, options, facility); +#else + ELPP_UNUSED(processIdent); + ELPP_UNUSED(options); + ELPP_UNUSED(facility); +#endif // defined(ELPP_SYSLOG) + } + virtual ~SysLogInitializer(void) { +#if defined(ELPP_SYSLOG) + closelog(); +#endif // defined(ELPP_SYSLOG) + } +}; +#define ELPP_INITIALIZE_SYSLOG(id, opt, fac) el::SysLogInitializer elSyslogInit(id, opt, fac) +/// @brief Static helpers for developers +class Helpers : base::StaticClass { + public: + /// @brief Shares logging repository (base::Storage) + static inline void + setStorage(base::type::StoragePointer storage) { + ELPP = storage; + } + /// @return Main storage repository + static inline base::type::StoragePointer + storage() { + return ELPP; + } + /// @brief Sets application arguments and figures out whats active for logging and whats not. + static inline void + setArgs(int argc, char** argv) { + ELPP->setApplicationArguments(argc, argv); + } + /// @copydoc setArgs(int argc, char** argv) + static inline void + setArgs(int argc, const char** argv) { + ELPP->setApplicationArguments(argc, const_cast(argv)); + } + /// @brief Sets thread name for current thread. Requires std::thread + static inline void + setThreadName(const std::string& name) { + ELPP->setThreadName(name); + } + static inline std::string + getThreadName() { + return ELPP->getThreadName(base::threading::getCurrentThreadId()); + } +#if defined(ELPP_FEATURE_ALL) || defined(ELPP_FEATURE_CRASH_LOG) + /// @brief Overrides default crash handler and installs custom handler. + /// @param crashHandler A functor with no return type that takes single int argument. + /// Handler is a typedef with specification: void (*Handler)(int) + static inline void + setCrashHandler(const el::base::debug::CrashHandler::Handler& crashHandler) { + el::elCrashHandler.setHandler(crashHandler); + } + /// @brief Abort due to crash with signal in parameter + /// @param sig Crash signal + static void + crashAbort(int sig, const char* sourceFile = "", unsigned int long line = 0); + /// @brief Logs reason of crash as per sig + /// @param sig Crash signal + /// @param stackTraceIfAvailable Includes stack trace if available + /// @param level Logging level + /// @param logger Logger to use for logging + static void + logCrashReason(int sig, bool stackTraceIfAvailable = false, Level level = Level::Fatal, + const char* logger = base::consts::kDefaultLoggerId); +#endif // defined(ELPP_FEATURE_ALL) || defined(ELPP_FEATURE_CRASH_LOG) + /// @brief Installs pre rollout callback, this callback is triggered when log file is about to be rolled out + /// (can be useful for backing up) + static inline void + installPreRollOutCallback(const PreRollOutCallback& callback) { + ELPP->setPreRollOutCallback(callback); + } + /// @brief Uninstalls pre rollout callback + static inline void + uninstallPreRollOutCallback(void) { + ELPP->unsetPreRollOutCallback(); + } + /// @brief Installs post log dispatch callback, this callback is triggered when log is dispatched + template + static inline bool + installLogDispatchCallback(const std::string& id) { + return ELPP->installLogDispatchCallback(id); + } + /// @brief Uninstalls log dispatch callback + template + static inline void + uninstallLogDispatchCallback(const std::string& id) { + ELPP->uninstallLogDispatchCallback(id); + } + template + static inline T* + logDispatchCallback(const std::string& id) { + return ELPP->logDispatchCallback(id); + } +#if defined(ELPP_FEATURE_ALL) || defined(ELPP_FEATURE_PERFORMANCE_TRACKING) + /// @brief Installs post performance tracking callback, this callback is triggered when performance tracking is + /// finished + template + static inline bool + installPerformanceTrackingCallback(const std::string& id) { + return ELPP->installPerformanceTrackingCallback(id); + } + /// @brief Uninstalls post performance tracking handler + template + static inline void + uninstallPerformanceTrackingCallback(const std::string& id) { + ELPP->uninstallPerformanceTrackingCallback(id); + } + template + static inline T* + performanceTrackingCallback(const std::string& id) { + return ELPP->performanceTrackingCallback(id); + } +#endif // defined(ELPP_FEATURE_ALL) || defined(ELPP_FEATURE_PERFORMANCE_TRACKING) + /// @brief Converts template to std::string - useful for loggable classes to log containers within + /// log(std::ostream&) const + template + static std::string + convertTemplateToStdString(const T& templ) { + el::Logger* logger = ELPP->registeredLoggers()->get(el::base::consts::kDefaultLoggerId); + if (logger == nullptr) { + return std::string(); + } + base::MessageBuilder b; + b.initialize(logger); + logger->acquireLock(); + b << templ; +#if defined(ELPP_UNICODE) + std::string s = std::string(logger->stream().str().begin(), logger->stream().str().end()); +#else + std::string s = logger->stream().str(); +#endif // defined(ELPP_UNICODE) + logger->stream().str(ELPP_LITERAL("")); + logger->releaseLock(); + return s; + } + /// @brief Returns command line arguments (pointer) provided to easylogging++ + static inline const el::base::utils::CommandLineArgs* + commandLineArgs(void) { + return ELPP->commandLineArgs(); + } + /// @brief Reserve space for custom format specifiers for performance + /// @see std::vector::reserve + static inline void + reserveCustomFormatSpecifiers(std::size_t size) { + ELPP->m_customFormatSpecifiers.reserve(size); + } + /// @brief Installs user defined format specifier and handler + static inline void + installCustomFormatSpecifier(const CustomFormatSpecifier& customFormatSpecifier) { + ELPP->installCustomFormatSpecifier(customFormatSpecifier); + } + /// @brief Uninstalls user defined format specifier and handler + static inline bool + uninstallCustomFormatSpecifier(const char* formatSpecifier) { + return ELPP->uninstallCustomFormatSpecifier(formatSpecifier); + } + /// @brief Returns true if custom format specifier is installed + static inline bool + hasCustomFormatSpecifier(const char* formatSpecifier) { + return ELPP->hasCustomFormatSpecifier(formatSpecifier); + } + static inline void + validateFileRolling(Logger* logger, Level level) { + if (ELPP == nullptr || logger == nullptr) + return; + logger->m_typedConfigurations->validateFileRolling(level, ELPP->preRollOutCallback()); + } +}; +/// @brief Static helpers to deal with loggers and their configurations +class Loggers : base::StaticClass { + public: + /// @brief Gets existing or registers new logger + static Logger* + getLogger(const std::string& identity, bool registerIfNotAvailable = true); + /// @brief Changes default log builder for future loggers + static void + setDefaultLogBuilder(el::LogBuilderPtr& logBuilderPtr); + /// @brief Installs logger registration callback, this callback is triggered when new logger is registered + template + static inline bool + installLoggerRegistrationCallback(const std::string& id) { + return ELPP->registeredLoggers()->installLoggerRegistrationCallback(id); + } + /// @brief Uninstalls log dispatch callback + template + static inline void + uninstallLoggerRegistrationCallback(const std::string& id) { + ELPP->registeredLoggers()->uninstallLoggerRegistrationCallback(id); + } + template + static inline T* + loggerRegistrationCallback(const std::string& id) { + return ELPP->registeredLoggers()->loggerRegistrationCallback(id); + } + /// @brief Unregisters logger - use it only when you know what you are doing, you may unregister + /// loggers initialized / used by third-party libs. + static bool + unregisterLogger(const std::string& identity); + /// @brief Whether or not logger with id is registered + static bool + hasLogger(const std::string& identity); + /// @brief Reconfigures specified logger with new configurations + static Logger* + reconfigureLogger(Logger* logger, const Configurations& configurations); + /// @brief Reconfigures logger with new configurations after looking it up using identity + static Logger* + reconfigureLogger(const std::string& identity, const Configurations& configurations); + /// @brief Reconfigures logger's single configuration + static Logger* + reconfigureLogger(const std::string& identity, ConfigurationType configurationType, const std::string& value); + /// @brief Reconfigures all the existing loggers with new configurations + static void + reconfigureAllLoggers(const Configurations& configurations); + /// @brief Reconfigures single configuration for all the loggers + static inline void + reconfigureAllLoggers(ConfigurationType configurationType, const std::string& value) { + reconfigureAllLoggers(Level::Global, configurationType, value); + } + /// @brief Reconfigures single configuration for all the loggers for specified level + static void + reconfigureAllLoggers(Level level, ConfigurationType configurationType, const std::string& value); + /// @brief Sets default configurations. This configuration is used for future (and conditionally for existing) + /// loggers + static void + setDefaultConfigurations(const Configurations& configurations, bool reconfigureExistingLoggers = false); + /// @brief Returns current default + static const Configurations* + defaultConfigurations(void); + /// @brief Returns log stream reference pointer if needed by user + static const base::LogStreamsReferenceMap* + logStreamsReference(void); + /// @brief Default typed configuration based on existing defaultConf + static base::TypedConfigurations + defaultTypedConfigurations(void); + /// @brief Populates all logger IDs in current repository. + /// @param [out] targetList List of fill up. + static std::vector* + populateAllLoggerIds(std::vector* targetList); + /// @brief Sets configurations from global configuration file. + static void + configureFromGlobal(const char* globalConfigurationFilePath); + /// @brief Configures loggers using command line arg. Ensure you have already set command line args, + /// @return False if invalid argument or argument with no value provided, true if attempted to configure logger. + /// If true is returned that does not mean it has been configured successfully, it only means that it + /// has attempeted to configure logger using configuration file provided in argument + static bool + configureFromArg(const char* argKey); + /// @brief Flushes all loggers for all levels - Be careful if you dont know how many loggers are registered + static void + flushAll(void); + /// @brief Adds logging flag used internally. + static inline void + addFlag(LoggingFlag flag) { + ELPP->addFlag(flag); + } + /// @brief Removes logging flag used internally. + static inline void + removeFlag(LoggingFlag flag) { + ELPP->removeFlag(flag); + } + /// @brief Determines whether or not certain flag is active + static inline bool + hasFlag(LoggingFlag flag) { + return ELPP->hasFlag(flag); + } + /// @brief Adds flag and removes it when scope goes out + class ScopedAddFlag { + public: + ScopedAddFlag(LoggingFlag flag) : m_flag(flag) { + Loggers::addFlag(m_flag); + } + ~ScopedAddFlag(void) { + Loggers::removeFlag(m_flag); + } + + private: + LoggingFlag m_flag; + }; + /// @brief Removes flag and add it when scope goes out + class ScopedRemoveFlag { + public: + ScopedRemoveFlag(LoggingFlag flag) : m_flag(flag) { + Loggers::removeFlag(m_flag); + } + ~ScopedRemoveFlag(void) { + Loggers::addFlag(m_flag); + } + + private: + LoggingFlag m_flag; + }; + /// @brief Sets hierarchy for logging. Needs to enable logging flag (HierarchicalLogging) + static void + setLoggingLevel(Level level) { + ELPP->setLoggingLevel(level); + } + /// @brief Sets verbose level on the fly + static void + setVerboseLevel(base::type::VerboseLevel level); + /// @brief Gets current verbose level + static base::type::VerboseLevel + verboseLevel(void); + /// @brief Sets vmodules as specified (on the fly) + static void + setVModules(const char* modules); + /// @brief Clears vmodules + static void + clearVModules(void); +}; +class VersionInfo : base::StaticClass { + public: + /// @brief Current version number + static const std::string + version(void); + + /// @brief Release date of current version + static const std::string + releaseDate(void); +}; +} // namespace el +#undef VLOG_IS_ON +/// @brief Determines whether verbose logging is on for specified level current file. +#define VLOG_IS_ON(verboseLevel) (ELPP->vRegistry()->allowed(verboseLevel, __FILE__)) +#undef TIMED_BLOCK +#undef TIMED_SCOPE +#undef TIMED_SCOPE_IF +#undef TIMED_FUNC +#undef TIMED_FUNC_IF +#undef ELPP_MIN_UNIT +#if defined(ELPP_PERFORMANCE_MICROSECONDS) +#define ELPP_MIN_UNIT el::base::TimestampUnit::Microsecond +#else +#define ELPP_MIN_UNIT el::base::TimestampUnit::Millisecond +#endif // (defined(ELPP_PERFORMANCE_MICROSECONDS)) +/// @brief Performance tracked scope. Performance gets written when goes out of scope using +/// 'performance' logger. +/// +/// @detail Please note in order to check the performance at a certain time you can use obj->checkpoint(); +/// @see el::base::PerformanceTracker +/// @see el::base::PerformanceTracker::checkpoint +// Note: Do not surround this definition with null macro because of obj instance +#define TIMED_SCOPE_IF(obj, blockname, condition) \ + el::base::type::PerformanceTrackerPtr obj(condition ? new el::base::PerformanceTracker(blockname, ELPP_MIN_UNIT) \ + : nullptr) +#define TIMED_SCOPE(obj, blockname) TIMED_SCOPE_IF(obj, blockname, true) +#define TIMED_BLOCK(obj, blockName) \ + for (struct { \ + int i; \ + el::base::type::PerformanceTrackerPtr timer; \ + } obj = {0, \ + el::base::type::PerformanceTrackerPtr(new el::base::PerformanceTracker(blockName, ELPP_MIN_UNIT))}; \ + obj.i < 1; ++obj.i) +/// @brief Performance tracked function. Performance gets written when goes out of scope using +/// 'performance' logger. +/// +/// @detail Please note in order to check the performance at a certain time you can use obj->checkpoint(); +/// @see el::base::PerformanceTracker +/// @see el::base::PerformanceTracker::checkpoint +#define TIMED_FUNC_IF(obj, condition) TIMED_SCOPE_IF(obj, ELPP_FUNC, condition) +#define TIMED_FUNC(obj) TIMED_SCOPE(obj, ELPP_FUNC) +#undef PERFORMANCE_CHECKPOINT +#undef PERFORMANCE_CHECKPOINT_WITH_ID +#define PERFORMANCE_CHECKPOINT(obj) obj->checkpoint(std::string(), __FILE__, __LINE__, ELPP_FUNC) +#define PERFORMANCE_CHECKPOINT_WITH_ID(obj, id) obj->checkpoint(id, __FILE__, __LINE__, ELPP_FUNC) +#undef ELPP_COUNTER +#undef ELPP_COUNTER_POS +/// @brief Gets hit counter for file/line +#define ELPP_COUNTER (ELPP->hitCounters()->getCounter(__FILE__, __LINE__)) +/// @brief Gets hit counter position for file/line, -1 if not registered yet +#define ELPP_COUNTER_POS (ELPP_COUNTER == nullptr ? -1 : ELPP_COUNTER->hitCounts()) +// Undef levels to support LOG(LEVEL) +#undef INFO +#undef WARNING +#undef DEBUG +#undef ERROR +#undef FATAL +#undef TRACE +#undef VERBOSE +// Undef existing +#undef CINFO +#undef CWARNING +#undef CDEBUG +#undef CFATAL +#undef CERROR +#undef CTRACE +#undef CVERBOSE +#undef CINFO_IF +#undef CWARNING_IF +#undef CDEBUG_IF +#undef CERROR_IF +#undef CFATAL_IF +#undef CTRACE_IF +#undef CVERBOSE_IF +#undef CINFO_EVERY_N +#undef CWARNING_EVERY_N +#undef CDEBUG_EVERY_N +#undef CERROR_EVERY_N +#undef CFATAL_EVERY_N +#undef CTRACE_EVERY_N +#undef CVERBOSE_EVERY_N +#undef CINFO_AFTER_N +#undef CWARNING_AFTER_N +#undef CDEBUG_AFTER_N +#undef CERROR_AFTER_N +#undef CFATAL_AFTER_N +#undef CTRACE_AFTER_N +#undef CVERBOSE_AFTER_N +#undef CINFO_N_TIMES +#undef CWARNING_N_TIMES +#undef CDEBUG_N_TIMES +#undef CERROR_N_TIMES +#undef CFATAL_N_TIMES +#undef CTRACE_N_TIMES +#undef CVERBOSE_N_TIMES +// Normal logs +#if ELPP_INFO_LOG +#define CINFO(writer, dispatchAction, ...) ELPP_WRITE_LOG(writer, el::Level::Info, dispatchAction, __VA_ARGS__) +#else +#define CINFO(writer, dispatchAction, ...) el::base::NullWriter() +#endif // ELPP_INFO_LOG +#if ELPP_WARNING_LOG +#define CWARNING(writer, dispatchAction, ...) ELPP_WRITE_LOG(writer, el::Level::Warning, dispatchAction, __VA_ARGS__) +#else +#define CWARNING(writer, dispatchAction, ...) el::base::NullWriter() +#endif // ELPP_WARNING_LOG +#if ELPP_DEBUG_LOG +#define CDEBUG(writer, dispatchAction, ...) ELPP_WRITE_LOG(writer, el::Level::Debug, dispatchAction, __VA_ARGS__) +#else +#define CDEBUG(writer, dispatchAction, ...) el::base::NullWriter() +#endif // ELPP_DEBUG_LOG +#if ELPP_ERROR_LOG +#define CERROR(writer, dispatchAction, ...) ELPP_WRITE_LOG(writer, el::Level::Error, dispatchAction, __VA_ARGS__) +#else +#define CERROR(writer, dispatchAction, ...) el::base::NullWriter() +#endif // ELPP_ERROR_LOG +#if ELPP_FATAL_LOG +#define CFATAL(writer, dispatchAction, ...) ELPP_WRITE_LOG(writer, el::Level::Fatal, dispatchAction, __VA_ARGS__) +#else +#define CFATAL(writer, dispatchAction, ...) el::base::NullWriter() +#endif // ELPP_FATAL_LOG +#if ELPP_TRACE_LOG +#define CTRACE(writer, dispatchAction, ...) ELPP_WRITE_LOG(writer, el::Level::Trace, dispatchAction, __VA_ARGS__) +#else +#define CTRACE(writer, dispatchAction, ...) el::base::NullWriter() +#endif // ELPP_TRACE_LOG +#if ELPP_VERBOSE_LOG +#define CVERBOSE(writer, vlevel, dispatchAction, ...) \ + if (VLOG_IS_ON(vlevel)) \ + writer(el::Level::Verbose, __FILE__, __LINE__, ELPP_FUNC, dispatchAction, vlevel) \ + .construct(el_getVALength(__VA_ARGS__), __VA_ARGS__) +#else +#define CVERBOSE(writer, vlevel, dispatchAction, ...) el::base::NullWriter() +#endif // ELPP_VERBOSE_LOG +// Conditional logs +#if ELPP_INFO_LOG +#define CINFO_IF(writer, condition_, dispatchAction, ...) \ + ELPP_WRITE_LOG_IF(writer, (condition_), el::Level::Info, dispatchAction, __VA_ARGS__) +#else +#define CINFO_IF(writer, condition_, dispatchAction, ...) el::base::NullWriter() +#endif // ELPP_INFO_LOG +#if ELPP_WARNING_LOG +#define CWARNING_IF(writer, condition_, dispatchAction, ...) \ + ELPP_WRITE_LOG_IF(writer, (condition_), el::Level::Warning, dispatchAction, __VA_ARGS__) +#else +#define CWARNING_IF(writer, condition_, dispatchAction, ...) el::base::NullWriter() +#endif // ELPP_WARNING_LOG +#if ELPP_DEBUG_LOG +#define CDEBUG_IF(writer, condition_, dispatchAction, ...) \ + ELPP_WRITE_LOG_IF(writer, (condition_), el::Level::Debug, dispatchAction, __VA_ARGS__) +#else +#define CDEBUG_IF(writer, condition_, dispatchAction, ...) el::base::NullWriter() +#endif // ELPP_DEBUG_LOG +#if ELPP_ERROR_LOG +#define CERROR_IF(writer, condition_, dispatchAction, ...) \ + ELPP_WRITE_LOG_IF(writer, (condition_), el::Level::Error, dispatchAction, __VA_ARGS__) +#else +#define CERROR_IF(writer, condition_, dispatchAction, ...) el::base::NullWriter() +#endif // ELPP_ERROR_LOG +#if ELPP_FATAL_LOG +#define CFATAL_IF(writer, condition_, dispatchAction, ...) \ + ELPP_WRITE_LOG_IF(writer, (condition_), el::Level::Fatal, dispatchAction, __VA_ARGS__) +#else +#define CFATAL_IF(writer, condition_, dispatchAction, ...) el::base::NullWriter() +#endif // ELPP_FATAL_LOG +#if ELPP_TRACE_LOG +#define CTRACE_IF(writer, condition_, dispatchAction, ...) \ + ELPP_WRITE_LOG_IF(writer, (condition_), el::Level::Trace, dispatchAction, __VA_ARGS__) +#else +#define CTRACE_IF(writer, condition_, dispatchAction, ...) el::base::NullWriter() +#endif // ELPP_TRACE_LOG +#if ELPP_VERBOSE_LOG +#define CVERBOSE_IF(writer, condition_, vlevel, dispatchAction, ...) \ + if (VLOG_IS_ON(vlevel) && (condition_)) \ + writer(el::Level::Verbose, __FILE__, __LINE__, ELPP_FUNC, dispatchAction, vlevel) \ + .construct(el_getVALength(__VA_ARGS__), __VA_ARGS__) +#else +#define CVERBOSE_IF(writer, condition_, vlevel, dispatchAction, ...) el::base::NullWriter() +#endif // ELPP_VERBOSE_LOG +// Occasional logs +#if ELPP_INFO_LOG +#define CINFO_EVERY_N(writer, occasion, dispatchAction, ...) \ + ELPP_WRITE_LOG_EVERY_N(writer, occasion, el::Level::Info, dispatchAction, __VA_ARGS__) +#else +#define CINFO_EVERY_N(writer, occasion, dispatchAction, ...) el::base::NullWriter() +#endif // ELPP_INFO_LOG +#if ELPP_WARNING_LOG +#define CWARNING_EVERY_N(writer, occasion, dispatchAction, ...) \ + ELPP_WRITE_LOG_EVERY_N(writer, occasion, el::Level::Warning, dispatchAction, __VA_ARGS__) +#else +#define CWARNING_EVERY_N(writer, occasion, dispatchAction, ...) el::base::NullWriter() +#endif // ELPP_WARNING_LOG +#if ELPP_DEBUG_LOG +#define CDEBUG_EVERY_N(writer, occasion, dispatchAction, ...) \ + ELPP_WRITE_LOG_EVERY_N(writer, occasion, el::Level::Debug, dispatchAction, __VA_ARGS__) +#else +#define CDEBUG_EVERY_N(writer, occasion, dispatchAction, ...) el::base::NullWriter() +#endif // ELPP_DEBUG_LOG +#if ELPP_ERROR_LOG +#define CERROR_EVERY_N(writer, occasion, dispatchAction, ...) \ + ELPP_WRITE_LOG_EVERY_N(writer, occasion, el::Level::Error, dispatchAction, __VA_ARGS__) +#else +#define CERROR_EVERY_N(writer, occasion, dispatchAction, ...) el::base::NullWriter() +#endif // ELPP_ERROR_LOG +#if ELPP_FATAL_LOG +#define CFATAL_EVERY_N(writer, occasion, dispatchAction, ...) \ + ELPP_WRITE_LOG_EVERY_N(writer, occasion, el::Level::Fatal, dispatchAction, __VA_ARGS__) +#else +#define CFATAL_EVERY_N(writer, occasion, dispatchAction, ...) el::base::NullWriter() +#endif // ELPP_FATAL_LOG +#if ELPP_TRACE_LOG +#define CTRACE_EVERY_N(writer, occasion, dispatchAction, ...) \ + ELPP_WRITE_LOG_EVERY_N(writer, occasion, el::Level::Trace, dispatchAction, __VA_ARGS__) +#else +#define CTRACE_EVERY_N(writer, occasion, dispatchAction, ...) el::base::NullWriter() +#endif // ELPP_TRACE_LOG +#if ELPP_VERBOSE_LOG +#define CVERBOSE_EVERY_N(writer, occasion, vlevel, dispatchAction, ...) \ + CVERBOSE_IF(writer, ELPP->validateEveryNCounter(__FILE__, __LINE__, occasion), vlevel, dispatchAction, __VA_ARGS__) +#else +#define CVERBOSE_EVERY_N(writer, occasion, vlevel, dispatchAction, ...) el::base::NullWriter() +#endif // ELPP_VERBOSE_LOG +// After N logs +#if ELPP_INFO_LOG +#define CINFO_AFTER_N(writer, n, dispatchAction, ...) \ + ELPP_WRITE_LOG_AFTER_N(writer, n, el::Level::Info, dispatchAction, __VA_ARGS__) +#else +#define CINFO_AFTER_N(writer, n, dispatchAction, ...) el::base::NullWriter() +#endif // ELPP_INFO_LOG +#if ELPP_WARNING_LOG +#define CWARNING_AFTER_N(writer, n, dispatchAction, ...) \ + ELPP_WRITE_LOG_AFTER_N(writer, n, el::Level::Warning, dispatchAction, __VA_ARGS__) +#else +#define CWARNING_AFTER_N(writer, n, dispatchAction, ...) el::base::NullWriter() +#endif // ELPP_WARNING_LOG +#if ELPP_DEBUG_LOG +#define CDEBUG_AFTER_N(writer, n, dispatchAction, ...) \ + ELPP_WRITE_LOG_AFTER_N(writer, n, el::Level::Debug, dispatchAction, __VA_ARGS__) +#else +#define CDEBUG_AFTER_N(writer, n, dispatchAction, ...) el::base::NullWriter() +#endif // ELPP_DEBUG_LOG +#if ELPP_ERROR_LOG +#define CERROR_AFTER_N(writer, n, dispatchAction, ...) \ + ELPP_WRITE_LOG_AFTER_N(writer, n, el::Level::Error, dispatchAction, __VA_ARGS__) +#else +#define CERROR_AFTER_N(writer, n, dispatchAction, ...) el::base::NullWriter() +#endif // ELPP_ERROR_LOG +#if ELPP_FATAL_LOG +#define CFATAL_AFTER_N(writer, n, dispatchAction, ...) \ + ELPP_WRITE_LOG_AFTER_N(writer, n, el::Level::Fatal, dispatchAction, __VA_ARGS__) +#else +#define CFATAL_AFTER_N(writer, n, dispatchAction, ...) el::base::NullWriter() +#endif // ELPP_FATAL_LOG +#if ELPP_TRACE_LOG +#define CTRACE_AFTER_N(writer, n, dispatchAction, ...) \ + ELPP_WRITE_LOG_AFTER_N(writer, n, el::Level::Trace, dispatchAction, __VA_ARGS__) +#else +#define CTRACE_AFTER_N(writer, n, dispatchAction, ...) el::base::NullWriter() +#endif // ELPP_TRACE_LOG +#if ELPP_VERBOSE_LOG +#define CVERBOSE_AFTER_N(writer, n, vlevel, dispatchAction, ...) \ + CVERBOSE_IF(writer, ELPP->validateAfterNCounter(__FILE__, __LINE__, n), vlevel, dispatchAction, __VA_ARGS__) +#else +#define CVERBOSE_AFTER_N(writer, n, vlevel, dispatchAction, ...) el::base::NullWriter() +#endif // ELPP_VERBOSE_LOG +// N Times logs +#if ELPP_INFO_LOG +#define CINFO_N_TIMES(writer, n, dispatchAction, ...) \ + ELPP_WRITE_LOG_N_TIMES(writer, n, el::Level::Info, dispatchAction, __VA_ARGS__) +#else +#define CINFO_N_TIMES(writer, n, dispatchAction, ...) el::base::NullWriter() +#endif // ELPP_INFO_LOG +#if ELPP_WARNING_LOG +#define CWARNING_N_TIMES(writer, n, dispatchAction, ...) \ + ELPP_WRITE_LOG_N_TIMES(writer, n, el::Level::Warning, dispatchAction, __VA_ARGS__) +#else +#define CWARNING_N_TIMES(writer, n, dispatchAction, ...) el::base::NullWriter() +#endif // ELPP_WARNING_LOG +#if ELPP_DEBUG_LOG +#define CDEBUG_N_TIMES(writer, n, dispatchAction, ...) \ + ELPP_WRITE_LOG_N_TIMES(writer, n, el::Level::Debug, dispatchAction, __VA_ARGS__) +#else +#define CDEBUG_N_TIMES(writer, n, dispatchAction, ...) el::base::NullWriter() +#endif // ELPP_DEBUG_LOG +#if ELPP_ERROR_LOG +#define CERROR_N_TIMES(writer, n, dispatchAction, ...) \ + ELPP_WRITE_LOG_N_TIMES(writer, n, el::Level::Error, dispatchAction, __VA_ARGS__) +#else +#define CERROR_N_TIMES(writer, n, dispatchAction, ...) el::base::NullWriter() +#endif // ELPP_ERROR_LOG +#if ELPP_FATAL_LOG +#define CFATAL_N_TIMES(writer, n, dispatchAction, ...) \ + ELPP_WRITE_LOG_N_TIMES(writer, n, el::Level::Fatal, dispatchAction, __VA_ARGS__) +#else +#define CFATAL_N_TIMES(writer, n, dispatchAction, ...) el::base::NullWriter() +#endif // ELPP_FATAL_LOG +#if ELPP_TRACE_LOG +#define CTRACE_N_TIMES(writer, n, dispatchAction, ...) \ + ELPP_WRITE_LOG_N_TIMES(writer, n, el::Level::Trace, dispatchAction, __VA_ARGS__) +#else +#define CTRACE_N_TIMES(writer, n, dispatchAction, ...) el::base::NullWriter() +#endif // ELPP_TRACE_LOG +#if ELPP_VERBOSE_LOG +#define CVERBOSE_N_TIMES(writer, n, vlevel, dispatchAction, ...) \ + CVERBOSE_IF(writer, ELPP->validateNTimesCounter(__FILE__, __LINE__, n), vlevel, dispatchAction, __VA_ARGS__) +#else +#define CVERBOSE_N_TIMES(writer, n, vlevel, dispatchAction, ...) el::base::NullWriter() +#endif // ELPP_VERBOSE_LOG +// +// Custom Loggers - Requires (level, dispatchAction, loggerId/s) +// +// undef existing +#undef CLOG +#undef CLOG_VERBOSE +#undef CVLOG +#undef CLOG_IF +#undef CLOG_VERBOSE_IF +#undef CVLOG_IF +#undef CLOG_EVERY_N +#undef CVLOG_EVERY_N +#undef CLOG_AFTER_N +#undef CVLOG_AFTER_N +#undef CLOG_N_TIMES +#undef CVLOG_N_TIMES +// Normal logs +#define CLOG(LEVEL, ...) C##LEVEL(el::base::Writer, el::base::DispatchAction::NormalLog, __VA_ARGS__) +#define CVLOG(vlevel, ...) CVERBOSE(el::base::Writer, vlevel, el::base::DispatchAction::NormalLog, __VA_ARGS__) +// Conditional logs +#define CLOG_IF(condition, LEVEL, ...) \ + C##LEVEL##_IF(el::base::Writer, condition, el::base::DispatchAction::NormalLog, __VA_ARGS__) +#define CVLOG_IF(condition, vlevel, ...) \ + CVERBOSE_IF(el::base::Writer, condition, vlevel, el::base::DispatchAction::NormalLog, __VA_ARGS__) +// Hit counts based logs +#define CLOG_EVERY_N(n, LEVEL, ...) \ + C##LEVEL##_EVERY_N(el::base::Writer, n, el::base::DispatchAction::NormalLog, __VA_ARGS__) +#define CVLOG_EVERY_N(n, vlevel, ...) \ + CVERBOSE_EVERY_N(el::base::Writer, n, vlevel, el::base::DispatchAction::NormalLog, __VA_ARGS__) +#define CLOG_AFTER_N(n, LEVEL, ...) \ + C##LEVEL##_AFTER_N(el::base::Writer, n, el::base::DispatchAction::NormalLog, __VA_ARGS__) +#define CVLOG_AFTER_N(n, vlevel, ...) \ + CVERBOSE_AFTER_N(el::base::Writer, n, vlevel, el::base::DispatchAction::NormalLog, __VA_ARGS__) +#define CLOG_N_TIMES(n, LEVEL, ...) \ + C##LEVEL##_N_TIMES(el::base::Writer, n, el::base::DispatchAction::NormalLog, __VA_ARGS__) +#define CVLOG_N_TIMES(n, vlevel, ...) \ + CVERBOSE_N_TIMES(el::base::Writer, n, vlevel, el::base::DispatchAction::NormalLog, __VA_ARGS__) +// +// Default Loggers macro using CLOG(), CLOG_VERBOSE() and CVLOG() macros +// +// undef existing +#undef LOG +#undef VLOG +#undef LOG_IF +#undef VLOG_IF +#undef LOG_EVERY_N +#undef VLOG_EVERY_N +#undef LOG_AFTER_N +#undef VLOG_AFTER_N +#undef LOG_N_TIMES +#undef VLOG_N_TIMES +#undef ELPP_CURR_FILE_LOGGER_ID +#if defined(ELPP_DEFAULT_LOGGER) +#define ELPP_CURR_FILE_LOGGER_ID ELPP_DEFAULT_LOGGER +#else +#define ELPP_CURR_FILE_LOGGER_ID el::base::consts::kDefaultLoggerId +#endif +#undef ELPP_TRACE +#define ELPP_TRACE CLOG(TRACE, ELPP_CURR_FILE_LOGGER_ID) +// Normal logs +#define LOG(LEVEL) CLOG(LEVEL, ELPP_CURR_FILE_LOGGER_ID) +#define VLOG(vlevel) CVLOG(vlevel, ELPP_CURR_FILE_LOGGER_ID) +// Conditional logs +#define LOG_IF(condition, LEVEL) CLOG_IF(condition, LEVEL, ELPP_CURR_FILE_LOGGER_ID) +#define VLOG_IF(condition, vlevel) CVLOG_IF(condition, vlevel, ELPP_CURR_FILE_LOGGER_ID) +// Hit counts based logs +#define LOG_EVERY_N(n, LEVEL) CLOG_EVERY_N(n, LEVEL, ELPP_CURR_FILE_LOGGER_ID) +#define VLOG_EVERY_N(n, vlevel) CVLOG_EVERY_N(n, vlevel, ELPP_CURR_FILE_LOGGER_ID) +#define LOG_AFTER_N(n, LEVEL) CLOG_AFTER_N(n, LEVEL, ELPP_CURR_FILE_LOGGER_ID) +#define VLOG_AFTER_N(n, vlevel) CVLOG_AFTER_N(n, vlevel, ELPP_CURR_FILE_LOGGER_ID) +#define LOG_N_TIMES(n, LEVEL) CLOG_N_TIMES(n, LEVEL, ELPP_CURR_FILE_LOGGER_ID) +#define VLOG_N_TIMES(n, vlevel) CVLOG_N_TIMES(n, vlevel, ELPP_CURR_FILE_LOGGER_ID) +// Generic PLOG() +#undef CPLOG +#undef CPLOG_IF +#undef PLOG +#undef PLOG_IF +#undef DCPLOG +#undef DCPLOG_IF +#undef DPLOG +#undef DPLOG_IF +#define CPLOG(LEVEL, ...) C##LEVEL(el::base::PErrorWriter, el::base::DispatchAction::NormalLog, __VA_ARGS__) +#define CPLOG_IF(condition, LEVEL, ...) \ + C##LEVEL##_IF(el::base::PErrorWriter, condition, el::base::DispatchAction::NormalLog, __VA_ARGS__) +#define DCPLOG(LEVEL, ...) \ + if (ELPP_DEBUG_LOG) \ + C##LEVEL(el::base::PErrorWriter, el::base::DispatchAction::NormalLog, __VA_ARGS__) +#define DCPLOG_IF(condition, LEVEL, ...) \ + C##LEVEL##_IF(el::base::PErrorWriter, (ELPP_DEBUG_LOG) && (condition), el::base::DispatchAction::NormalLog, \ + __VA_ARGS__) +#define PLOG(LEVEL) CPLOG(LEVEL, ELPP_CURR_FILE_LOGGER_ID) +#define PLOG_IF(condition, LEVEL) CPLOG_IF(condition, LEVEL, ELPP_CURR_FILE_LOGGER_ID) +#define DPLOG(LEVEL) DCPLOG(LEVEL, ELPP_CURR_FILE_LOGGER_ID) +#define DPLOG_IF(condition, LEVEL) DCPLOG_IF(condition, LEVEL, ELPP_CURR_FILE_LOGGER_ID) +// Generic SYSLOG() +#undef CSYSLOG +#undef CSYSLOG_IF +#undef CSYSLOG_EVERY_N +#undef CSYSLOG_AFTER_N +#undef CSYSLOG_N_TIMES +#undef SYSLOG +#undef SYSLOG_IF +#undef SYSLOG_EVERY_N +#undef SYSLOG_AFTER_N +#undef SYSLOG_N_TIMES +#undef DCSYSLOG +#undef DCSYSLOG_IF +#undef DCSYSLOG_EVERY_N +#undef DCSYSLOG_AFTER_N +#undef DCSYSLOG_N_TIMES +#undef DSYSLOG +#undef DSYSLOG_IF +#undef DSYSLOG_EVERY_N +#undef DSYSLOG_AFTER_N +#undef DSYSLOG_N_TIMES +#if defined(ELPP_SYSLOG) +#define CSYSLOG(LEVEL, ...) C##LEVEL(el::base::Writer, el::base::DispatchAction::SysLog, __VA_ARGS__) +#define CSYSLOG_IF(condition, LEVEL, ...) \ + C##LEVEL##_IF(el::base::Writer, condition, el::base::DispatchAction::SysLog, __VA_ARGS__) +#define CSYSLOG_EVERY_N(n, LEVEL, ...) \ + C##LEVEL##_EVERY_N(el::base::Writer, n, el::base::DispatchAction::SysLog, __VA_ARGS__) +#define CSYSLOG_AFTER_N(n, LEVEL, ...) \ + C##LEVEL##_AFTER_N(el::base::Writer, n, el::base::DispatchAction::SysLog, __VA_ARGS__) +#define CSYSLOG_N_TIMES(n, LEVEL, ...) \ + C##LEVEL##_N_TIMES(el::base::Writer, n, el::base::DispatchAction::SysLog, __VA_ARGS__) +#define SYSLOG(LEVEL) CSYSLOG(LEVEL, el::base::consts::kSysLogLoggerId) +#define SYSLOG_IF(condition, LEVEL) CSYSLOG_IF(condition, LEVEL, el::base::consts::kSysLogLoggerId) +#define SYSLOG_EVERY_N(n, LEVEL) CSYSLOG_EVERY_N(n, LEVEL, el::base::consts::kSysLogLoggerId) +#define SYSLOG_AFTER_N(n, LEVEL) CSYSLOG_AFTER_N(n, LEVEL, el::base::consts::kSysLogLoggerId) +#define SYSLOG_N_TIMES(n, LEVEL) CSYSLOG_N_TIMES(n, LEVEL, el::base::consts::kSysLogLoggerId) +#define DCSYSLOG(LEVEL, ...) \ + if (ELPP_DEBUG_LOG) \ + C##LEVEL(el::base::Writer, el::base::DispatchAction::SysLog, __VA_ARGS__) +#define DCSYSLOG_IF(condition, LEVEL, ...) \ + C##LEVEL##_IF(el::base::Writer, (ELPP_DEBUG_LOG) && (condition), el::base::DispatchAction::SysLog, __VA_ARGS__) +#define DCSYSLOG_EVERY_N(n, LEVEL, ...) \ + if (ELPP_DEBUG_LOG) \ + C##LEVEL##_EVERY_N(el::base::Writer, n, el::base::DispatchAction::SysLog, __VA_ARGS__) +#define DCSYSLOG_AFTER_N(n, LEVEL, ...) \ + if (ELPP_DEBUG_LOG) \ + C##LEVEL##_AFTER_N(el::base::Writer, n, el::base::DispatchAction::SysLog, __VA_ARGS__) +#define DCSYSLOG_N_TIMES(n, LEVEL, ...) \ + if (ELPP_DEBUG_LOG) \ + C##LEVEL##_EVERY_N(el::base::Writer, n, el::base::DispatchAction::SysLog, __VA_ARGS__) +#define DSYSLOG(LEVEL) DCSYSLOG(LEVEL, el::base::consts::kSysLogLoggerId) +#define DSYSLOG_IF(condition, LEVEL) DCSYSLOG_IF(condition, LEVEL, el::base::consts::kSysLogLoggerId) +#define DSYSLOG_EVERY_N(n, LEVEL) DCSYSLOG_EVERY_N(n, LEVEL, el::base::consts::kSysLogLoggerId) +#define DSYSLOG_AFTER_N(n, LEVEL) DCSYSLOG_AFTER_N(n, LEVEL, el::base::consts::kSysLogLoggerId) +#define DSYSLOG_N_TIMES(n, LEVEL) DCSYSLOG_N_TIMES(n, LEVEL, el::base::consts::kSysLogLoggerId) +#else +#define CSYSLOG(LEVEL, ...) el::base::NullWriter() +#define CSYSLOG_IF(condition, LEVEL, ...) el::base::NullWriter() +#define CSYSLOG_EVERY_N(n, LEVEL, ...) el::base::NullWriter() +#define CSYSLOG_AFTER_N(n, LEVEL, ...) el::base::NullWriter() +#define CSYSLOG_N_TIMES(n, LEVEL, ...) el::base::NullWriter() +#define SYSLOG(LEVEL) el::base::NullWriter() +#define SYSLOG_IF(condition, LEVEL) el::base::NullWriter() +#define SYSLOG_EVERY_N(n, LEVEL) el::base::NullWriter() +#define SYSLOG_AFTER_N(n, LEVEL) el::base::NullWriter() +#define SYSLOG_N_TIMES(n, LEVEL) el::base::NullWriter() +#define DCSYSLOG(LEVEL, ...) el::base::NullWriter() +#define DCSYSLOG_IF(condition, LEVEL, ...) el::base::NullWriter() +#define DCSYSLOG_EVERY_N(n, LEVEL, ...) el::base::NullWriter() +#define DCSYSLOG_AFTER_N(n, LEVEL, ...) el::base::NullWriter() +#define DCSYSLOG_N_TIMES(n, LEVEL, ...) el::base::NullWriter() +#define DSYSLOG(LEVEL) el::base::NullWriter() +#define DSYSLOG_IF(condition, LEVEL) el::base::NullWriter() +#define DSYSLOG_EVERY_N(n, LEVEL) el::base::NullWriter() +#define DSYSLOG_AFTER_N(n, LEVEL) el::base::NullWriter() +#define DSYSLOG_N_TIMES(n, LEVEL) el::base::NullWriter() +#endif // defined(ELPP_SYSLOG) +// +// Custom Debug Only Loggers - Requires (level, loggerId/s) +// +// undef existing +#undef DCLOG +#undef DCVLOG +#undef DCLOG_IF +#undef DCVLOG_IF +#undef DCLOG_EVERY_N +#undef DCVLOG_EVERY_N +#undef DCLOG_AFTER_N +#undef DCVLOG_AFTER_N +#undef DCLOG_N_TIMES +#undef DCVLOG_N_TIMES +// Normal logs +#define DCLOG(LEVEL, ...) \ + if (ELPP_DEBUG_LOG) \ + CLOG(LEVEL, __VA_ARGS__) +#define DCLOG_VERBOSE(vlevel, ...) \ + if (ELPP_DEBUG_LOG) \ + CLOG_VERBOSE(vlevel, __VA_ARGS__) +#define DCVLOG(vlevel, ...) \ + if (ELPP_DEBUG_LOG) \ + CVLOG(vlevel, __VA_ARGS__) +// Conditional logs +#define DCLOG_IF(condition, LEVEL, ...) \ + if (ELPP_DEBUG_LOG) \ + CLOG_IF(condition, LEVEL, __VA_ARGS__) +#define DCVLOG_IF(condition, vlevel, ...) \ + if (ELPP_DEBUG_LOG) \ + CVLOG_IF(condition, vlevel, __VA_ARGS__) +// Hit counts based logs +#define DCLOG_EVERY_N(n, LEVEL, ...) \ + if (ELPP_DEBUG_LOG) \ + CLOG_EVERY_N(n, LEVEL, __VA_ARGS__) +#define DCVLOG_EVERY_N(n, vlevel, ...) \ + if (ELPP_DEBUG_LOG) \ + CVLOG_EVERY_N(n, vlevel, __VA_ARGS__) +#define DCLOG_AFTER_N(n, LEVEL, ...) \ + if (ELPP_DEBUG_LOG) \ + CLOG_AFTER_N(n, LEVEL, __VA_ARGS__) +#define DCVLOG_AFTER_N(n, vlevel, ...) \ + if (ELPP_DEBUG_LOG) \ + CVLOG_AFTER_N(n, vlevel, __VA_ARGS__) +#define DCLOG_N_TIMES(n, LEVEL, ...) \ + if (ELPP_DEBUG_LOG) \ + CLOG_N_TIMES(n, LEVEL, __VA_ARGS__) +#define DCVLOG_N_TIMES(n, vlevel, ...) \ + if (ELPP_DEBUG_LOG) \ + CVLOG_N_TIMES(n, vlevel, __VA_ARGS__) +// +// Default Debug Only Loggers macro using CLOG(), CLOG_VERBOSE() and CVLOG() macros +// +#if !defined(ELPP_NO_DEBUG_MACROS) +// undef existing +#undef DLOG +#undef DVLOG +#undef DLOG_IF +#undef DVLOG_IF +#undef DLOG_EVERY_N +#undef DVLOG_EVERY_N +#undef DLOG_AFTER_N +#undef DVLOG_AFTER_N +#undef DLOG_N_TIMES +#undef DVLOG_N_TIMES +// Normal logs +#define DLOG(LEVEL) DCLOG(LEVEL, ELPP_CURR_FILE_LOGGER_ID) +#define DVLOG(vlevel) DCVLOG(vlevel, ELPP_CURR_FILE_LOGGER_ID) +// Conditional logs +#define DLOG_IF(condition, LEVEL) DCLOG_IF(condition, LEVEL, ELPP_CURR_FILE_LOGGER_ID) +#define DVLOG_IF(condition, vlevel) DCVLOG_IF(condition, vlevel, ELPP_CURR_FILE_LOGGER_ID) +// Hit counts based logs +#define DLOG_EVERY_N(n, LEVEL) DCLOG_EVERY_N(n, LEVEL, ELPP_CURR_FILE_LOGGER_ID) +#define DVLOG_EVERY_N(n, vlevel) DCVLOG_EVERY_N(n, vlevel, ELPP_CURR_FILE_LOGGER_ID) +#define DLOG_AFTER_N(n, LEVEL) DCLOG_AFTER_N(n, LEVEL, ELPP_CURR_FILE_LOGGER_ID) +#define DVLOG_AFTER_N(n, vlevel) DCVLOG_AFTER_N(n, vlevel, ELPP_CURR_FILE_LOGGER_ID) +#define DLOG_N_TIMES(n, LEVEL) DCLOG_N_TIMES(n, LEVEL, ELPP_CURR_FILE_LOGGER_ID) +#define DVLOG_N_TIMES(n, vlevel) DCVLOG_N_TIMES(n, vlevel, ELPP_CURR_FILE_LOGGER_ID) +#endif // defined(ELPP_NO_DEBUG_MACROS) +#if !defined(ELPP_NO_CHECK_MACROS) +// Check macros +#undef CCHECK +#undef CPCHECK +#undef CCHECK_EQ +#undef CCHECK_NE +#undef CCHECK_LT +#undef CCHECK_GT +#undef CCHECK_LE +#undef CCHECK_GE +#undef CCHECK_BOUNDS +#undef CCHECK_NOTNULL +#undef CCHECK_STRCASEEQ +#undef CCHECK_STRCASENE +#undef CHECK +#undef PCHECK +#undef CHECK_EQ +#undef CHECK_NE +#undef CHECK_LT +#undef CHECK_GT +#undef CHECK_LE +#undef CHECK_GE +#undef CHECK_BOUNDS +#undef CHECK_NOTNULL +#undef CHECK_STRCASEEQ +#undef CHECK_STRCASENE +#define CCHECK(condition, ...) CLOG_IF(!(condition), FATAL, __VA_ARGS__) << "Check failed: [" << #condition << "] " +#define CPCHECK(condition, ...) CPLOG_IF(!(condition), FATAL, __VA_ARGS__) << "Check failed: [" << #condition << "] " +#define CHECK(condition) CCHECK(condition, ELPP_CURR_FILE_LOGGER_ID) +#define PCHECK(condition) CPCHECK(condition, ELPP_CURR_FILE_LOGGER_ID) +#define CCHECK_EQ(a, b, ...) CCHECK(a == b, __VA_ARGS__) +#define CCHECK_NE(a, b, ...) CCHECK(a != b, __VA_ARGS__) +#define CCHECK_LT(a, b, ...) CCHECK(a < b, __VA_ARGS__) +#define CCHECK_GT(a, b, ...) CCHECK(a > b, __VA_ARGS__) +#define CCHECK_LE(a, b, ...) CCHECK(a <= b, __VA_ARGS__) +#define CCHECK_GE(a, b, ...) CCHECK(a >= b, __VA_ARGS__) +#define CCHECK_BOUNDS(val, min, max, ...) CCHECK(val >= min && val <= max, __VA_ARGS__) +#define CHECK_EQ(a, b) CCHECK_EQ(a, b, ELPP_CURR_FILE_LOGGER_ID) +#define CHECK_NE(a, b) CCHECK_NE(a, b, ELPP_CURR_FILE_LOGGER_ID) +#define CHECK_LT(a, b) CCHECK_LT(a, b, ELPP_CURR_FILE_LOGGER_ID) +#define CHECK_GT(a, b) CCHECK_GT(a, b, ELPP_CURR_FILE_LOGGER_ID) +#define CHECK_LE(a, b) CCHECK_LE(a, b, ELPP_CURR_FILE_LOGGER_ID) +#define CHECK_GE(a, b) CCHECK_GE(a, b, ELPP_CURR_FILE_LOGGER_ID) +#define CHECK_BOUNDS(val, min, max) CCHECK_BOUNDS(val, min, max, ELPP_CURR_FILE_LOGGER_ID) +#define CCHECK_NOTNULL(ptr, ...) CCHECK((ptr) != nullptr, __VA_ARGS__) +#define CCHECK_STREQ(str1, str2, ...) \ + CLOG_IF(!el::base::utils::Str::cStringEq(str1, str2), FATAL, __VA_ARGS__) \ + << "Check failed: [" << #str1 << " == " << #str2 << "] " +#define CCHECK_STRNE(str1, str2, ...) \ + CLOG_IF(el::base::utils::Str::cStringEq(str1, str2), FATAL, __VA_ARGS__) \ + << "Check failed: [" << #str1 << " != " << #str2 << "] " +#define CCHECK_STRCASEEQ(str1, str2, ...) \ + CLOG_IF(!el::base::utils::Str::cStringCaseEq(str1, str2), FATAL, __VA_ARGS__) \ + << "Check failed: [" << #str1 << " == " << #str2 << "] " +#define CCHECK_STRCASENE(str1, str2, ...) \ + CLOG_IF(el::base::utils::Str::cStringCaseEq(str1, str2), FATAL, __VA_ARGS__) \ + << "Check failed: [" << #str1 << " != " << #str2 << "] " +#define CHECK_NOTNULL(ptr) CCHECK_NOTNULL((ptr), ELPP_CURR_FILE_LOGGER_ID) +#define CHECK_STREQ(str1, str2) CCHECK_STREQ(str1, str2, ELPP_CURR_FILE_LOGGER_ID) +#define CHECK_STRNE(str1, str2) CCHECK_STRNE(str1, str2, ELPP_CURR_FILE_LOGGER_ID) +#define CHECK_STRCASEEQ(str1, str2) CCHECK_STRCASEEQ(str1, str2, ELPP_CURR_FILE_LOGGER_ID) +#define CHECK_STRCASENE(str1, str2) CCHECK_STRCASENE(str1, str2, ELPP_CURR_FILE_LOGGER_ID) +#undef DCCHECK +#undef DCCHECK_EQ +#undef DCCHECK_NE +#undef DCCHECK_LT +#undef DCCHECK_GT +#undef DCCHECK_LE +#undef DCCHECK_GE +#undef DCCHECK_BOUNDS +#undef DCCHECK_NOTNULL +#undef DCCHECK_STRCASEEQ +#undef DCCHECK_STRCASENE +#undef DCPCHECK +#undef DCHECK +#undef DCHECK_EQ +#undef DCHECK_NE +#undef DCHECK_LT +#undef DCHECK_GT +#undef DCHECK_LE +#undef DCHECK_GE +#undef DCHECK_BOUNDS_ +#undef DCHECK_NOTNULL +#undef DCHECK_STRCASEEQ +#undef DCHECK_STRCASENE +#undef DPCHECK +#define DCCHECK(condition, ...) \ + if (ELPP_DEBUG_LOG) \ + CCHECK(condition, __VA_ARGS__) +#define DCCHECK_EQ(a, b, ...) \ + if (ELPP_DEBUG_LOG) \ + CCHECK_EQ(a, b, __VA_ARGS__) +#define DCCHECK_NE(a, b, ...) \ + if (ELPP_DEBUG_LOG) \ + CCHECK_NE(a, b, __VA_ARGS__) +#define DCCHECK_LT(a, b, ...) \ + if (ELPP_DEBUG_LOG) \ + CCHECK_LT(a, b, __VA_ARGS__) +#define DCCHECK_GT(a, b, ...) \ + if (ELPP_DEBUG_LOG) \ + CCHECK_GT(a, b, __VA_ARGS__) +#define DCCHECK_LE(a, b, ...) \ + if (ELPP_DEBUG_LOG) \ + CCHECK_LE(a, b, __VA_ARGS__) +#define DCCHECK_GE(a, b, ...) \ + if (ELPP_DEBUG_LOG) \ + CCHECK_GE(a, b, __VA_ARGS__) +#define DCCHECK_BOUNDS(val, min, max, ...) \ + if (ELPP_DEBUG_LOG) \ + CCHECK_BOUNDS(val, min, max, __VA_ARGS__) +#define DCCHECK_NOTNULL(ptr, ...) \ + if (ELPP_DEBUG_LOG) \ + CCHECK_NOTNULL((ptr), __VA_ARGS__) +#define DCCHECK_STREQ(str1, str2, ...) \ + if (ELPP_DEBUG_LOG) \ + CCHECK_STREQ(str1, str2, __VA_ARGS__) +#define DCCHECK_STRNE(str1, str2, ...) \ + if (ELPP_DEBUG_LOG) \ + CCHECK_STRNE(str1, str2, __VA_ARGS__) +#define DCCHECK_STRCASEEQ(str1, str2, ...) \ + if (ELPP_DEBUG_LOG) \ + CCHECK_STRCASEEQ(str1, str2, __VA_ARGS__) +#define DCCHECK_STRCASENE(str1, str2, ...) \ + if (ELPP_DEBUG_LOG) \ + CCHECK_STRCASENE(str1, str2, __VA_ARGS__) +#define DCPCHECK(condition, ...) \ + if (ELPP_DEBUG_LOG) \ + CPCHECK(condition, __VA_ARGS__) +#define DCHECK(condition) DCCHECK(condition, ELPP_CURR_FILE_LOGGER_ID) +#define DCHECK_EQ(a, b) DCCHECK_EQ(a, b, ELPP_CURR_FILE_LOGGER_ID) +#define DCHECK_NE(a, b) DCCHECK_NE(a, b, ELPP_CURR_FILE_LOGGER_ID) +#define DCHECK_LT(a, b) DCCHECK_LT(a, b, ELPP_CURR_FILE_LOGGER_ID) +#define DCHECK_GT(a, b) DCCHECK_GT(a, b, ELPP_CURR_FILE_LOGGER_ID) +#define DCHECK_LE(a, b) DCCHECK_LE(a, b, ELPP_CURR_FILE_LOGGER_ID) +#define DCHECK_GE(a, b) DCCHECK_GE(a, b, ELPP_CURR_FILE_LOGGER_ID) +#define DCHECK_BOUNDS(val, min, max) DCCHECK_BOUNDS(val, min, max, ELPP_CURR_FILE_LOGGER_ID) +#define DCHECK_NOTNULL(ptr) DCCHECK_NOTNULL((ptr), ELPP_CURR_FILE_LOGGER_ID) +#define DCHECK_STREQ(str1, str2) DCCHECK_STREQ(str1, str2, ELPP_CURR_FILE_LOGGER_ID) +#define DCHECK_STRNE(str1, str2) DCCHECK_STRNE(str1, str2, ELPP_CURR_FILE_LOGGER_ID) +#define DCHECK_STRCASEEQ(str1, str2) DCCHECK_STRCASEEQ(str1, str2, ELPP_CURR_FILE_LOGGER_ID) +#define DCHECK_STRCASENE(str1, str2) DCCHECK_STRCASENE(str1, str2, ELPP_CURR_FILE_LOGGER_ID) +#define DPCHECK(condition) DCPCHECK(condition, ELPP_CURR_FILE_LOGGER_ID) +#endif // defined(ELPP_NO_CHECK_MACROS) +#if defined(ELPP_DISABLE_DEFAULT_CRASH_HANDLING) +#define ELPP_USE_DEF_CRASH_HANDLER false +#else +#define ELPP_USE_DEF_CRASH_HANDLER true +#endif // defined(ELPP_DISABLE_DEFAULT_CRASH_HANDLING) +#define ELPP_CRASH_HANDLER_INIT +#define ELPP_INIT_EASYLOGGINGPP(val) \ + namespace el { \ + namespace base { \ + el::base::type::StoragePointer elStorage(val); \ + } \ + el::base::debug::CrashHandler elCrashHandler(ELPP_USE_DEF_CRASH_HANDLER); \ + } + +#if ELPP_ASYNC_LOGGING +#define INITIALIZE_EASYLOGGINGPP \ + ELPP_INIT_EASYLOGGINGPP(new el::base::Storage(el::LogBuilderPtr(new el::base::DefaultLogBuilder()), \ + new el::base::AsyncDispatchWorker())) +#else +#define INITIALIZE_EASYLOGGINGPP \ + ELPP_INIT_EASYLOGGINGPP(new el::base::Storage(el::LogBuilderPtr(new el::base::DefaultLogBuilder()))) +#endif // ELPP_ASYNC_LOGGING +#define INITIALIZE_NULL_EASYLOGGINGPP \ + namespace el { \ + namespace base { \ + el::base::type::StoragePointer elStorage; \ + } \ + el::base::debug::CrashHandler elCrashHandler(ELPP_USE_DEF_CRASH_HANDLER); \ + } +#define SHARE_EASYLOGGINGPP(initializedStorage) \ + namespace el { \ + namespace base { \ + el::base::type::StoragePointer elStorage(initializedStorage); \ + } \ + el::base::debug::CrashHandler elCrashHandler(ELPP_USE_DEF_CRASH_HANDLER); \ + } + +#if defined(ELPP_UNICODE) +#define START_EASYLOGGINGPP(argc, argv) \ + el::Helpers::setArgs(argc, argv); \ + std::locale::global(std::locale("")) +#else +#define START_EASYLOGGINGPP(argc, argv) el::Helpers::setArgs(argc, argv) +#endif // defined(ELPP_UNICODE) +#endif // EASYLOGGINGPP_H diff --git a/core/src/external/nlohmann/json.hpp b/core/thirdparty/nlohmann/json.hpp similarity index 70% rename from core/src/external/nlohmann/json.hpp rename to core/thirdparty/nlohmann/json.hpp index 2a32a82963..4acdcd3aea 100644 --- a/core/src/external/nlohmann/json.hpp +++ b/core/thirdparty/nlohmann/json.hpp @@ -34,60 +34,53 @@ SOFTWARE. #define NLOHMANN_JSON_VERSION_MINOR 7 #define NLOHMANN_JSON_VERSION_PATCH 0 -#include // all_of, find, for_each -#include // assert -#include // and, not, or -#include // nullptr_t, ptrdiff_t, size_t -#include // hash, less -#include // initializer_list -#include // istream, ostream -#include // random_access_iterator_tag -#include // unique_ptr -#include // accumulate -#include // string, stoi, to_string -#include // declval, forward, move, pair, swap -#include // vector +#include // all_of, find, for_each +#include // assert +#include // and, not, or +#include // nullptr_t, ptrdiff_t, size_t +#include // hash, less +#include // initializer_list +#include // istream, ostream +#include // random_access_iterator_tag +#include // unique_ptr +#include // accumulate +#include // string, stoi, to_string +#include // declval, forward, move, pair, swap +#include // vector // #include - #include // #include - -#include // transform -#include // array -#include // and, not -#include // forward_list -#include // inserter, front_inserter, end -#include // map -#include // string -#include // tuple, make_tuple -#include // is_arithmetic, is_same, is_enum, underlying_type, is_convertible -#include // unordered_map -#include // pair, declval -#include // valarray +#include // transform +#include // array +#include // and, not +#include // forward_list +#include // inserter, front_inserter, end +#include // map +#include // string +#include // tuple, make_tuple +#include // is_arithmetic, is_same, is_enum, underlying_type, is_convertible +#include // unordered_map +#include // pair, declval +#include // valarray // #include - -#include // exception -#include // runtime_error -#include // to_string +#include // exception +#include // runtime_error +#include // to_string // #include +#include // size_t -#include // size_t - -namespace nlohmann -{ -namespace detail -{ +namespace nlohmann { +namespace detail { /// struct to capture the start position of the current token -struct position_t -{ +struct position_t { /// the total number of characters read std::size_t chars_read_total = 0; /// the number of characters read in the current line @@ -96,19 +89,17 @@ struct position_t std::size_t lines_read = 0; /// conversion to size_t to preserve SAX interface - constexpr operator size_t() const - { + constexpr operator size_t() const { return chars_read_total; } }; -} // namespace detail -} // namespace nlohmann +} // namespace detail +} // namespace nlohmann // #include - -#include // pair +#include // pair // #include /* Hedley - https://nemequ.github.io/hedley * Created by Evan Nemerson @@ -124,1567 +115,1478 @@ struct position_t #if !defined(JSON_HEDLEY_VERSION) || (JSON_HEDLEY_VERSION < 9) #if defined(JSON_HEDLEY_VERSION) - #undef JSON_HEDLEY_VERSION +#undef JSON_HEDLEY_VERSION #endif #define JSON_HEDLEY_VERSION 9 #if defined(JSON_HEDLEY_STRINGIFY_EX) - #undef JSON_HEDLEY_STRINGIFY_EX +#undef JSON_HEDLEY_STRINGIFY_EX #endif #define JSON_HEDLEY_STRINGIFY_EX(x) #x #if defined(JSON_HEDLEY_STRINGIFY) - #undef JSON_HEDLEY_STRINGIFY +#undef JSON_HEDLEY_STRINGIFY #endif #define JSON_HEDLEY_STRINGIFY(x) JSON_HEDLEY_STRINGIFY_EX(x) #if defined(JSON_HEDLEY_CONCAT_EX) - #undef JSON_HEDLEY_CONCAT_EX +#undef JSON_HEDLEY_CONCAT_EX #endif -#define JSON_HEDLEY_CONCAT_EX(a,b) a##b +#define JSON_HEDLEY_CONCAT_EX(a, b) a##b #if defined(JSON_HEDLEY_CONCAT) - #undef JSON_HEDLEY_CONCAT +#undef JSON_HEDLEY_CONCAT #endif -#define JSON_HEDLEY_CONCAT(a,b) JSON_HEDLEY_CONCAT_EX(a,b) +#define JSON_HEDLEY_CONCAT(a, b) JSON_HEDLEY_CONCAT_EX(a, b) #if defined(JSON_HEDLEY_VERSION_ENCODE) - #undef JSON_HEDLEY_VERSION_ENCODE +#undef JSON_HEDLEY_VERSION_ENCODE #endif -#define JSON_HEDLEY_VERSION_ENCODE(major,minor,revision) (((major) * 1000000) + ((minor) * 1000) + (revision)) +#define JSON_HEDLEY_VERSION_ENCODE(major, minor, revision) (((major)*1000000) + ((minor)*1000) + (revision)) #if defined(JSON_HEDLEY_VERSION_DECODE_MAJOR) - #undef JSON_HEDLEY_VERSION_DECODE_MAJOR +#undef JSON_HEDLEY_VERSION_DECODE_MAJOR #endif #define JSON_HEDLEY_VERSION_DECODE_MAJOR(version) ((version) / 1000000) #if defined(JSON_HEDLEY_VERSION_DECODE_MINOR) - #undef JSON_HEDLEY_VERSION_DECODE_MINOR +#undef JSON_HEDLEY_VERSION_DECODE_MINOR #endif #define JSON_HEDLEY_VERSION_DECODE_MINOR(version) (((version) % 1000000) / 1000) #if defined(JSON_HEDLEY_VERSION_DECODE_REVISION) - #undef JSON_HEDLEY_VERSION_DECODE_REVISION +#undef JSON_HEDLEY_VERSION_DECODE_REVISION #endif #define JSON_HEDLEY_VERSION_DECODE_REVISION(version) ((version) % 1000) #if defined(JSON_HEDLEY_GNUC_VERSION) - #undef JSON_HEDLEY_GNUC_VERSION +#undef JSON_HEDLEY_GNUC_VERSION #endif #if defined(__GNUC__) && defined(__GNUC_PATCHLEVEL__) - #define JSON_HEDLEY_GNUC_VERSION JSON_HEDLEY_VERSION_ENCODE(__GNUC__, __GNUC_MINOR__, __GNUC_PATCHLEVEL__) +#define JSON_HEDLEY_GNUC_VERSION JSON_HEDLEY_VERSION_ENCODE(__GNUC__, __GNUC_MINOR__, __GNUC_PATCHLEVEL__) #elif defined(__GNUC__) - #define JSON_HEDLEY_GNUC_VERSION JSON_HEDLEY_VERSION_ENCODE(__GNUC__, __GNUC_MINOR__, 0) +#define JSON_HEDLEY_GNUC_VERSION JSON_HEDLEY_VERSION_ENCODE(__GNUC__, __GNUC_MINOR__, 0) #endif #if defined(JSON_HEDLEY_GNUC_VERSION_CHECK) - #undef JSON_HEDLEY_GNUC_VERSION_CHECK +#undef JSON_HEDLEY_GNUC_VERSION_CHECK #endif #if defined(JSON_HEDLEY_GNUC_VERSION) - #define JSON_HEDLEY_GNUC_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_GNUC_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#define JSON_HEDLEY_GNUC_VERSION_CHECK(major, minor, patch) \ + (JSON_HEDLEY_GNUC_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) #else - #define JSON_HEDLEY_GNUC_VERSION_CHECK(major,minor,patch) (0) +#define JSON_HEDLEY_GNUC_VERSION_CHECK(major, minor, patch) (0) #endif #if defined(JSON_HEDLEY_MSVC_VERSION) - #undef JSON_HEDLEY_MSVC_VERSION +#undef JSON_HEDLEY_MSVC_VERSION #endif #if defined(_MSC_FULL_VER) && (_MSC_FULL_VER >= 140000000) - #define JSON_HEDLEY_MSVC_VERSION JSON_HEDLEY_VERSION_ENCODE(_MSC_FULL_VER / 10000000, (_MSC_FULL_VER % 10000000) / 100000, (_MSC_FULL_VER % 100000) / 100) +#define JSON_HEDLEY_MSVC_VERSION \ + JSON_HEDLEY_VERSION_ENCODE(_MSC_FULL_VER / 10000000, (_MSC_FULL_VER % 10000000) / 100000, \ + (_MSC_FULL_VER % 100000) / 100) #elif defined(_MSC_FULL_VER) - #define JSON_HEDLEY_MSVC_VERSION JSON_HEDLEY_VERSION_ENCODE(_MSC_FULL_VER / 1000000, (_MSC_FULL_VER % 1000000) / 10000, (_MSC_FULL_VER % 10000) / 10) +#define JSON_HEDLEY_MSVC_VERSION \ + JSON_HEDLEY_VERSION_ENCODE(_MSC_FULL_VER / 1000000, (_MSC_FULL_VER % 1000000) / 10000, (_MSC_FULL_VER % 10000) / 10) #elif defined(_MSC_VER) - #define JSON_HEDLEY_MSVC_VERSION JSON_HEDLEY_VERSION_ENCODE(_MSC_VER / 100, _MSC_VER % 100, 0) +#define JSON_HEDLEY_MSVC_VERSION JSON_HEDLEY_VERSION_ENCODE(_MSC_VER / 100, _MSC_VER % 100, 0) #endif #if defined(JSON_HEDLEY_MSVC_VERSION_CHECK) - #undef JSON_HEDLEY_MSVC_VERSION_CHECK +#undef JSON_HEDLEY_MSVC_VERSION_CHECK #endif #if !defined(_MSC_VER) - #define JSON_HEDLEY_MSVC_VERSION_CHECK(major,minor,patch) (0) +#define JSON_HEDLEY_MSVC_VERSION_CHECK(major, minor, patch) (0) #elif defined(_MSC_VER) && (_MSC_VER >= 1400) - #define JSON_HEDLEY_MSVC_VERSION_CHECK(major,minor,patch) (_MSC_FULL_VER >= ((major * 10000000) + (minor * 100000) + (patch))) +#define JSON_HEDLEY_MSVC_VERSION_CHECK(major, minor, patch) \ + (_MSC_FULL_VER >= ((major * 10000000) + (minor * 100000) + (patch))) #elif defined(_MSC_VER) && (_MSC_VER >= 1200) - #define JSON_HEDLEY_MSVC_VERSION_CHECK(major,minor,patch) (_MSC_FULL_VER >= ((major * 1000000) + (minor * 10000) + (patch))) +#define JSON_HEDLEY_MSVC_VERSION_CHECK(major, minor, patch) \ + (_MSC_FULL_VER >= ((major * 1000000) + (minor * 10000) + (patch))) #else - #define JSON_HEDLEY_MSVC_VERSION_CHECK(major,minor,patch) (_MSC_VER >= ((major * 100) + (minor))) +#define JSON_HEDLEY_MSVC_VERSION_CHECK(major, minor, patch) (_MSC_VER >= ((major * 100) + (minor))) #endif #if defined(JSON_HEDLEY_INTEL_VERSION) - #undef JSON_HEDLEY_INTEL_VERSION +#undef JSON_HEDLEY_INTEL_VERSION #endif #if defined(__INTEL_COMPILER) && defined(__INTEL_COMPILER_UPDATE) - #define JSON_HEDLEY_INTEL_VERSION JSON_HEDLEY_VERSION_ENCODE(__INTEL_COMPILER / 100, __INTEL_COMPILER % 100, __INTEL_COMPILER_UPDATE) +#define JSON_HEDLEY_INTEL_VERSION \ + JSON_HEDLEY_VERSION_ENCODE(__INTEL_COMPILER / 100, __INTEL_COMPILER % 100, __INTEL_COMPILER_UPDATE) #elif defined(__INTEL_COMPILER) - #define JSON_HEDLEY_INTEL_VERSION JSON_HEDLEY_VERSION_ENCODE(__INTEL_COMPILER / 100, __INTEL_COMPILER % 100, 0) +#define JSON_HEDLEY_INTEL_VERSION JSON_HEDLEY_VERSION_ENCODE(__INTEL_COMPILER / 100, __INTEL_COMPILER % 100, 0) #endif #if defined(JSON_HEDLEY_INTEL_VERSION_CHECK) - #undef JSON_HEDLEY_INTEL_VERSION_CHECK +#undef JSON_HEDLEY_INTEL_VERSION_CHECK #endif #if defined(JSON_HEDLEY_INTEL_VERSION) - #define JSON_HEDLEY_INTEL_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_INTEL_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#define JSON_HEDLEY_INTEL_VERSION_CHECK(major, minor, patch) \ + (JSON_HEDLEY_INTEL_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) #else - #define JSON_HEDLEY_INTEL_VERSION_CHECK(major,minor,patch) (0) +#define JSON_HEDLEY_INTEL_VERSION_CHECK(major, minor, patch) (0) #endif #if defined(JSON_HEDLEY_PGI_VERSION) - #undef JSON_HEDLEY_PGI_VERSION +#undef JSON_HEDLEY_PGI_VERSION #endif #if defined(__PGI) && defined(__PGIC__) && defined(__PGIC_MINOR__) && defined(__PGIC_PATCHLEVEL__) - #define JSON_HEDLEY_PGI_VERSION JSON_HEDLEY_VERSION_ENCODE(__PGIC__, __PGIC_MINOR__, __PGIC_PATCHLEVEL__) +#define JSON_HEDLEY_PGI_VERSION JSON_HEDLEY_VERSION_ENCODE(__PGIC__, __PGIC_MINOR__, __PGIC_PATCHLEVEL__) #endif #if defined(JSON_HEDLEY_PGI_VERSION_CHECK) - #undef JSON_HEDLEY_PGI_VERSION_CHECK +#undef JSON_HEDLEY_PGI_VERSION_CHECK #endif #if defined(JSON_HEDLEY_PGI_VERSION) - #define JSON_HEDLEY_PGI_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_PGI_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#define JSON_HEDLEY_PGI_VERSION_CHECK(major, minor, patch) \ + (JSON_HEDLEY_PGI_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) #else - #define JSON_HEDLEY_PGI_VERSION_CHECK(major,minor,patch) (0) +#define JSON_HEDLEY_PGI_VERSION_CHECK(major, minor, patch) (0) #endif #if defined(JSON_HEDLEY_SUNPRO_VERSION) - #undef JSON_HEDLEY_SUNPRO_VERSION +#undef JSON_HEDLEY_SUNPRO_VERSION #endif #if defined(__SUNPRO_C) && (__SUNPRO_C > 0x1000) - #define JSON_HEDLEY_SUNPRO_VERSION JSON_HEDLEY_VERSION_ENCODE((((__SUNPRO_C >> 16) & 0xf) * 10) + ((__SUNPRO_C >> 12) & 0xf), (((__SUNPRO_C >> 8) & 0xf) * 10) + ((__SUNPRO_C >> 4) & 0xf), (__SUNPRO_C & 0xf) * 10) +#define JSON_HEDLEY_SUNPRO_VERSION \ + JSON_HEDLEY_VERSION_ENCODE((((__SUNPRO_C >> 16) & 0xf) * 10) + ((__SUNPRO_C >> 12) & 0xf), \ + (((__SUNPRO_C >> 8) & 0xf) * 10) + ((__SUNPRO_C >> 4) & 0xf), (__SUNPRO_C & 0xf) * 10) #elif defined(__SUNPRO_C) - #define JSON_HEDLEY_SUNPRO_VERSION JSON_HEDLEY_VERSION_ENCODE((__SUNPRO_C >> 8) & 0xf, (__SUNPRO_C >> 4) & 0xf, (__SUNPRO_C) & 0xf) +#define JSON_HEDLEY_SUNPRO_VERSION \ + JSON_HEDLEY_VERSION_ENCODE((__SUNPRO_C >> 8) & 0xf, (__SUNPRO_C >> 4) & 0xf, (__SUNPRO_C)&0xf) #elif defined(__SUNPRO_CC) && (__SUNPRO_CC > 0x1000) - #define JSON_HEDLEY_SUNPRO_VERSION JSON_HEDLEY_VERSION_ENCODE((((__SUNPRO_CC >> 16) & 0xf) * 10) + ((__SUNPRO_CC >> 12) & 0xf), (((__SUNPRO_CC >> 8) & 0xf) * 10) + ((__SUNPRO_CC >> 4) & 0xf), (__SUNPRO_CC & 0xf) * 10) +#define JSON_HEDLEY_SUNPRO_VERSION \ + JSON_HEDLEY_VERSION_ENCODE((((__SUNPRO_CC >> 16) & 0xf) * 10) + ((__SUNPRO_CC >> 12) & 0xf), \ + (((__SUNPRO_CC >> 8) & 0xf) * 10) + ((__SUNPRO_CC >> 4) & 0xf), \ + (__SUNPRO_CC & 0xf) * 10) #elif defined(__SUNPRO_CC) - #define JSON_HEDLEY_SUNPRO_VERSION JSON_HEDLEY_VERSION_ENCODE((__SUNPRO_CC >> 8) & 0xf, (__SUNPRO_CC >> 4) & 0xf, (__SUNPRO_CC) & 0xf) +#define JSON_HEDLEY_SUNPRO_VERSION \ + JSON_HEDLEY_VERSION_ENCODE((__SUNPRO_CC >> 8) & 0xf, (__SUNPRO_CC >> 4) & 0xf, (__SUNPRO_CC)&0xf) #endif #if defined(JSON_HEDLEY_SUNPRO_VERSION_CHECK) - #undef JSON_HEDLEY_SUNPRO_VERSION_CHECK +#undef JSON_HEDLEY_SUNPRO_VERSION_CHECK #endif #if defined(JSON_HEDLEY_SUNPRO_VERSION) - #define JSON_HEDLEY_SUNPRO_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_SUNPRO_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#define JSON_HEDLEY_SUNPRO_VERSION_CHECK(major, minor, patch) \ + (JSON_HEDLEY_SUNPRO_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) #else - #define JSON_HEDLEY_SUNPRO_VERSION_CHECK(major,minor,patch) (0) +#define JSON_HEDLEY_SUNPRO_VERSION_CHECK(major, minor, patch) (0) #endif #if defined(JSON_HEDLEY_EMSCRIPTEN_VERSION) - #undef JSON_HEDLEY_EMSCRIPTEN_VERSION +#undef JSON_HEDLEY_EMSCRIPTEN_VERSION #endif #if defined(__EMSCRIPTEN__) - #define JSON_HEDLEY_EMSCRIPTEN_VERSION JSON_HEDLEY_VERSION_ENCODE(__EMSCRIPTEN_major__, __EMSCRIPTEN_minor__, __EMSCRIPTEN_tiny__) +#define JSON_HEDLEY_EMSCRIPTEN_VERSION \ + JSON_HEDLEY_VERSION_ENCODE(__EMSCRIPTEN_major__, __EMSCRIPTEN_minor__, __EMSCRIPTEN_tiny__) #endif #if defined(JSON_HEDLEY_EMSCRIPTEN_VERSION_CHECK) - #undef JSON_HEDLEY_EMSCRIPTEN_VERSION_CHECK +#undef JSON_HEDLEY_EMSCRIPTEN_VERSION_CHECK #endif #if defined(JSON_HEDLEY_EMSCRIPTEN_VERSION) - #define JSON_HEDLEY_EMSCRIPTEN_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_EMSCRIPTEN_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#define JSON_HEDLEY_EMSCRIPTEN_VERSION_CHECK(major, minor, patch) \ + (JSON_HEDLEY_EMSCRIPTEN_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) #else - #define JSON_HEDLEY_EMSCRIPTEN_VERSION_CHECK(major,minor,patch) (0) +#define JSON_HEDLEY_EMSCRIPTEN_VERSION_CHECK(major, minor, patch) (0) #endif #if defined(JSON_HEDLEY_ARM_VERSION) - #undef JSON_HEDLEY_ARM_VERSION +#undef JSON_HEDLEY_ARM_VERSION #endif #if defined(__CC_ARM) && defined(__ARMCOMPILER_VERSION) - #define JSON_HEDLEY_ARM_VERSION JSON_HEDLEY_VERSION_ENCODE(__ARMCOMPILER_VERSION / 1000000, (__ARMCOMPILER_VERSION % 1000000) / 10000, (__ARMCOMPILER_VERSION % 10000) / 100) +#define JSON_HEDLEY_ARM_VERSION \ + JSON_HEDLEY_VERSION_ENCODE(__ARMCOMPILER_VERSION / 1000000, (__ARMCOMPILER_VERSION % 1000000) / 10000, \ + (__ARMCOMPILER_VERSION % 10000) / 100) #elif defined(__CC_ARM) && defined(__ARMCC_VERSION) - #define JSON_HEDLEY_ARM_VERSION JSON_HEDLEY_VERSION_ENCODE(__ARMCC_VERSION / 1000000, (__ARMCC_VERSION % 1000000) / 10000, (__ARMCC_VERSION % 10000) / 100) +#define JSON_HEDLEY_ARM_VERSION \ + JSON_HEDLEY_VERSION_ENCODE(__ARMCC_VERSION / 1000000, (__ARMCC_VERSION % 1000000) / 10000, \ + (__ARMCC_VERSION % 10000) / 100) #endif #if defined(JSON_HEDLEY_ARM_VERSION_CHECK) - #undef JSON_HEDLEY_ARM_VERSION_CHECK +#undef JSON_HEDLEY_ARM_VERSION_CHECK #endif #if defined(JSON_HEDLEY_ARM_VERSION) - #define JSON_HEDLEY_ARM_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_ARM_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#define JSON_HEDLEY_ARM_VERSION_CHECK(major, minor, patch) \ + (JSON_HEDLEY_ARM_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) #else - #define JSON_HEDLEY_ARM_VERSION_CHECK(major,minor,patch) (0) +#define JSON_HEDLEY_ARM_VERSION_CHECK(major, minor, patch) (0) #endif #if defined(JSON_HEDLEY_IBM_VERSION) - #undef JSON_HEDLEY_IBM_VERSION +#undef JSON_HEDLEY_IBM_VERSION #endif #if defined(__ibmxl__) - #define JSON_HEDLEY_IBM_VERSION JSON_HEDLEY_VERSION_ENCODE(__ibmxl_version__, __ibmxl_release__, __ibmxl_modification__) +#define JSON_HEDLEY_IBM_VERSION JSON_HEDLEY_VERSION_ENCODE(__ibmxl_version__, __ibmxl_release__, __ibmxl_modification__) #elif defined(__xlC__) && defined(__xlC_ver__) - #define JSON_HEDLEY_IBM_VERSION JSON_HEDLEY_VERSION_ENCODE(__xlC__ >> 8, __xlC__ & 0xff, (__xlC_ver__ >> 8) & 0xff) +#define JSON_HEDLEY_IBM_VERSION JSON_HEDLEY_VERSION_ENCODE(__xlC__ >> 8, __xlC__ & 0xff, (__xlC_ver__ >> 8) & 0xff) #elif defined(__xlC__) - #define JSON_HEDLEY_IBM_VERSION JSON_HEDLEY_VERSION_ENCODE(__xlC__ >> 8, __xlC__ & 0xff, 0) +#define JSON_HEDLEY_IBM_VERSION JSON_HEDLEY_VERSION_ENCODE(__xlC__ >> 8, __xlC__ & 0xff, 0) #endif #if defined(JSON_HEDLEY_IBM_VERSION_CHECK) - #undef JSON_HEDLEY_IBM_VERSION_CHECK +#undef JSON_HEDLEY_IBM_VERSION_CHECK #endif #if defined(JSON_HEDLEY_IBM_VERSION) - #define JSON_HEDLEY_IBM_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_IBM_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#define JSON_HEDLEY_IBM_VERSION_CHECK(major, minor, patch) \ + (JSON_HEDLEY_IBM_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) #else - #define JSON_HEDLEY_IBM_VERSION_CHECK(major,minor,patch) (0) +#define JSON_HEDLEY_IBM_VERSION_CHECK(major, minor, patch) (0) #endif #if defined(JSON_HEDLEY_TI_VERSION) - #undef JSON_HEDLEY_TI_VERSION +#undef JSON_HEDLEY_TI_VERSION #endif #if defined(__TI_COMPILER_VERSION__) - #define JSON_HEDLEY_TI_VERSION JSON_HEDLEY_VERSION_ENCODE(__TI_COMPILER_VERSION__ / 1000000, (__TI_COMPILER_VERSION__ % 1000000) / 1000, (__TI_COMPILER_VERSION__ % 1000)) +#define JSON_HEDLEY_TI_VERSION \ + JSON_HEDLEY_VERSION_ENCODE(__TI_COMPILER_VERSION__ / 1000000, (__TI_COMPILER_VERSION__ % 1000000) / 1000, \ + (__TI_COMPILER_VERSION__ % 1000)) #endif #if defined(JSON_HEDLEY_TI_VERSION_CHECK) - #undef JSON_HEDLEY_TI_VERSION_CHECK +#undef JSON_HEDLEY_TI_VERSION_CHECK #endif #if defined(JSON_HEDLEY_TI_VERSION) - #define JSON_HEDLEY_TI_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_TI_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#define JSON_HEDLEY_TI_VERSION_CHECK(major, minor, patch) \ + (JSON_HEDLEY_TI_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) #else - #define JSON_HEDLEY_TI_VERSION_CHECK(major,minor,patch) (0) +#define JSON_HEDLEY_TI_VERSION_CHECK(major, minor, patch) (0) #endif #if defined(JSON_HEDLEY_CRAY_VERSION) - #undef JSON_HEDLEY_CRAY_VERSION +#undef JSON_HEDLEY_CRAY_VERSION #endif #if defined(_CRAYC) - #if defined(_RELEASE_PATCHLEVEL) - #define JSON_HEDLEY_CRAY_VERSION JSON_HEDLEY_VERSION_ENCODE(_RELEASE_MAJOR, _RELEASE_MINOR, _RELEASE_PATCHLEVEL) - #else - #define JSON_HEDLEY_CRAY_VERSION JSON_HEDLEY_VERSION_ENCODE(_RELEASE_MAJOR, _RELEASE_MINOR, 0) - #endif +#if defined(_RELEASE_PATCHLEVEL) +#define JSON_HEDLEY_CRAY_VERSION JSON_HEDLEY_VERSION_ENCODE(_RELEASE_MAJOR, _RELEASE_MINOR, _RELEASE_PATCHLEVEL) +#else +#define JSON_HEDLEY_CRAY_VERSION JSON_HEDLEY_VERSION_ENCODE(_RELEASE_MAJOR, _RELEASE_MINOR, 0) +#endif #endif #if defined(JSON_HEDLEY_CRAY_VERSION_CHECK) - #undef JSON_HEDLEY_CRAY_VERSION_CHECK +#undef JSON_HEDLEY_CRAY_VERSION_CHECK #endif #if defined(JSON_HEDLEY_CRAY_VERSION) - #define JSON_HEDLEY_CRAY_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_CRAY_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#define JSON_HEDLEY_CRAY_VERSION_CHECK(major, minor, patch) \ + (JSON_HEDLEY_CRAY_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) #else - #define JSON_HEDLEY_CRAY_VERSION_CHECK(major,minor,patch) (0) +#define JSON_HEDLEY_CRAY_VERSION_CHECK(major, minor, patch) (0) #endif #if defined(JSON_HEDLEY_IAR_VERSION) - #undef JSON_HEDLEY_IAR_VERSION +#undef JSON_HEDLEY_IAR_VERSION #endif #if defined(__IAR_SYSTEMS_ICC__) - #if __VER__ > 1000 - #define JSON_HEDLEY_IAR_VERSION JSON_HEDLEY_VERSION_ENCODE((__VER__ / 1000000), ((__VER__ / 1000) % 1000), (__VER__ % 1000)) - #else - #define JSON_HEDLEY_IAR_VERSION JSON_HEDLEY_VERSION_ENCODE(VER / 100, __VER__ % 100, 0) - #endif +#if __VER__ > 1000 +#define JSON_HEDLEY_IAR_VERSION \ + JSON_HEDLEY_VERSION_ENCODE((__VER__ / 1000000), ((__VER__ / 1000) % 1000), (__VER__ % 1000)) +#else +#define JSON_HEDLEY_IAR_VERSION JSON_HEDLEY_VERSION_ENCODE(VER / 100, __VER__ % 100, 0) +#endif #endif #if defined(JSON_HEDLEY_IAR_VERSION_CHECK) - #undef JSON_HEDLEY_IAR_VERSION_CHECK +#undef JSON_HEDLEY_IAR_VERSION_CHECK #endif #if defined(JSON_HEDLEY_IAR_VERSION) - #define JSON_HEDLEY_IAR_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_IAR_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#define JSON_HEDLEY_IAR_VERSION_CHECK(major, minor, patch) \ + (JSON_HEDLEY_IAR_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) #else - #define JSON_HEDLEY_IAR_VERSION_CHECK(major,minor,patch) (0) +#define JSON_HEDLEY_IAR_VERSION_CHECK(major, minor, patch) (0) #endif #if defined(JSON_HEDLEY_TINYC_VERSION) - #undef JSON_HEDLEY_TINYC_VERSION +#undef JSON_HEDLEY_TINYC_VERSION #endif #if defined(__TINYC__) - #define JSON_HEDLEY_TINYC_VERSION JSON_HEDLEY_VERSION_ENCODE(__TINYC__ / 1000, (__TINYC__ / 100) % 10, __TINYC__ % 100) +#define JSON_HEDLEY_TINYC_VERSION JSON_HEDLEY_VERSION_ENCODE(__TINYC__ / 1000, (__TINYC__ / 100) % 10, __TINYC__ % 100) #endif #if defined(JSON_HEDLEY_TINYC_VERSION_CHECK) - #undef JSON_HEDLEY_TINYC_VERSION_CHECK +#undef JSON_HEDLEY_TINYC_VERSION_CHECK #endif #if defined(JSON_HEDLEY_TINYC_VERSION) - #define JSON_HEDLEY_TINYC_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_TINYC_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#define JSON_HEDLEY_TINYC_VERSION_CHECK(major, minor, patch) \ + (JSON_HEDLEY_TINYC_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) #else - #define JSON_HEDLEY_TINYC_VERSION_CHECK(major,minor,patch) (0) +#define JSON_HEDLEY_TINYC_VERSION_CHECK(major, minor, patch) (0) #endif #if defined(JSON_HEDLEY_DMC_VERSION) - #undef JSON_HEDLEY_DMC_VERSION +#undef JSON_HEDLEY_DMC_VERSION #endif #if defined(__DMC__) - #define JSON_HEDLEY_DMC_VERSION JSON_HEDLEY_VERSION_ENCODE(__DMC__ >> 8, (__DMC__ >> 4) & 0xf, __DMC__ & 0xf) +#define JSON_HEDLEY_DMC_VERSION JSON_HEDLEY_VERSION_ENCODE(__DMC__ >> 8, (__DMC__ >> 4) & 0xf, __DMC__ & 0xf) #endif #if defined(JSON_HEDLEY_DMC_VERSION_CHECK) - #undef JSON_HEDLEY_DMC_VERSION_CHECK +#undef JSON_HEDLEY_DMC_VERSION_CHECK #endif #if defined(JSON_HEDLEY_DMC_VERSION) - #define JSON_HEDLEY_DMC_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_DMC_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#define JSON_HEDLEY_DMC_VERSION_CHECK(major, minor, patch) \ + (JSON_HEDLEY_DMC_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) #else - #define JSON_HEDLEY_DMC_VERSION_CHECK(major,minor,patch) (0) +#define JSON_HEDLEY_DMC_VERSION_CHECK(major, minor, patch) (0) #endif #if defined(JSON_HEDLEY_COMPCERT_VERSION) - #undef JSON_HEDLEY_COMPCERT_VERSION +#undef JSON_HEDLEY_COMPCERT_VERSION #endif #if defined(__COMPCERT_VERSION__) - #define JSON_HEDLEY_COMPCERT_VERSION JSON_HEDLEY_VERSION_ENCODE(__COMPCERT_VERSION__ / 10000, (__COMPCERT_VERSION__ / 100) % 100, __COMPCERT_VERSION__ % 100) +#define JSON_HEDLEY_COMPCERT_VERSION \ + JSON_HEDLEY_VERSION_ENCODE(__COMPCERT_VERSION__ / 10000, (__COMPCERT_VERSION__ / 100) % 100, \ + __COMPCERT_VERSION__ % 100) #endif #if defined(JSON_HEDLEY_COMPCERT_VERSION_CHECK) - #undef JSON_HEDLEY_COMPCERT_VERSION_CHECK +#undef JSON_HEDLEY_COMPCERT_VERSION_CHECK #endif #if defined(JSON_HEDLEY_COMPCERT_VERSION) - #define JSON_HEDLEY_COMPCERT_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_COMPCERT_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#define JSON_HEDLEY_COMPCERT_VERSION_CHECK(major, minor, patch) \ + (JSON_HEDLEY_COMPCERT_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) #else - #define JSON_HEDLEY_COMPCERT_VERSION_CHECK(major,minor,patch) (0) +#define JSON_HEDLEY_COMPCERT_VERSION_CHECK(major, minor, patch) (0) #endif #if defined(JSON_HEDLEY_PELLES_VERSION) - #undef JSON_HEDLEY_PELLES_VERSION +#undef JSON_HEDLEY_PELLES_VERSION #endif #if defined(__POCC__) - #define JSON_HEDLEY_PELLES_VERSION JSON_HEDLEY_VERSION_ENCODE(__POCC__ / 100, __POCC__ % 100, 0) +#define JSON_HEDLEY_PELLES_VERSION JSON_HEDLEY_VERSION_ENCODE(__POCC__ / 100, __POCC__ % 100, 0) #endif #if defined(JSON_HEDLEY_PELLES_VERSION_CHECK) - #undef JSON_HEDLEY_PELLES_VERSION_CHECK +#undef JSON_HEDLEY_PELLES_VERSION_CHECK #endif #if defined(JSON_HEDLEY_PELLES_VERSION) - #define JSON_HEDLEY_PELLES_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_PELLES_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#define JSON_HEDLEY_PELLES_VERSION_CHECK(major, minor, patch) \ + (JSON_HEDLEY_PELLES_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) #else - #define JSON_HEDLEY_PELLES_VERSION_CHECK(major,minor,patch) (0) +#define JSON_HEDLEY_PELLES_VERSION_CHECK(major, minor, patch) (0) #endif #if defined(JSON_HEDLEY_GCC_VERSION) - #undef JSON_HEDLEY_GCC_VERSION +#undef JSON_HEDLEY_GCC_VERSION #endif -#if \ - defined(JSON_HEDLEY_GNUC_VERSION) && \ - !defined(__clang__) && \ - !defined(JSON_HEDLEY_INTEL_VERSION) && \ - !defined(JSON_HEDLEY_PGI_VERSION) && \ - !defined(JSON_HEDLEY_ARM_VERSION) && \ - !defined(JSON_HEDLEY_TI_VERSION) && \ +#if defined(JSON_HEDLEY_GNUC_VERSION) && !defined(__clang__) && !defined(JSON_HEDLEY_INTEL_VERSION) && \ + !defined(JSON_HEDLEY_PGI_VERSION) && !defined(JSON_HEDLEY_ARM_VERSION) && !defined(JSON_HEDLEY_TI_VERSION) && \ !defined(__COMPCERT__) - #define JSON_HEDLEY_GCC_VERSION JSON_HEDLEY_GNUC_VERSION +#define JSON_HEDLEY_GCC_VERSION JSON_HEDLEY_GNUC_VERSION #endif #if defined(JSON_HEDLEY_GCC_VERSION_CHECK) - #undef JSON_HEDLEY_GCC_VERSION_CHECK +#undef JSON_HEDLEY_GCC_VERSION_CHECK #endif #if defined(JSON_HEDLEY_GCC_VERSION) - #define JSON_HEDLEY_GCC_VERSION_CHECK(major,minor,patch) (JSON_HEDLEY_GCC_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) +#define JSON_HEDLEY_GCC_VERSION_CHECK(major, minor, patch) \ + (JSON_HEDLEY_GCC_VERSION >= JSON_HEDLEY_VERSION_ENCODE(major, minor, patch)) #else - #define JSON_HEDLEY_GCC_VERSION_CHECK(major,minor,patch) (0) +#define JSON_HEDLEY_GCC_VERSION_CHECK(major, minor, patch) (0) #endif #if defined(JSON_HEDLEY_HAS_ATTRIBUTE) - #undef JSON_HEDLEY_HAS_ATTRIBUTE +#undef JSON_HEDLEY_HAS_ATTRIBUTE #endif #if defined(__has_attribute) - #define JSON_HEDLEY_HAS_ATTRIBUTE(attribute) __has_attribute(attribute) +#define JSON_HEDLEY_HAS_ATTRIBUTE(attribute) __has_attribute(attribute) #else - #define JSON_HEDLEY_HAS_ATTRIBUTE(attribute) (0) +#define JSON_HEDLEY_HAS_ATTRIBUTE(attribute) (0) #endif #if defined(JSON_HEDLEY_GNUC_HAS_ATTRIBUTE) - #undef JSON_HEDLEY_GNUC_HAS_ATTRIBUTE +#undef JSON_HEDLEY_GNUC_HAS_ATTRIBUTE #endif #if defined(__has_attribute) - #define JSON_HEDLEY_GNUC_HAS_ATTRIBUTE(attribute,major,minor,patch) __has_attribute(attribute) +#define JSON_HEDLEY_GNUC_HAS_ATTRIBUTE(attribute, major, minor, patch) __has_attribute(attribute) #else - #define JSON_HEDLEY_GNUC_HAS_ATTRIBUTE(attribute,major,minor,patch) JSON_HEDLEY_GNUC_VERSION_CHECK(major,minor,patch) +#define JSON_HEDLEY_GNUC_HAS_ATTRIBUTE(attribute, major, minor, patch) \ + JSON_HEDLEY_GNUC_VERSION_CHECK(major, minor, patch) #endif #if defined(JSON_HEDLEY_GCC_HAS_ATTRIBUTE) - #undef JSON_HEDLEY_GCC_HAS_ATTRIBUTE +#undef JSON_HEDLEY_GCC_HAS_ATTRIBUTE #endif #if defined(__has_attribute) - #define JSON_HEDLEY_GCC_HAS_ATTRIBUTE(attribute,major,minor,patch) __has_attribute(attribute) +#define JSON_HEDLEY_GCC_HAS_ATTRIBUTE(attribute, major, minor, patch) __has_attribute(attribute) #else - #define JSON_HEDLEY_GCC_HAS_ATTRIBUTE(attribute,major,minor,patch) JSON_HEDLEY_GCC_VERSION_CHECK(major,minor,patch) +#define JSON_HEDLEY_GCC_HAS_ATTRIBUTE(attribute, major, minor, patch) JSON_HEDLEY_GCC_VERSION_CHECK(major, minor, patch) #endif #if defined(JSON_HEDLEY_HAS_CPP_ATTRIBUTE) - #undef JSON_HEDLEY_HAS_CPP_ATTRIBUTE +#undef JSON_HEDLEY_HAS_CPP_ATTRIBUTE #endif #if defined(__has_cpp_attribute) && defined(__cplusplus) - #define JSON_HEDLEY_HAS_CPP_ATTRIBUTE(attribute) __has_cpp_attribute(attribute) +#define JSON_HEDLEY_HAS_CPP_ATTRIBUTE(attribute) __has_cpp_attribute(attribute) #else - #define JSON_HEDLEY_HAS_CPP_ATTRIBUTE(attribute) (0) +#define JSON_HEDLEY_HAS_CPP_ATTRIBUTE(attribute) (0) #endif #if defined(JSON_HEDLEY_GNUC_HAS_CPP_ATTRIBUTE) - #undef JSON_HEDLEY_GNUC_HAS_CPP_ATTRIBUTE +#undef JSON_HEDLEY_GNUC_HAS_CPP_ATTRIBUTE #endif #if defined(__has_cpp_attribute) && defined(__cplusplus) - #define JSON_HEDLEY_GNUC_HAS_CPP_ATTRIBUTE(attribute,major,minor,patch) __has_cpp_attribute(attribute) +#define JSON_HEDLEY_GNUC_HAS_CPP_ATTRIBUTE(attribute, major, minor, patch) __has_cpp_attribute(attribute) #else - #define JSON_HEDLEY_GNUC_HAS_CPP_ATTRIBUTE(attribute,major,minor,patch) JSON_HEDLEY_GNUC_VERSION_CHECK(major,minor,patch) +#define JSON_HEDLEY_GNUC_HAS_CPP_ATTRIBUTE(attribute, major, minor, patch) \ + JSON_HEDLEY_GNUC_VERSION_CHECK(major, minor, patch) #endif #if defined(JSON_HEDLEY_GCC_HAS_CPP_ATTRIBUTE) - #undef JSON_HEDLEY_GCC_HAS_CPP_ATTRIBUTE +#undef JSON_HEDLEY_GCC_HAS_CPP_ATTRIBUTE #endif #if defined(__has_cpp_attribute) && defined(__cplusplus) - #define JSON_HEDLEY_GCC_HAS_CPP_ATTRIBUTE(attribute,major,minor,patch) __has_cpp_attribute(attribute) +#define JSON_HEDLEY_GCC_HAS_CPP_ATTRIBUTE(attribute, major, minor, patch) __has_cpp_attribute(attribute) #else - #define JSON_HEDLEY_GCC_HAS_CPP_ATTRIBUTE(attribute,major,minor,patch) JSON_HEDLEY_GCC_VERSION_CHECK(major,minor,patch) +#define JSON_HEDLEY_GCC_HAS_CPP_ATTRIBUTE(attribute, major, minor, patch) \ + JSON_HEDLEY_GCC_VERSION_CHECK(major, minor, patch) #endif #if defined(JSON_HEDLEY_HAS_BUILTIN) - #undef JSON_HEDLEY_HAS_BUILTIN +#undef JSON_HEDLEY_HAS_BUILTIN #endif #if defined(__has_builtin) - #define JSON_HEDLEY_HAS_BUILTIN(builtin) __has_builtin(builtin) +#define JSON_HEDLEY_HAS_BUILTIN(builtin) __has_builtin(builtin) #else - #define JSON_HEDLEY_HAS_BUILTIN(builtin) (0) +#define JSON_HEDLEY_HAS_BUILTIN(builtin) (0) #endif #if defined(JSON_HEDLEY_GNUC_HAS_BUILTIN) - #undef JSON_HEDLEY_GNUC_HAS_BUILTIN +#undef JSON_HEDLEY_GNUC_HAS_BUILTIN #endif #if defined(__has_builtin) - #define JSON_HEDLEY_GNUC_HAS_BUILTIN(builtin,major,minor,patch) __has_builtin(builtin) +#define JSON_HEDLEY_GNUC_HAS_BUILTIN(builtin, major, minor, patch) __has_builtin(builtin) #else - #define JSON_HEDLEY_GNUC_HAS_BUILTIN(builtin,major,minor,patch) JSON_HEDLEY_GNUC_VERSION_CHECK(major,minor,patch) +#define JSON_HEDLEY_GNUC_HAS_BUILTIN(builtin, major, minor, patch) JSON_HEDLEY_GNUC_VERSION_CHECK(major, minor, patch) #endif #if defined(JSON_HEDLEY_GCC_HAS_BUILTIN) - #undef JSON_HEDLEY_GCC_HAS_BUILTIN +#undef JSON_HEDLEY_GCC_HAS_BUILTIN #endif #if defined(__has_builtin) - #define JSON_HEDLEY_GCC_HAS_BUILTIN(builtin,major,minor,patch) __has_builtin(builtin) +#define JSON_HEDLEY_GCC_HAS_BUILTIN(builtin, major, minor, patch) __has_builtin(builtin) #else - #define JSON_HEDLEY_GCC_HAS_BUILTIN(builtin,major,minor,patch) JSON_HEDLEY_GCC_VERSION_CHECK(major,minor,patch) +#define JSON_HEDLEY_GCC_HAS_BUILTIN(builtin, major, minor, patch) JSON_HEDLEY_GCC_VERSION_CHECK(major, minor, patch) #endif #if defined(JSON_HEDLEY_HAS_FEATURE) - #undef JSON_HEDLEY_HAS_FEATURE +#undef JSON_HEDLEY_HAS_FEATURE #endif #if defined(__has_feature) - #define JSON_HEDLEY_HAS_FEATURE(feature) __has_feature(feature) +#define JSON_HEDLEY_HAS_FEATURE(feature) __has_feature(feature) #else - #define JSON_HEDLEY_HAS_FEATURE(feature) (0) +#define JSON_HEDLEY_HAS_FEATURE(feature) (0) #endif #if defined(JSON_HEDLEY_GNUC_HAS_FEATURE) - #undef JSON_HEDLEY_GNUC_HAS_FEATURE +#undef JSON_HEDLEY_GNUC_HAS_FEATURE #endif #if defined(__has_feature) - #define JSON_HEDLEY_GNUC_HAS_FEATURE(feature,major,minor,patch) __has_feature(feature) +#define JSON_HEDLEY_GNUC_HAS_FEATURE(feature, major, minor, patch) __has_feature(feature) #else - #define JSON_HEDLEY_GNUC_HAS_FEATURE(feature,major,minor,patch) JSON_HEDLEY_GNUC_VERSION_CHECK(major,minor,patch) +#define JSON_HEDLEY_GNUC_HAS_FEATURE(feature, major, minor, patch) JSON_HEDLEY_GNUC_VERSION_CHECK(major, minor, patch) #endif #if defined(JSON_HEDLEY_GCC_HAS_FEATURE) - #undef JSON_HEDLEY_GCC_HAS_FEATURE +#undef JSON_HEDLEY_GCC_HAS_FEATURE #endif #if defined(__has_feature) - #define JSON_HEDLEY_GCC_HAS_FEATURE(feature,major,minor,patch) __has_feature(feature) +#define JSON_HEDLEY_GCC_HAS_FEATURE(feature, major, minor, patch) __has_feature(feature) #else - #define JSON_HEDLEY_GCC_HAS_FEATURE(feature,major,minor,patch) JSON_HEDLEY_GCC_VERSION_CHECK(major,minor,patch) +#define JSON_HEDLEY_GCC_HAS_FEATURE(feature, major, minor, patch) JSON_HEDLEY_GCC_VERSION_CHECK(major, minor, patch) #endif #if defined(JSON_HEDLEY_HAS_EXTENSION) - #undef JSON_HEDLEY_HAS_EXTENSION +#undef JSON_HEDLEY_HAS_EXTENSION #endif #if defined(__has_extension) - #define JSON_HEDLEY_HAS_EXTENSION(extension) __has_extension(extension) +#define JSON_HEDLEY_HAS_EXTENSION(extension) __has_extension(extension) #else - #define JSON_HEDLEY_HAS_EXTENSION(extension) (0) +#define JSON_HEDLEY_HAS_EXTENSION(extension) (0) #endif #if defined(JSON_HEDLEY_GNUC_HAS_EXTENSION) - #undef JSON_HEDLEY_GNUC_HAS_EXTENSION +#undef JSON_HEDLEY_GNUC_HAS_EXTENSION #endif #if defined(__has_extension) - #define JSON_HEDLEY_GNUC_HAS_EXTENSION(extension,major,minor,patch) __has_extension(extension) +#define JSON_HEDLEY_GNUC_HAS_EXTENSION(extension, major, minor, patch) __has_extension(extension) #else - #define JSON_HEDLEY_GNUC_HAS_EXTENSION(extension,major,minor,patch) JSON_HEDLEY_GNUC_VERSION_CHECK(major,minor,patch) +#define JSON_HEDLEY_GNUC_HAS_EXTENSION(extension, major, minor, patch) \ + JSON_HEDLEY_GNUC_VERSION_CHECK(major, minor, patch) #endif #if defined(JSON_HEDLEY_GCC_HAS_EXTENSION) - #undef JSON_HEDLEY_GCC_HAS_EXTENSION +#undef JSON_HEDLEY_GCC_HAS_EXTENSION #endif #if defined(__has_extension) - #define JSON_HEDLEY_GCC_HAS_EXTENSION(extension,major,minor,patch) __has_extension(extension) +#define JSON_HEDLEY_GCC_HAS_EXTENSION(extension, major, minor, patch) __has_extension(extension) #else - #define JSON_HEDLEY_GCC_HAS_EXTENSION(extension,major,minor,patch) JSON_HEDLEY_GCC_VERSION_CHECK(major,minor,patch) +#define JSON_HEDLEY_GCC_HAS_EXTENSION(extension, major, minor, patch) JSON_HEDLEY_GCC_VERSION_CHECK(major, minor, patch) #endif #if defined(JSON_HEDLEY_HAS_DECLSPEC_ATTRIBUTE) - #undef JSON_HEDLEY_HAS_DECLSPEC_ATTRIBUTE +#undef JSON_HEDLEY_HAS_DECLSPEC_ATTRIBUTE #endif #if defined(__has_declspec_attribute) - #define JSON_HEDLEY_HAS_DECLSPEC_ATTRIBUTE(attribute) __has_declspec_attribute(attribute) +#define JSON_HEDLEY_HAS_DECLSPEC_ATTRIBUTE(attribute) __has_declspec_attribute(attribute) #else - #define JSON_HEDLEY_HAS_DECLSPEC_ATTRIBUTE(attribute) (0) +#define JSON_HEDLEY_HAS_DECLSPEC_ATTRIBUTE(attribute) (0) #endif #if defined(JSON_HEDLEY_GNUC_HAS_DECLSPEC_ATTRIBUTE) - #undef JSON_HEDLEY_GNUC_HAS_DECLSPEC_ATTRIBUTE +#undef JSON_HEDLEY_GNUC_HAS_DECLSPEC_ATTRIBUTE #endif #if defined(__has_declspec_attribute) - #define JSON_HEDLEY_GNUC_HAS_DECLSPEC_ATTRIBUTE(attribute,major,minor,patch) __has_declspec_attribute(attribute) +#define JSON_HEDLEY_GNUC_HAS_DECLSPEC_ATTRIBUTE(attribute, major, minor, patch) __has_declspec_attribute(attribute) #else - #define JSON_HEDLEY_GNUC_HAS_DECLSPEC_ATTRIBUTE(attribute,major,minor,patch) JSON_HEDLEY_GNUC_VERSION_CHECK(major,minor,patch) +#define JSON_HEDLEY_GNUC_HAS_DECLSPEC_ATTRIBUTE(attribute, major, minor, patch) \ + JSON_HEDLEY_GNUC_VERSION_CHECK(major, minor, patch) #endif #if defined(JSON_HEDLEY_GCC_HAS_DECLSPEC_ATTRIBUTE) - #undef JSON_HEDLEY_GCC_HAS_DECLSPEC_ATTRIBUTE +#undef JSON_HEDLEY_GCC_HAS_DECLSPEC_ATTRIBUTE #endif #if defined(__has_declspec_attribute) - #define JSON_HEDLEY_GCC_HAS_DECLSPEC_ATTRIBUTE(attribute,major,minor,patch) __has_declspec_attribute(attribute) +#define JSON_HEDLEY_GCC_HAS_DECLSPEC_ATTRIBUTE(attribute, major, minor, patch) __has_declspec_attribute(attribute) #else - #define JSON_HEDLEY_GCC_HAS_DECLSPEC_ATTRIBUTE(attribute,major,minor,patch) JSON_HEDLEY_GCC_VERSION_CHECK(major,minor,patch) +#define JSON_HEDLEY_GCC_HAS_DECLSPEC_ATTRIBUTE(attribute, major, minor, patch) \ + JSON_HEDLEY_GCC_VERSION_CHECK(major, minor, patch) #endif #if defined(JSON_HEDLEY_HAS_WARNING) - #undef JSON_HEDLEY_HAS_WARNING +#undef JSON_HEDLEY_HAS_WARNING #endif #if defined(__has_warning) - #define JSON_HEDLEY_HAS_WARNING(warning) __has_warning(warning) +#define JSON_HEDLEY_HAS_WARNING(warning) __has_warning(warning) #else - #define JSON_HEDLEY_HAS_WARNING(warning) (0) +#define JSON_HEDLEY_HAS_WARNING(warning) (0) #endif #if defined(JSON_HEDLEY_GNUC_HAS_WARNING) - #undef JSON_HEDLEY_GNUC_HAS_WARNING +#undef JSON_HEDLEY_GNUC_HAS_WARNING #endif #if defined(__has_warning) - #define JSON_HEDLEY_GNUC_HAS_WARNING(warning,major,minor,patch) __has_warning(warning) +#define JSON_HEDLEY_GNUC_HAS_WARNING(warning, major, minor, patch) __has_warning(warning) #else - #define JSON_HEDLEY_GNUC_HAS_WARNING(warning,major,minor,patch) JSON_HEDLEY_GNUC_VERSION_CHECK(major,minor,patch) +#define JSON_HEDLEY_GNUC_HAS_WARNING(warning, major, minor, patch) JSON_HEDLEY_GNUC_VERSION_CHECK(major, minor, patch) #endif #if defined(JSON_HEDLEY_GCC_HAS_WARNING) - #undef JSON_HEDLEY_GCC_HAS_WARNING +#undef JSON_HEDLEY_GCC_HAS_WARNING #endif #if defined(__has_warning) - #define JSON_HEDLEY_GCC_HAS_WARNING(warning,major,minor,patch) __has_warning(warning) +#define JSON_HEDLEY_GCC_HAS_WARNING(warning, major, minor, patch) __has_warning(warning) #else - #define JSON_HEDLEY_GCC_HAS_WARNING(warning,major,minor,patch) JSON_HEDLEY_GCC_VERSION_CHECK(major,minor,patch) +#define JSON_HEDLEY_GCC_HAS_WARNING(warning, major, minor, patch) JSON_HEDLEY_GCC_VERSION_CHECK(major, minor, patch) #endif -#if \ - (defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L)) || \ - defined(__clang__) || \ - JSON_HEDLEY_GCC_VERSION_CHECK(3,0,0) || \ - JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ - JSON_HEDLEY_IAR_VERSION_CHECK(8,0,0) || \ - JSON_HEDLEY_PGI_VERSION_CHECK(18,4,0) || \ - JSON_HEDLEY_ARM_VERSION_CHECK(4,1,0) || \ - JSON_HEDLEY_TI_VERSION_CHECK(6,0,0) || \ - JSON_HEDLEY_CRAY_VERSION_CHECK(5,0,0) || \ - JSON_HEDLEY_TINYC_VERSION_CHECK(0,9,17) || \ - JSON_HEDLEY_SUNPRO_VERSION_CHECK(8,0,0) || \ - (JSON_HEDLEY_IBM_VERSION_CHECK(10,1,0) && defined(__C99_PRAGMA_OPERATOR)) - #define JSON_HEDLEY_PRAGMA(value) _Pragma(#value) -#elif JSON_HEDLEY_MSVC_VERSION_CHECK(15,0,0) - #define JSON_HEDLEY_PRAGMA(value) __pragma(value) +#if (defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L)) || defined(__clang__) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(3, 0, 0) || JSON_HEDLEY_INTEL_VERSION_CHECK(13, 0, 0) || \ + JSON_HEDLEY_IAR_VERSION_CHECK(8, 0, 0) || JSON_HEDLEY_PGI_VERSION_CHECK(18, 4, 0) || \ + JSON_HEDLEY_ARM_VERSION_CHECK(4, 1, 0) || JSON_HEDLEY_TI_VERSION_CHECK(6, 0, 0) || \ + JSON_HEDLEY_CRAY_VERSION_CHECK(5, 0, 0) || JSON_HEDLEY_TINYC_VERSION_CHECK(0, 9, 17) || \ + JSON_HEDLEY_SUNPRO_VERSION_CHECK(8, 0, 0) || \ + (JSON_HEDLEY_IBM_VERSION_CHECK(10, 1, 0) && defined(__C99_PRAGMA_OPERATOR)) +#define JSON_HEDLEY_PRAGMA(value) _Pragma(#value) +#elif JSON_HEDLEY_MSVC_VERSION_CHECK(15, 0, 0) +#define JSON_HEDLEY_PRAGMA(value) __pragma(value) #else - #define JSON_HEDLEY_PRAGMA(value) +#define JSON_HEDLEY_PRAGMA(value) #endif #if defined(JSON_HEDLEY_DIAGNOSTIC_PUSH) - #undef JSON_HEDLEY_DIAGNOSTIC_PUSH +#undef JSON_HEDLEY_DIAGNOSTIC_PUSH #endif #if defined(JSON_HEDLEY_DIAGNOSTIC_POP) - #undef JSON_HEDLEY_DIAGNOSTIC_POP +#undef JSON_HEDLEY_DIAGNOSTIC_POP #endif #if defined(__clang__) - #define JSON_HEDLEY_DIAGNOSTIC_PUSH _Pragma("clang diagnostic push") - #define JSON_HEDLEY_DIAGNOSTIC_POP _Pragma("clang diagnostic pop") -#elif JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) - #define JSON_HEDLEY_DIAGNOSTIC_PUSH _Pragma("warning(push)") - #define JSON_HEDLEY_DIAGNOSTIC_POP _Pragma("warning(pop)") -#elif JSON_HEDLEY_GCC_VERSION_CHECK(4,6,0) - #define JSON_HEDLEY_DIAGNOSTIC_PUSH _Pragma("GCC diagnostic push") - #define JSON_HEDLEY_DIAGNOSTIC_POP _Pragma("GCC diagnostic pop") -#elif JSON_HEDLEY_MSVC_VERSION_CHECK(15,0,0) - #define JSON_HEDLEY_DIAGNOSTIC_PUSH __pragma(warning(push)) - #define JSON_HEDLEY_DIAGNOSTIC_POP __pragma(warning(pop)) -#elif JSON_HEDLEY_ARM_VERSION_CHECK(5,6,0) - #define JSON_HEDLEY_DIAGNOSTIC_PUSH _Pragma("push") - #define JSON_HEDLEY_DIAGNOSTIC_POP _Pragma("pop") -#elif JSON_HEDLEY_TI_VERSION_CHECK(8,1,0) - #define JSON_HEDLEY_DIAGNOSTIC_PUSH _Pragma("diag_push") - #define JSON_HEDLEY_DIAGNOSTIC_POP _Pragma("diag_pop") -#elif JSON_HEDLEY_PELLES_VERSION_CHECK(2,90,0) - #define JSON_HEDLEY_DIAGNOSTIC_PUSH _Pragma("warning(push)") - #define JSON_HEDLEY_DIAGNOSTIC_POP _Pragma("warning(pop)") +#define JSON_HEDLEY_DIAGNOSTIC_PUSH _Pragma("clang diagnostic push") +#define JSON_HEDLEY_DIAGNOSTIC_POP _Pragma("clang diagnostic pop") +#elif JSON_HEDLEY_INTEL_VERSION_CHECK(13, 0, 0) +#define JSON_HEDLEY_DIAGNOSTIC_PUSH _Pragma("warning(push)") +#define JSON_HEDLEY_DIAGNOSTIC_POP _Pragma("warning(pop)") +#elif JSON_HEDLEY_GCC_VERSION_CHECK(4, 6, 0) +#define JSON_HEDLEY_DIAGNOSTIC_PUSH _Pragma("GCC diagnostic push") +#define JSON_HEDLEY_DIAGNOSTIC_POP _Pragma("GCC diagnostic pop") +#elif JSON_HEDLEY_MSVC_VERSION_CHECK(15, 0, 0) +#define JSON_HEDLEY_DIAGNOSTIC_PUSH __pragma(warning(push)) +#define JSON_HEDLEY_DIAGNOSTIC_POP __pragma(warning(pop)) +#elif JSON_HEDLEY_ARM_VERSION_CHECK(5, 6, 0) +#define JSON_HEDLEY_DIAGNOSTIC_PUSH _Pragma("push") +#define JSON_HEDLEY_DIAGNOSTIC_POP _Pragma("pop") +#elif JSON_HEDLEY_TI_VERSION_CHECK(8, 1, 0) +#define JSON_HEDLEY_DIAGNOSTIC_PUSH _Pragma("diag_push") +#define JSON_HEDLEY_DIAGNOSTIC_POP _Pragma("diag_pop") +#elif JSON_HEDLEY_PELLES_VERSION_CHECK(2, 90, 0) +#define JSON_HEDLEY_DIAGNOSTIC_PUSH _Pragma("warning(push)") +#define JSON_HEDLEY_DIAGNOSTIC_POP _Pragma("warning(pop)") #else - #define JSON_HEDLEY_DIAGNOSTIC_PUSH - #define JSON_HEDLEY_DIAGNOSTIC_POP +#define JSON_HEDLEY_DIAGNOSTIC_PUSH +#define JSON_HEDLEY_DIAGNOSTIC_POP #endif #if defined(JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED) - #undef JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED +#undef JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED #endif #if JSON_HEDLEY_HAS_WARNING("-Wdeprecated-declarations") - #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED _Pragma("clang diagnostic ignored \"-Wdeprecated-declarations\"") -#elif JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) - #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED _Pragma("warning(disable:1478 1786)") -#elif JSON_HEDLEY_PGI_VERSION_CHECK(17,10,0) - #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED _Pragma("diag_suppress 1215,1444") -#elif JSON_HEDLEY_GCC_VERSION_CHECK(4,3,0) - #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED _Pragma("GCC diagnostic ignored \"-Wdeprecated-declarations\"") -#elif JSON_HEDLEY_MSVC_VERSION_CHECK(15,0,0) - #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED __pragma(warning(disable:4996)) -#elif JSON_HEDLEY_TI_VERSION_CHECK(8,0,0) - #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED _Pragma("diag_suppress 1291,1718") -#elif JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,13,0) && !defined(__cplusplus) - #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED _Pragma("error_messages(off,E_DEPRECATED_ATT,E_DEPRECATED_ATT_MESS)") -#elif JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,13,0) && defined(__cplusplus) - #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED _Pragma("error_messages(off,symdeprecated,symdeprecated2)") -#elif JSON_HEDLEY_IAR_VERSION_CHECK(8,0,0) - #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED _Pragma("diag_suppress=Pe1444,Pe1215") -#elif JSON_HEDLEY_PELLES_VERSION_CHECK(2,90,0) - #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED _Pragma("warn(disable:2241)") +#define JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED _Pragma("clang diagnostic ignored \"-Wdeprecated-declarations\"") +#elif JSON_HEDLEY_INTEL_VERSION_CHECK(13, 0, 0) +#define JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED _Pragma("warning(disable:1478 1786)") +#elif JSON_HEDLEY_PGI_VERSION_CHECK(17, 10, 0) +#define JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED _Pragma("diag_suppress 1215,1444") +#elif JSON_HEDLEY_GCC_VERSION_CHECK(4, 3, 0) +#define JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED _Pragma("GCC diagnostic ignored \"-Wdeprecated-declarations\"") +#elif JSON_HEDLEY_MSVC_VERSION_CHECK(15, 0, 0) +#define JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED __pragma(warning(disable : 4996)) +#elif JSON_HEDLEY_TI_VERSION_CHECK(8, 0, 0) +#define JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED _Pragma("diag_suppress 1291,1718") +#elif JSON_HEDLEY_SUNPRO_VERSION_CHECK(5, 13, 0) && !defined(__cplusplus) +#define JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED _Pragma("error_messages(off,E_DEPRECATED_ATT,E_DEPRECATED_ATT_MESS)") +#elif JSON_HEDLEY_SUNPRO_VERSION_CHECK(5, 13, 0) && defined(__cplusplus) +#define JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED _Pragma("error_messages(off,symdeprecated,symdeprecated2)") +#elif JSON_HEDLEY_IAR_VERSION_CHECK(8, 0, 0) +#define JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED _Pragma("diag_suppress=Pe1444,Pe1215") +#elif JSON_HEDLEY_PELLES_VERSION_CHECK(2, 90, 0) +#define JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED _Pragma("warn(disable:2241)") #else - #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED +#define JSON_HEDLEY_DIAGNOSTIC_DISABLE_DEPRECATED #endif #if defined(JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS) - #undef JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS +#undef JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS #endif #if JSON_HEDLEY_HAS_WARNING("-Wunknown-pragmas") - #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS _Pragma("clang diagnostic ignored \"-Wunknown-pragmas\"") -#elif JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) - #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS _Pragma("warning(disable:161)") -#elif JSON_HEDLEY_PGI_VERSION_CHECK(17,10,0) - #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS _Pragma("diag_suppress 1675") -#elif JSON_HEDLEY_GCC_VERSION_CHECK(4,3,0) - #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS _Pragma("GCC diagnostic ignored \"-Wunknown-pragmas\"") -#elif JSON_HEDLEY_MSVC_VERSION_CHECK(15,0,0) - #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS __pragma(warning(disable:4068)) -#elif JSON_HEDLEY_TI_VERSION_CHECK(8,0,0) - #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS _Pragma("diag_suppress 163") -#elif JSON_HEDLEY_IAR_VERSION_CHECK(8,0,0) - #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS _Pragma("diag_suppress=Pe161") +#define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS _Pragma("clang diagnostic ignored \"-Wunknown-pragmas\"") +#elif JSON_HEDLEY_INTEL_VERSION_CHECK(13, 0, 0) +#define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS _Pragma("warning(disable:161)") +#elif JSON_HEDLEY_PGI_VERSION_CHECK(17, 10, 0) +#define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS _Pragma("diag_suppress 1675") +#elif JSON_HEDLEY_GCC_VERSION_CHECK(4, 3, 0) +#define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS _Pragma("GCC diagnostic ignored \"-Wunknown-pragmas\"") +#elif JSON_HEDLEY_MSVC_VERSION_CHECK(15, 0, 0) +#define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS __pragma(warning(disable : 4068)) +#elif JSON_HEDLEY_TI_VERSION_CHECK(8, 0, 0) +#define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS _Pragma("diag_suppress 163") +#elif JSON_HEDLEY_IAR_VERSION_CHECK(8, 0, 0) +#define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS _Pragma("diag_suppress=Pe161") #else - #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS +#define JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS #endif #if defined(JSON_HEDLEY_DIAGNOSTIC_DISABLE_CAST_QUAL) - #undef JSON_HEDLEY_DIAGNOSTIC_DISABLE_CAST_QUAL +#undef JSON_HEDLEY_DIAGNOSTIC_DISABLE_CAST_QUAL #endif #if JSON_HEDLEY_HAS_WARNING("-Wcast-qual") - #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_CAST_QUAL _Pragma("clang diagnostic ignored \"-Wcast-qual\"") -#elif JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) - #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_CAST_QUAL _Pragma("warning(disable:2203 2331)") -#elif JSON_HEDLEY_GCC_VERSION_CHECK(3,0,0) - #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_CAST_QUAL _Pragma("GCC diagnostic ignored \"-Wcast-qual\"") +#define JSON_HEDLEY_DIAGNOSTIC_DISABLE_CAST_QUAL _Pragma("clang diagnostic ignored \"-Wcast-qual\"") +#elif JSON_HEDLEY_INTEL_VERSION_CHECK(13, 0, 0) +#define JSON_HEDLEY_DIAGNOSTIC_DISABLE_CAST_QUAL _Pragma("warning(disable:2203 2331)") +#elif JSON_HEDLEY_GCC_VERSION_CHECK(3, 0, 0) +#define JSON_HEDLEY_DIAGNOSTIC_DISABLE_CAST_QUAL _Pragma("GCC diagnostic ignored \"-Wcast-qual\"") #else - #define JSON_HEDLEY_DIAGNOSTIC_DISABLE_CAST_QUAL +#define JSON_HEDLEY_DIAGNOSTIC_DISABLE_CAST_QUAL #endif #if defined(JSON_HEDLEY_DEPRECATED) - #undef JSON_HEDLEY_DEPRECATED +#undef JSON_HEDLEY_DEPRECATED #endif #if defined(JSON_HEDLEY_DEPRECATED_FOR) - #undef JSON_HEDLEY_DEPRECATED_FOR +#undef JSON_HEDLEY_DEPRECATED_FOR #endif #if defined(__cplusplus) && (__cplusplus >= 201402L) - #define JSON_HEDLEY_DEPRECATED(since) [[deprecated("Since " #since)]] - #define JSON_HEDLEY_DEPRECATED_FOR(since, replacement) [[deprecated("Since " #since "; use " #replacement)]] -#elif \ - JSON_HEDLEY_HAS_EXTENSION(attribute_deprecated_with_message) || \ - JSON_HEDLEY_GCC_VERSION_CHECK(4,5,0) || \ - JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ - JSON_HEDLEY_ARM_VERSION_CHECK(5,6,0) || \ - JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,13,0) || \ - JSON_HEDLEY_PGI_VERSION_CHECK(17,10,0) || \ - JSON_HEDLEY_TI_VERSION_CHECK(8,3,0) - #define JSON_HEDLEY_DEPRECATED(since) __attribute__((__deprecated__("Since " #since))) - #define JSON_HEDLEY_DEPRECATED_FOR(since, replacement) __attribute__((__deprecated__("Since " #since "; use " #replacement))) -#elif \ - JSON_HEDLEY_HAS_ATTRIBUTE(deprecated) || \ - JSON_HEDLEY_GCC_VERSION_CHECK(3,1,0) || \ - JSON_HEDLEY_ARM_VERSION_CHECK(4,1,0) || \ - JSON_HEDLEY_TI_VERSION_CHECK(8,0,0) || \ - (JSON_HEDLEY_TI_VERSION_CHECK(7,3,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) - #define JSON_HEDLEY_DEPRECATED(since) __attribute__((__deprecated__)) - #define JSON_HEDLEY_DEPRECATED_FOR(since, replacement) __attribute__((__deprecated__)) -#elif JSON_HEDLEY_MSVC_VERSION_CHECK(14,0,0) - #define JSON_HEDLEY_DEPRECATED(since) __declspec(deprecated("Since " # since)) - #define JSON_HEDLEY_DEPRECATED_FOR(since, replacement) __declspec(deprecated("Since " #since "; use " #replacement)) -#elif \ - JSON_HEDLEY_MSVC_VERSION_CHECK(13,10,0) || \ - JSON_HEDLEY_PELLES_VERSION_CHECK(6,50,0) - #define JSON_HEDLEY_DEPRECATED(since) _declspec(deprecated) - #define JSON_HEDLEY_DEPRECATED_FOR(since, replacement) __declspec(deprecated) -#elif JSON_HEDLEY_IAR_VERSION_CHECK(8,0,0) - #define JSON_HEDLEY_DEPRECATED(since) _Pragma("deprecated") - #define JSON_HEDLEY_DEPRECATED_FOR(since, replacement) _Pragma("deprecated") +#define JSON_HEDLEY_DEPRECATED(since) [[deprecated("Since " #since)]] +#define JSON_HEDLEY_DEPRECATED_FOR(since, replacement) [[deprecated("Since " #since "; use " #replacement)]] +#elif JSON_HEDLEY_HAS_EXTENSION(attribute_deprecated_with_message) || JSON_HEDLEY_GCC_VERSION_CHECK(4, 5, 0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13, 0, 0) || JSON_HEDLEY_ARM_VERSION_CHECK(5, 6, 0) || \ + JSON_HEDLEY_SUNPRO_VERSION_CHECK(5, 13, 0) || JSON_HEDLEY_PGI_VERSION_CHECK(17, 10, 0) || \ + JSON_HEDLEY_TI_VERSION_CHECK(8, 3, 0) +#define JSON_HEDLEY_DEPRECATED(since) __attribute__((__deprecated__("Since " #since))) +#define JSON_HEDLEY_DEPRECATED_FOR(since, replacement) \ + __attribute__((__deprecated__("Since " #since "; use " #replacement))) +#elif JSON_HEDLEY_HAS_ATTRIBUTE(deprecated) || JSON_HEDLEY_GCC_VERSION_CHECK(3, 1, 0) || \ + JSON_HEDLEY_ARM_VERSION_CHECK(4, 1, 0) || JSON_HEDLEY_TI_VERSION_CHECK(8, 0, 0) || \ + (JSON_HEDLEY_TI_VERSION_CHECK(7, 3, 0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) +#define JSON_HEDLEY_DEPRECATED(since) __attribute__((__deprecated__)) +#define JSON_HEDLEY_DEPRECATED_FOR(since, replacement) __attribute__((__deprecated__)) +#elif JSON_HEDLEY_MSVC_VERSION_CHECK(14, 0, 0) +#define JSON_HEDLEY_DEPRECATED(since) __declspec(deprecated("Since " #since)) +#define JSON_HEDLEY_DEPRECATED_FOR(since, replacement) __declspec(deprecated("Since " #since "; use " #replacement)) +#elif JSON_HEDLEY_MSVC_VERSION_CHECK(13, 10, 0) || JSON_HEDLEY_PELLES_VERSION_CHECK(6, 50, 0) +#define JSON_HEDLEY_DEPRECATED(since) _declspec(deprecated) +#define JSON_HEDLEY_DEPRECATED_FOR(since, replacement) __declspec(deprecated) +#elif JSON_HEDLEY_IAR_VERSION_CHECK(8, 0, 0) +#define JSON_HEDLEY_DEPRECATED(since) _Pragma("deprecated") +#define JSON_HEDLEY_DEPRECATED_FOR(since, replacement) _Pragma("deprecated") #else - #define JSON_HEDLEY_DEPRECATED(since) - #define JSON_HEDLEY_DEPRECATED_FOR(since, replacement) +#define JSON_HEDLEY_DEPRECATED(since) +#define JSON_HEDLEY_DEPRECATED_FOR(since, replacement) #endif #if defined(JSON_HEDLEY_UNAVAILABLE) - #undef JSON_HEDLEY_UNAVAILABLE +#undef JSON_HEDLEY_UNAVAILABLE #endif -#if \ - JSON_HEDLEY_HAS_ATTRIBUTE(warning) || \ - JSON_HEDLEY_GCC_VERSION_CHECK(4,3,0) || \ - JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) - #define JSON_HEDLEY_UNAVAILABLE(available_since) __attribute__((__warning__("Not available until " #available_since))) +#if JSON_HEDLEY_HAS_ATTRIBUTE(warning) || JSON_HEDLEY_GCC_VERSION_CHECK(4, 3, 0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13, 0, 0) +#define JSON_HEDLEY_UNAVAILABLE(available_since) __attribute__((__warning__("Not available until " #available_since))) #else - #define JSON_HEDLEY_UNAVAILABLE(available_since) +#define JSON_HEDLEY_UNAVAILABLE(available_since) #endif #if defined(JSON_HEDLEY_WARN_UNUSED_RESULT) - #undef JSON_HEDLEY_WARN_UNUSED_RESULT +#undef JSON_HEDLEY_WARN_UNUSED_RESULT #endif #if defined(__cplusplus) && (__cplusplus >= 201703L) - #define JSON_HEDLEY_WARN_UNUSED_RESULT [[nodiscard]] -#elif \ - JSON_HEDLEY_HAS_ATTRIBUTE(warn_unused_result) || \ - JSON_HEDLEY_GCC_VERSION_CHECK(3,4,0) || \ - JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ - JSON_HEDLEY_TI_VERSION_CHECK(8,0,0) || \ - (JSON_HEDLEY_TI_VERSION_CHECK(7,3,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ - (JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,15,0) && defined(__cplusplus)) || \ - JSON_HEDLEY_PGI_VERSION_CHECK(17,10,0) - #define JSON_HEDLEY_WARN_UNUSED_RESULT __attribute__((__warn_unused_result__)) +#define JSON_HEDLEY_WARN_UNUSED_RESULT [[nodiscard]] +#elif JSON_HEDLEY_HAS_ATTRIBUTE(warn_unused_result) || JSON_HEDLEY_GCC_VERSION_CHECK(3, 4, 0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13, 0, 0) || JSON_HEDLEY_TI_VERSION_CHECK(8, 0, 0) || \ + (JSON_HEDLEY_TI_VERSION_CHECK(7, 3, 0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + (JSON_HEDLEY_SUNPRO_VERSION_CHECK(5, 15, 0) && defined(__cplusplus)) || JSON_HEDLEY_PGI_VERSION_CHECK(17, 10, 0) +#define JSON_HEDLEY_WARN_UNUSED_RESULT __attribute__((__warn_unused_result__)) #elif defined(_Check_return_) /* SAL */ - #define JSON_HEDLEY_WARN_UNUSED_RESULT _Check_return_ +#define JSON_HEDLEY_WARN_UNUSED_RESULT _Check_return_ #else - #define JSON_HEDLEY_WARN_UNUSED_RESULT +#define JSON_HEDLEY_WARN_UNUSED_RESULT #endif #if defined(JSON_HEDLEY_SENTINEL) - #undef JSON_HEDLEY_SENTINEL +#undef JSON_HEDLEY_SENTINEL #endif -#if \ - JSON_HEDLEY_HAS_ATTRIBUTE(sentinel) || \ - JSON_HEDLEY_GCC_VERSION_CHECK(4,0,0) || \ - JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ - JSON_HEDLEY_ARM_VERSION_CHECK(5,4,0) - #define JSON_HEDLEY_SENTINEL(position) __attribute__((__sentinel__(position))) +#if JSON_HEDLEY_HAS_ATTRIBUTE(sentinel) || JSON_HEDLEY_GCC_VERSION_CHECK(4, 0, 0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13, 0, 0) || JSON_HEDLEY_ARM_VERSION_CHECK(5, 4, 0) +#define JSON_HEDLEY_SENTINEL(position) __attribute__((__sentinel__(position))) #else - #define JSON_HEDLEY_SENTINEL(position) +#define JSON_HEDLEY_SENTINEL(position) #endif #if defined(JSON_HEDLEY_NO_RETURN) - #undef JSON_HEDLEY_NO_RETURN +#undef JSON_HEDLEY_NO_RETURN #endif -#if JSON_HEDLEY_IAR_VERSION_CHECK(8,0,0) - #define JSON_HEDLEY_NO_RETURN __noreturn -#elif JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) - #define JSON_HEDLEY_NO_RETURN __attribute__((__noreturn__)) +#if JSON_HEDLEY_IAR_VERSION_CHECK(8, 0, 0) +#define JSON_HEDLEY_NO_RETURN __noreturn +#elif JSON_HEDLEY_INTEL_VERSION_CHECK(13, 0, 0) +#define JSON_HEDLEY_NO_RETURN __attribute__((__noreturn__)) #elif defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L - #define JSON_HEDLEY_NO_RETURN _Noreturn +#define JSON_HEDLEY_NO_RETURN _Noreturn #elif defined(__cplusplus) && (__cplusplus >= 201103L) - #define JSON_HEDLEY_NO_RETURN [[noreturn]] -#elif \ - JSON_HEDLEY_HAS_ATTRIBUTE(noreturn) || \ - JSON_HEDLEY_GCC_VERSION_CHECK(3,2,0) || \ - JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,11,0) || \ - JSON_HEDLEY_ARM_VERSION_CHECK(4,1,0) || \ - JSON_HEDLEY_IBM_VERSION_CHECK(10,1,0) || \ - JSON_HEDLEY_TI_VERSION_CHECK(18,0,0) || \ - (JSON_HEDLEY_TI_VERSION_CHECK(17,3,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) - #define JSON_HEDLEY_NO_RETURN __attribute__((__noreturn__)) -#elif JSON_HEDLEY_MSVC_VERSION_CHECK(13,10,0) - #define JSON_HEDLEY_NO_RETURN __declspec(noreturn) -#elif JSON_HEDLEY_TI_VERSION_CHECK(6,0,0) && defined(__cplusplus) - #define JSON_HEDLEY_NO_RETURN _Pragma("FUNC_NEVER_RETURNS;") -#elif JSON_HEDLEY_COMPCERT_VERSION_CHECK(3,2,0) - #define JSON_HEDLEY_NO_RETURN __attribute((noreturn)) -#elif JSON_HEDLEY_PELLES_VERSION_CHECK(9,0,0) - #define JSON_HEDLEY_NO_RETURN __declspec(noreturn) +#define JSON_HEDLEY_NO_RETURN [[noreturn]] +#elif JSON_HEDLEY_HAS_ATTRIBUTE(noreturn) || JSON_HEDLEY_GCC_VERSION_CHECK(3, 2, 0) || \ + JSON_HEDLEY_SUNPRO_VERSION_CHECK(5, 11, 0) || JSON_HEDLEY_ARM_VERSION_CHECK(4, 1, 0) || \ + JSON_HEDLEY_IBM_VERSION_CHECK(10, 1, 0) || JSON_HEDLEY_TI_VERSION_CHECK(18, 0, 0) || \ + (JSON_HEDLEY_TI_VERSION_CHECK(17, 3, 0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) +#define JSON_HEDLEY_NO_RETURN __attribute__((__noreturn__)) +#elif JSON_HEDLEY_MSVC_VERSION_CHECK(13, 10, 0) +#define JSON_HEDLEY_NO_RETURN __declspec(noreturn) +#elif JSON_HEDLEY_TI_VERSION_CHECK(6, 0, 0) && defined(__cplusplus) +#define JSON_HEDLEY_NO_RETURN _Pragma("FUNC_NEVER_RETURNS;") +#elif JSON_HEDLEY_COMPCERT_VERSION_CHECK(3, 2, 0) +#define JSON_HEDLEY_NO_RETURN __attribute((noreturn)) +#elif JSON_HEDLEY_PELLES_VERSION_CHECK(9, 0, 0) +#define JSON_HEDLEY_NO_RETURN __declspec(noreturn) #else - #define JSON_HEDLEY_NO_RETURN +#define JSON_HEDLEY_NO_RETURN #endif #if defined(JSON_HEDLEY_UNREACHABLE) - #undef JSON_HEDLEY_UNREACHABLE +#undef JSON_HEDLEY_UNREACHABLE #endif #if defined(JSON_HEDLEY_UNREACHABLE_RETURN) - #undef JSON_HEDLEY_UNREACHABLE_RETURN +#undef JSON_HEDLEY_UNREACHABLE_RETURN #endif -#if \ - (JSON_HEDLEY_HAS_BUILTIN(__builtin_unreachable) && (!defined(JSON_HEDLEY_ARM_VERSION))) || \ - JSON_HEDLEY_GCC_VERSION_CHECK(4,5,0) || \ - JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ - JSON_HEDLEY_IBM_VERSION_CHECK(13,1,5) - #define JSON_HEDLEY_UNREACHABLE() __builtin_unreachable() -#elif JSON_HEDLEY_MSVC_VERSION_CHECK(13,10,0) - #define JSON_HEDLEY_UNREACHABLE() __assume(0) -#elif JSON_HEDLEY_TI_VERSION_CHECK(6,0,0) - #if defined(__cplusplus) - #define JSON_HEDLEY_UNREACHABLE() std::_nassert(0) - #else - #define JSON_HEDLEY_UNREACHABLE() _nassert(0) - #endif - #define JSON_HEDLEY_UNREACHABLE_RETURN(value) return value -#elif defined(EXIT_FAILURE) - #define JSON_HEDLEY_UNREACHABLE() abort() +#if (JSON_HEDLEY_HAS_BUILTIN(__builtin_unreachable) && (!defined(JSON_HEDLEY_ARM_VERSION))) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(4, 5, 0) || JSON_HEDLEY_INTEL_VERSION_CHECK(13, 0, 0) || \ + JSON_HEDLEY_IBM_VERSION_CHECK(13, 1, 5) +#define JSON_HEDLEY_UNREACHABLE() __builtin_unreachable() +#elif JSON_HEDLEY_MSVC_VERSION_CHECK(13, 10, 0) +#define JSON_HEDLEY_UNREACHABLE() __assume(0) +#elif JSON_HEDLEY_TI_VERSION_CHECK(6, 0, 0) +#if defined(__cplusplus) +#define JSON_HEDLEY_UNREACHABLE() std::_nassert(0) #else - #define JSON_HEDLEY_UNREACHABLE() - #define JSON_HEDLEY_UNREACHABLE_RETURN(value) return value +#define JSON_HEDLEY_UNREACHABLE() _nassert(0) +#endif +#define JSON_HEDLEY_UNREACHABLE_RETURN(value) return value +#elif defined(EXIT_FAILURE) +#define JSON_HEDLEY_UNREACHABLE() abort() +#else +#define JSON_HEDLEY_UNREACHABLE() +#define JSON_HEDLEY_UNREACHABLE_RETURN(value) return value #endif #if !defined(JSON_HEDLEY_UNREACHABLE_RETURN) - #define JSON_HEDLEY_UNREACHABLE_RETURN(value) JSON_HEDLEY_UNREACHABLE() +#define JSON_HEDLEY_UNREACHABLE_RETURN(value) JSON_HEDLEY_UNREACHABLE() #endif #if defined(JSON_HEDLEY_ASSUME) - #undef JSON_HEDLEY_ASSUME +#undef JSON_HEDLEY_ASSUME #endif -#if \ - JSON_HEDLEY_MSVC_VERSION_CHECK(13,10,0) || \ - JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) - #define JSON_HEDLEY_ASSUME(expr) __assume(expr) +#if JSON_HEDLEY_MSVC_VERSION_CHECK(13, 10, 0) || JSON_HEDLEY_INTEL_VERSION_CHECK(13, 0, 0) +#define JSON_HEDLEY_ASSUME(expr) __assume(expr) #elif JSON_HEDLEY_HAS_BUILTIN(__builtin_assume) - #define JSON_HEDLEY_ASSUME(expr) __builtin_assume(expr) -#elif JSON_HEDLEY_TI_VERSION_CHECK(6,0,0) - #if defined(__cplusplus) - #define JSON_HEDLEY_ASSUME(expr) std::_nassert(expr) - #else - #define JSON_HEDLEY_ASSUME(expr) _nassert(expr) - #endif -#elif \ - (JSON_HEDLEY_HAS_BUILTIN(__builtin_unreachable) && !defined(JSON_HEDLEY_ARM_VERSION)) || \ - JSON_HEDLEY_GCC_VERSION_CHECK(4,5,0) || \ - JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ - JSON_HEDLEY_IBM_VERSION_CHECK(13,1,5) - #define JSON_HEDLEY_ASSUME(expr) ((void) ((expr) ? 1 : (__builtin_unreachable(), 1))) +#define JSON_HEDLEY_ASSUME(expr) __builtin_assume(expr) +#elif JSON_HEDLEY_TI_VERSION_CHECK(6, 0, 0) +#if defined(__cplusplus) +#define JSON_HEDLEY_ASSUME(expr) std::_nassert(expr) #else - #define JSON_HEDLEY_ASSUME(expr) ((void) (expr)) +#define JSON_HEDLEY_ASSUME(expr) _nassert(expr) +#endif +#elif (JSON_HEDLEY_HAS_BUILTIN(__builtin_unreachable) && !defined(JSON_HEDLEY_ARM_VERSION)) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(4, 5, 0) || JSON_HEDLEY_INTEL_VERSION_CHECK(13, 0, 0) || \ + JSON_HEDLEY_IBM_VERSION_CHECK(13, 1, 5) +#define JSON_HEDLEY_ASSUME(expr) ((void)((expr) ? 1 : (__builtin_unreachable(), 1))) +#else +#define JSON_HEDLEY_ASSUME(expr) ((void)(expr)) #endif - JSON_HEDLEY_DIAGNOSTIC_PUSH -#if \ - JSON_HEDLEY_HAS_WARNING("-Wvariadic-macros") || \ - JSON_HEDLEY_GCC_VERSION_CHECK(4,0,0) - #if defined(__clang__) - #pragma clang diagnostic ignored "-Wvariadic-macros" - #elif defined(JSON_HEDLEY_GCC_VERSION) - #pragma GCC diagnostic ignored "-Wvariadic-macros" - #endif +#if JSON_HEDLEY_HAS_WARNING("-Wvariadic-macros") || JSON_HEDLEY_GCC_VERSION_CHECK(4, 0, 0) +#if defined(__clang__) +#pragma clang diagnostic ignored "-Wvariadic-macros" +#elif defined(JSON_HEDLEY_GCC_VERSION) +#pragma GCC diagnostic ignored "-Wvariadic-macros" +#endif #endif #if defined(JSON_HEDLEY_NON_NULL) - #undef JSON_HEDLEY_NON_NULL +#undef JSON_HEDLEY_NON_NULL #endif -#if \ - JSON_HEDLEY_HAS_ATTRIBUTE(nonnull) || \ - JSON_HEDLEY_GCC_VERSION_CHECK(3,3,0) || \ - JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ - JSON_HEDLEY_ARM_VERSION_CHECK(4,1,0) - #define JSON_HEDLEY_NON_NULL(...) __attribute__((__nonnull__(__VA_ARGS__))) +#if JSON_HEDLEY_HAS_ATTRIBUTE(nonnull) || JSON_HEDLEY_GCC_VERSION_CHECK(3, 3, 0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13, 0, 0) || JSON_HEDLEY_ARM_VERSION_CHECK(4, 1, 0) +#define JSON_HEDLEY_NON_NULL(...) __attribute__((__nonnull__(__VA_ARGS__))) #else - #define JSON_HEDLEY_NON_NULL(...) +#define JSON_HEDLEY_NON_NULL(...) #endif JSON_HEDLEY_DIAGNOSTIC_POP #if defined(JSON_HEDLEY_PRINTF_FORMAT) - #undef JSON_HEDLEY_PRINTF_FORMAT +#undef JSON_HEDLEY_PRINTF_FORMAT #endif -#if defined(__MINGW32__) && JSON_HEDLEY_GCC_HAS_ATTRIBUTE(format,4,4,0) && !defined(__USE_MINGW_ANSI_STDIO) - #define JSON_HEDLEY_PRINTF_FORMAT(string_idx,first_to_check) __attribute__((__format__(ms_printf, string_idx, first_to_check))) -#elif defined(__MINGW32__) && JSON_HEDLEY_GCC_HAS_ATTRIBUTE(format,4,4,0) && defined(__USE_MINGW_ANSI_STDIO) - #define JSON_HEDLEY_PRINTF_FORMAT(string_idx,first_to_check) __attribute__((__format__(gnu_printf, string_idx, first_to_check))) -#elif \ - JSON_HEDLEY_HAS_ATTRIBUTE(format) || \ - JSON_HEDLEY_GCC_VERSION_CHECK(3,1,0) || \ - JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ - JSON_HEDLEY_ARM_VERSION_CHECK(5,6,0) || \ - JSON_HEDLEY_IBM_VERSION_CHECK(10,1,0) || \ - JSON_HEDLEY_TI_VERSION_CHECK(8,0,0) || \ - (JSON_HEDLEY_TI_VERSION_CHECK(7,3,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) - #define JSON_HEDLEY_PRINTF_FORMAT(string_idx,first_to_check) __attribute__((__format__(__printf__, string_idx, first_to_check))) -#elif JSON_HEDLEY_PELLES_VERSION_CHECK(6,0,0) - #define JSON_HEDLEY_PRINTF_FORMAT(string_idx,first_to_check) __declspec(vaformat(printf,string_idx,first_to_check)) +#if defined(__MINGW32__) && JSON_HEDLEY_GCC_HAS_ATTRIBUTE(format, 4, 4, 0) && !defined(__USE_MINGW_ANSI_STDIO) +#define JSON_HEDLEY_PRINTF_FORMAT(string_idx, first_to_check) \ + __attribute__((__format__(ms_printf, string_idx, first_to_check))) +#elif defined(__MINGW32__) && JSON_HEDLEY_GCC_HAS_ATTRIBUTE(format, 4, 4, 0) && defined(__USE_MINGW_ANSI_STDIO) +#define JSON_HEDLEY_PRINTF_FORMAT(string_idx, first_to_check) \ + __attribute__((__format__(gnu_printf, string_idx, first_to_check))) +#elif JSON_HEDLEY_HAS_ATTRIBUTE(format) || JSON_HEDLEY_GCC_VERSION_CHECK(3, 1, 0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13, 0, 0) || JSON_HEDLEY_ARM_VERSION_CHECK(5, 6, 0) || \ + JSON_HEDLEY_IBM_VERSION_CHECK(10, 1, 0) || JSON_HEDLEY_TI_VERSION_CHECK(8, 0, 0) || \ + (JSON_HEDLEY_TI_VERSION_CHECK(7, 3, 0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) +#define JSON_HEDLEY_PRINTF_FORMAT(string_idx, first_to_check) \ + __attribute__((__format__(__printf__, string_idx, first_to_check))) +#elif JSON_HEDLEY_PELLES_VERSION_CHECK(6, 0, 0) +#define JSON_HEDLEY_PRINTF_FORMAT(string_idx, first_to_check) __declspec(vaformat(printf, string_idx, first_to_check)) #else - #define JSON_HEDLEY_PRINTF_FORMAT(string_idx,first_to_check) +#define JSON_HEDLEY_PRINTF_FORMAT(string_idx, first_to_check) #endif #if defined(JSON_HEDLEY_CONSTEXPR) - #undef JSON_HEDLEY_CONSTEXPR +#undef JSON_HEDLEY_CONSTEXPR #endif #if defined(__cplusplus) - #if __cplusplus >= 201103L - #define JSON_HEDLEY_CONSTEXPR constexpr - #endif +#if __cplusplus >= 201103L +#define JSON_HEDLEY_CONSTEXPR constexpr +#endif #endif #if !defined(JSON_HEDLEY_CONSTEXPR) - #define JSON_HEDLEY_CONSTEXPR +#define JSON_HEDLEY_CONSTEXPR #endif #if defined(JSON_HEDLEY_PREDICT) - #undef JSON_HEDLEY_PREDICT +#undef JSON_HEDLEY_PREDICT #endif #if defined(JSON_HEDLEY_LIKELY) - #undef JSON_HEDLEY_LIKELY +#undef JSON_HEDLEY_LIKELY #endif #if defined(JSON_HEDLEY_UNLIKELY) - #undef JSON_HEDLEY_UNLIKELY +#undef JSON_HEDLEY_UNLIKELY #endif #if defined(JSON_HEDLEY_UNPREDICTABLE) - #undef JSON_HEDLEY_UNPREDICTABLE +#undef JSON_HEDLEY_UNPREDICTABLE #endif #if JSON_HEDLEY_HAS_BUILTIN(__builtin_unpredictable) - #define JSON_HEDLEY_UNPREDICTABLE(expr) __builtin_unpredictable(!!(expr)) +#define JSON_HEDLEY_UNPREDICTABLE(expr) __builtin_unpredictable(!!(expr)) #endif -#if \ - JSON_HEDLEY_HAS_BUILTIN(__builtin_expect_with_probability) || \ - JSON_HEDLEY_GCC_VERSION_CHECK(9,0,0) -# define JSON_HEDLEY_PREDICT(expr, value, probability) __builtin_expect_with_probability(expr, value, probability) -# define JSON_HEDLEY_PREDICT_TRUE(expr, probability) __builtin_expect_with_probability(!!(expr), 1, probability) -# define JSON_HEDLEY_PREDICT_FALSE(expr, probability) __builtin_expect_with_probability(!!(expr), 0, probability) -# define JSON_HEDLEY_LIKELY(expr) __builtin_expect(!!(expr), 1) -# define JSON_HEDLEY_UNLIKELY(expr) __builtin_expect(!!(expr), 0) +#if JSON_HEDLEY_HAS_BUILTIN(__builtin_expect_with_probability) || JSON_HEDLEY_GCC_VERSION_CHECK(9, 0, 0) +#define JSON_HEDLEY_PREDICT(expr, value, probability) __builtin_expect_with_probability(expr, value, probability) +#define JSON_HEDLEY_PREDICT_TRUE(expr, probability) __builtin_expect_with_probability(!!(expr), 1, probability) +#define JSON_HEDLEY_PREDICT_FALSE(expr, probability) __builtin_expect_with_probability(!!(expr), 0, probability) +#define JSON_HEDLEY_LIKELY(expr) __builtin_expect(!!(expr), 1) +#define JSON_HEDLEY_UNLIKELY(expr) __builtin_expect(!!(expr), 0) #if !defined(JSON_HEDLEY_BUILTIN_UNPREDICTABLE) - #define JSON_HEDLEY_BUILTIN_UNPREDICTABLE(expr) __builtin_expect_with_probability(!!(expr), 1, 0.5) +#define JSON_HEDLEY_BUILTIN_UNPREDICTABLE(expr) __builtin_expect_with_probability(!!(expr), 1, 0.5) #endif -#elif \ - JSON_HEDLEY_HAS_BUILTIN(__builtin_expect) || \ - JSON_HEDLEY_GCC_VERSION_CHECK(3,0,0) || \ - JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ - (JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,15,0) && defined(__cplusplus)) || \ - JSON_HEDLEY_ARM_VERSION_CHECK(4,1,0) || \ - JSON_HEDLEY_IBM_VERSION_CHECK(10,1,0) || \ - JSON_HEDLEY_TI_VERSION_CHECK(6,1,0) || \ - JSON_HEDLEY_TINYC_VERSION_CHECK(0,9,27) -# define JSON_HEDLEY_PREDICT(expr, expected, probability) \ - (((probability) >= 0.9) ? __builtin_expect(!!(expr), (expected)) : (((void) (expected)), !!(expr))) -# define JSON_HEDLEY_PREDICT_TRUE(expr, probability) \ - (__extension__ ({ \ - JSON_HEDLEY_CONSTEXPR double hedley_probability_ = (probability); \ - ((hedley_probability_ >= 0.9) ? __builtin_expect(!!(expr), 1) : ((hedley_probability_ <= 0.1) ? __builtin_expect(!!(expr), 0) : !!(expr))); \ +#elif JSON_HEDLEY_HAS_BUILTIN(__builtin_expect) || JSON_HEDLEY_GCC_VERSION_CHECK(3, 0, 0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13, 0, 0) || \ + (JSON_HEDLEY_SUNPRO_VERSION_CHECK(5, 15, 0) && defined(__cplusplus)) || JSON_HEDLEY_ARM_VERSION_CHECK(4, 1, 0) || \ + JSON_HEDLEY_IBM_VERSION_CHECK(10, 1, 0) || JSON_HEDLEY_TI_VERSION_CHECK(6, 1, 0) || \ + JSON_HEDLEY_TINYC_VERSION_CHECK(0, 9, 27) +#define JSON_HEDLEY_PREDICT(expr, expected, probability) \ + (((probability) >= 0.9) ? __builtin_expect(!!(expr), (expected)) : (((void)(expected)), !!(expr))) +#define JSON_HEDLEY_PREDICT_TRUE(expr, probability) \ + (__extension__({ \ + JSON_HEDLEY_CONSTEXPR double hedley_probability_ = (probability); \ + ((hedley_probability_ >= 0.9) ? __builtin_expect(!!(expr), 1) \ + : ((hedley_probability_ <= 0.1) ? __builtin_expect(!!(expr), 0) : !!(expr))); \ })) -# define JSON_HEDLEY_PREDICT_FALSE(expr, probability) \ - (__extension__ ({ \ - JSON_HEDLEY_CONSTEXPR double hedley_probability_ = (probability); \ - ((hedley_probability_ >= 0.9) ? __builtin_expect(!!(expr), 0) : ((hedley_probability_ <= 0.1) ? __builtin_expect(!!(expr), 1) : !!(expr))); \ +#define JSON_HEDLEY_PREDICT_FALSE(expr, probability) \ + (__extension__({ \ + JSON_HEDLEY_CONSTEXPR double hedley_probability_ = (probability); \ + ((hedley_probability_ >= 0.9) ? __builtin_expect(!!(expr), 0) \ + : ((hedley_probability_ <= 0.1) ? __builtin_expect(!!(expr), 1) : !!(expr))); \ })) -# define JSON_HEDLEY_LIKELY(expr) __builtin_expect(!!(expr), 1) -# define JSON_HEDLEY_UNLIKELY(expr) __builtin_expect(!!(expr), 0) +#define JSON_HEDLEY_LIKELY(expr) __builtin_expect(!!(expr), 1) +#define JSON_HEDLEY_UNLIKELY(expr) __builtin_expect(!!(expr), 0) #else -# define JSON_HEDLEY_PREDICT(expr, expected, probability) (((void) (expected)), !!(expr)) -# define JSON_HEDLEY_PREDICT_TRUE(expr, probability) (!!(expr)) -# define JSON_HEDLEY_PREDICT_FALSE(expr, probability) (!!(expr)) -# define JSON_HEDLEY_LIKELY(expr) (!!(expr)) -# define JSON_HEDLEY_UNLIKELY(expr) (!!(expr)) +#define JSON_HEDLEY_PREDICT(expr, expected, probability) (((void)(expected)), !!(expr)) +#define JSON_HEDLEY_PREDICT_TRUE(expr, probability) (!!(expr)) +#define JSON_HEDLEY_PREDICT_FALSE(expr, probability) (!!(expr)) +#define JSON_HEDLEY_LIKELY(expr) (!!(expr)) +#define JSON_HEDLEY_UNLIKELY(expr) (!!(expr)) #endif #if !defined(JSON_HEDLEY_UNPREDICTABLE) - #define JSON_HEDLEY_UNPREDICTABLE(expr) JSON_HEDLEY_PREDICT(expr, 1, 0.5) +#define JSON_HEDLEY_UNPREDICTABLE(expr) JSON_HEDLEY_PREDICT(expr, 1, 0.5) #endif #if defined(JSON_HEDLEY_MALLOC) - #undef JSON_HEDLEY_MALLOC +#undef JSON_HEDLEY_MALLOC #endif -#if \ - JSON_HEDLEY_HAS_ATTRIBUTE(malloc) || \ - JSON_HEDLEY_GCC_VERSION_CHECK(3,1,0) || \ - JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ - JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,11,0) || \ - JSON_HEDLEY_ARM_VERSION_CHECK(4,1,0) || \ - JSON_HEDLEY_IBM_VERSION_CHECK(12,1,0) || \ - JSON_HEDLEY_TI_VERSION_CHECK(8,0,0) || \ - (JSON_HEDLEY_TI_VERSION_CHECK(7,3,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) - #define JSON_HEDLEY_MALLOC __attribute__((__malloc__)) +#if JSON_HEDLEY_HAS_ATTRIBUTE(malloc) || JSON_HEDLEY_GCC_VERSION_CHECK(3, 1, 0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13, 0, 0) || JSON_HEDLEY_SUNPRO_VERSION_CHECK(5, 11, 0) || \ + JSON_HEDLEY_ARM_VERSION_CHECK(4, 1, 0) || JSON_HEDLEY_IBM_VERSION_CHECK(12, 1, 0) || \ + JSON_HEDLEY_TI_VERSION_CHECK(8, 0, 0) || \ + (JSON_HEDLEY_TI_VERSION_CHECK(7, 3, 0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) +#define JSON_HEDLEY_MALLOC __attribute__((__malloc__)) #elif JSON_HEDLEY_MSVC_VERSION_CHECK(14, 0, 0) - #define JSON_HEDLEY_MALLOC __declspec(restrict) +#define JSON_HEDLEY_MALLOC __declspec(restrict) #else - #define JSON_HEDLEY_MALLOC +#define JSON_HEDLEY_MALLOC #endif #if defined(JSON_HEDLEY_PURE) - #undef JSON_HEDLEY_PURE +#undef JSON_HEDLEY_PURE #endif -#if \ - JSON_HEDLEY_HAS_ATTRIBUTE(pure) || \ - JSON_HEDLEY_GCC_VERSION_CHECK(2,96,0) || \ - JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ - JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,11,0) || \ - JSON_HEDLEY_ARM_VERSION_CHECK(4,1,0) || \ - JSON_HEDLEY_IBM_VERSION_CHECK(10,1,0) || \ - JSON_HEDLEY_TI_VERSION_CHECK(8,0,0) || \ - (JSON_HEDLEY_TI_VERSION_CHECK(7,3,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ - JSON_HEDLEY_PGI_VERSION_CHECK(17,10,0) - #define JSON_HEDLEY_PURE __attribute__((__pure__)) -#elif JSON_HEDLEY_TI_VERSION_CHECK(6,0,0) && defined(__cplusplus) - #define JSON_HEDLEY_PURE _Pragma("FUNC_IS_PURE;") +#if JSON_HEDLEY_HAS_ATTRIBUTE(pure) || JSON_HEDLEY_GCC_VERSION_CHECK(2, 96, 0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13, 0, 0) || JSON_HEDLEY_SUNPRO_VERSION_CHECK(5, 11, 0) || \ + JSON_HEDLEY_ARM_VERSION_CHECK(4, 1, 0) || JSON_HEDLEY_IBM_VERSION_CHECK(10, 1, 0) || \ + JSON_HEDLEY_TI_VERSION_CHECK(8, 0, 0) || \ + (JSON_HEDLEY_TI_VERSION_CHECK(7, 3, 0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_PGI_VERSION_CHECK(17, 10, 0) +#define JSON_HEDLEY_PURE __attribute__((__pure__)) +#elif JSON_HEDLEY_TI_VERSION_CHECK(6, 0, 0) && defined(__cplusplus) +#define JSON_HEDLEY_PURE _Pragma("FUNC_IS_PURE;") #else - #define JSON_HEDLEY_PURE +#define JSON_HEDLEY_PURE #endif #if defined(JSON_HEDLEY_CONST) - #undef JSON_HEDLEY_CONST +#undef JSON_HEDLEY_CONST #endif -#if \ - JSON_HEDLEY_HAS_ATTRIBUTE(const) || \ - JSON_HEDLEY_GCC_VERSION_CHECK(2,5,0) || \ - JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ - JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,11,0) || \ - JSON_HEDLEY_ARM_VERSION_CHECK(4,1,0) || \ - JSON_HEDLEY_IBM_VERSION_CHECK(10,1,0) || \ - JSON_HEDLEY_TI_VERSION_CHECK(8,0,0) || \ - (JSON_HEDLEY_TI_VERSION_CHECK(7,3,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ - JSON_HEDLEY_PGI_VERSION_CHECK(17,10,0) - #define JSON_HEDLEY_CONST __attribute__((__const__)) +#if JSON_HEDLEY_HAS_ATTRIBUTE(const) || JSON_HEDLEY_GCC_VERSION_CHECK(2, 5, 0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13, 0, 0) || JSON_HEDLEY_SUNPRO_VERSION_CHECK(5, 11, 0) || \ + JSON_HEDLEY_ARM_VERSION_CHECK(4, 1, 0) || JSON_HEDLEY_IBM_VERSION_CHECK(10, 1, 0) || \ + JSON_HEDLEY_TI_VERSION_CHECK(8, 0, 0) || \ + (JSON_HEDLEY_TI_VERSION_CHECK(7, 3, 0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) || \ + JSON_HEDLEY_PGI_VERSION_CHECK(17, 10, 0) +#define JSON_HEDLEY_CONST __attribute__((__const__)) #else - #define JSON_HEDLEY_CONST JSON_HEDLEY_PURE +#define JSON_HEDLEY_CONST JSON_HEDLEY_PURE #endif #if defined(JSON_HEDLEY_RESTRICT) - #undef JSON_HEDLEY_RESTRICT +#undef JSON_HEDLEY_RESTRICT #endif #if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) && !defined(__cplusplus) - #define JSON_HEDLEY_RESTRICT restrict -#elif \ - JSON_HEDLEY_GCC_VERSION_CHECK(3,1,0) || \ - JSON_HEDLEY_MSVC_VERSION_CHECK(14,0,0) || \ - JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ - JSON_HEDLEY_ARM_VERSION_CHECK(4,1,0) || \ - JSON_HEDLEY_IBM_VERSION_CHECK(10,1,0) || \ - JSON_HEDLEY_PGI_VERSION_CHECK(17,10,0) || \ - JSON_HEDLEY_TI_VERSION_CHECK(8,0,0) || \ - (JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,14,0) && defined(__cplusplus)) || \ - JSON_HEDLEY_IAR_VERSION_CHECK(8,0,0) || \ - defined(__clang__) - #define JSON_HEDLEY_RESTRICT __restrict -#elif JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,3,0) && !defined(__cplusplus) - #define JSON_HEDLEY_RESTRICT _Restrict +#define JSON_HEDLEY_RESTRICT restrict +#elif JSON_HEDLEY_GCC_VERSION_CHECK(3, 1, 0) || JSON_HEDLEY_MSVC_VERSION_CHECK(14, 0, 0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13, 0, 0) || JSON_HEDLEY_ARM_VERSION_CHECK(4, 1, 0) || \ + JSON_HEDLEY_IBM_VERSION_CHECK(10, 1, 0) || JSON_HEDLEY_PGI_VERSION_CHECK(17, 10, 0) || \ + JSON_HEDLEY_TI_VERSION_CHECK(8, 0, 0) || (JSON_HEDLEY_SUNPRO_VERSION_CHECK(5, 14, 0) && defined(__cplusplus)) || \ + JSON_HEDLEY_IAR_VERSION_CHECK(8, 0, 0) || defined(__clang__) +#define JSON_HEDLEY_RESTRICT __restrict +#elif JSON_HEDLEY_SUNPRO_VERSION_CHECK(5, 3, 0) && !defined(__cplusplus) +#define JSON_HEDLEY_RESTRICT _Restrict #else - #define JSON_HEDLEY_RESTRICT +#define JSON_HEDLEY_RESTRICT #endif #if defined(JSON_HEDLEY_INLINE) - #undef JSON_HEDLEY_INLINE +#undef JSON_HEDLEY_INLINE #endif -#if \ - (defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L)) || \ - (defined(__cplusplus) && (__cplusplus >= 199711L)) - #define JSON_HEDLEY_INLINE inline -#elif \ - defined(JSON_HEDLEY_GCC_VERSION) || \ - JSON_HEDLEY_ARM_VERSION_CHECK(6,2,0) - #define JSON_HEDLEY_INLINE __inline__ -#elif \ - JSON_HEDLEY_MSVC_VERSION_CHECK(12,0,0) || \ - JSON_HEDLEY_ARM_VERSION_CHECK(4,1,0) || \ - JSON_HEDLEY_TI_VERSION_CHECK(8,0,0) - #define JSON_HEDLEY_INLINE __inline +#if (defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L)) || (defined(__cplusplus) && (__cplusplus >= 199711L)) +#define JSON_HEDLEY_INLINE inline +#elif defined(JSON_HEDLEY_GCC_VERSION) || JSON_HEDLEY_ARM_VERSION_CHECK(6, 2, 0) +#define JSON_HEDLEY_INLINE __inline__ +#elif JSON_HEDLEY_MSVC_VERSION_CHECK(12, 0, 0) || JSON_HEDLEY_ARM_VERSION_CHECK(4, 1, 0) || \ + JSON_HEDLEY_TI_VERSION_CHECK(8, 0, 0) +#define JSON_HEDLEY_INLINE __inline #else - #define JSON_HEDLEY_INLINE +#define JSON_HEDLEY_INLINE #endif #if defined(JSON_HEDLEY_ALWAYS_INLINE) - #undef JSON_HEDLEY_ALWAYS_INLINE +#undef JSON_HEDLEY_ALWAYS_INLINE #endif -#if \ - JSON_HEDLEY_HAS_ATTRIBUTE(always_inline) || \ - JSON_HEDLEY_GCC_VERSION_CHECK(4,0,0) || \ - JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ - JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,11,0) || \ - JSON_HEDLEY_ARM_VERSION_CHECK(4,1,0) || \ - JSON_HEDLEY_IBM_VERSION_CHECK(10,1,0) || \ - JSON_HEDLEY_TI_VERSION_CHECK(8,0,0) || \ - (JSON_HEDLEY_TI_VERSION_CHECK(7,3,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) - #define JSON_HEDLEY_ALWAYS_INLINE __attribute__((__always_inline__)) JSON_HEDLEY_INLINE -#elif JSON_HEDLEY_MSVC_VERSION_CHECK(12,0,0) - #define JSON_HEDLEY_ALWAYS_INLINE __forceinline -#elif JSON_HEDLEY_TI_VERSION_CHECK(7,0,0) && defined(__cplusplus) - #define JSON_HEDLEY_ALWAYS_INLINE _Pragma("FUNC_ALWAYS_INLINE;") -#elif JSON_HEDLEY_IAR_VERSION_CHECK(8,0,0) - #define JSON_HEDLEY_ALWAYS_INLINE _Pragma("inline=forced") +#if JSON_HEDLEY_HAS_ATTRIBUTE(always_inline) || JSON_HEDLEY_GCC_VERSION_CHECK(4, 0, 0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13, 0, 0) || JSON_HEDLEY_SUNPRO_VERSION_CHECK(5, 11, 0) || \ + JSON_HEDLEY_ARM_VERSION_CHECK(4, 1, 0) || JSON_HEDLEY_IBM_VERSION_CHECK(10, 1, 0) || \ + JSON_HEDLEY_TI_VERSION_CHECK(8, 0, 0) || \ + (JSON_HEDLEY_TI_VERSION_CHECK(7, 3, 0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) +#define JSON_HEDLEY_ALWAYS_INLINE __attribute__((__always_inline__)) JSON_HEDLEY_INLINE +#elif JSON_HEDLEY_MSVC_VERSION_CHECK(12, 0, 0) +#define JSON_HEDLEY_ALWAYS_INLINE __forceinline +#elif JSON_HEDLEY_TI_VERSION_CHECK(7, 0, 0) && defined(__cplusplus) +#define JSON_HEDLEY_ALWAYS_INLINE _Pragma("FUNC_ALWAYS_INLINE;") +#elif JSON_HEDLEY_IAR_VERSION_CHECK(8, 0, 0) +#define JSON_HEDLEY_ALWAYS_INLINE _Pragma("inline=forced") #else - #define JSON_HEDLEY_ALWAYS_INLINE JSON_HEDLEY_INLINE +#define JSON_HEDLEY_ALWAYS_INLINE JSON_HEDLEY_INLINE #endif #if defined(JSON_HEDLEY_NEVER_INLINE) - #undef JSON_HEDLEY_NEVER_INLINE +#undef JSON_HEDLEY_NEVER_INLINE #endif -#if \ - JSON_HEDLEY_HAS_ATTRIBUTE(noinline) || \ - JSON_HEDLEY_GCC_VERSION_CHECK(4,0,0) || \ - JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ - JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,11,0) || \ - JSON_HEDLEY_ARM_VERSION_CHECK(4,1,0) || \ - JSON_HEDLEY_IBM_VERSION_CHECK(10,1,0) || \ - JSON_HEDLEY_TI_VERSION_CHECK(8,0,0) || \ - (JSON_HEDLEY_TI_VERSION_CHECK(7,3,0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) - #define JSON_HEDLEY_NEVER_INLINE __attribute__((__noinline__)) -#elif JSON_HEDLEY_MSVC_VERSION_CHECK(13,10,0) - #define JSON_HEDLEY_NEVER_INLINE __declspec(noinline) -#elif JSON_HEDLEY_PGI_VERSION_CHECK(10,2,0) - #define JSON_HEDLEY_NEVER_INLINE _Pragma("noinline") -#elif JSON_HEDLEY_TI_VERSION_CHECK(6,0,0) && defined(__cplusplus) - #define JSON_HEDLEY_NEVER_INLINE _Pragma("FUNC_CANNOT_INLINE;") -#elif JSON_HEDLEY_IAR_VERSION_CHECK(8,0,0) - #define JSON_HEDLEY_NEVER_INLINE _Pragma("inline=never") -#elif JSON_HEDLEY_COMPCERT_VERSION_CHECK(3,2,0) - #define JSON_HEDLEY_NEVER_INLINE __attribute((noinline)) -#elif JSON_HEDLEY_PELLES_VERSION_CHECK(9,0,0) - #define JSON_HEDLEY_NEVER_INLINE __declspec(noinline) +#if JSON_HEDLEY_HAS_ATTRIBUTE(noinline) || JSON_HEDLEY_GCC_VERSION_CHECK(4, 0, 0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13, 0, 0) || JSON_HEDLEY_SUNPRO_VERSION_CHECK(5, 11, 0) || \ + JSON_HEDLEY_ARM_VERSION_CHECK(4, 1, 0) || JSON_HEDLEY_IBM_VERSION_CHECK(10, 1, 0) || \ + JSON_HEDLEY_TI_VERSION_CHECK(8, 0, 0) || \ + (JSON_HEDLEY_TI_VERSION_CHECK(7, 3, 0) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) +#define JSON_HEDLEY_NEVER_INLINE __attribute__((__noinline__)) +#elif JSON_HEDLEY_MSVC_VERSION_CHECK(13, 10, 0) +#define JSON_HEDLEY_NEVER_INLINE __declspec(noinline) +#elif JSON_HEDLEY_PGI_VERSION_CHECK(10, 2, 0) +#define JSON_HEDLEY_NEVER_INLINE _Pragma("noinline") +#elif JSON_HEDLEY_TI_VERSION_CHECK(6, 0, 0) && defined(__cplusplus) +#define JSON_HEDLEY_NEVER_INLINE _Pragma("FUNC_CANNOT_INLINE;") +#elif JSON_HEDLEY_IAR_VERSION_CHECK(8, 0, 0) +#define JSON_HEDLEY_NEVER_INLINE _Pragma("inline=never") +#elif JSON_HEDLEY_COMPCERT_VERSION_CHECK(3, 2, 0) +#define JSON_HEDLEY_NEVER_INLINE __attribute((noinline)) +#elif JSON_HEDLEY_PELLES_VERSION_CHECK(9, 0, 0) +#define JSON_HEDLEY_NEVER_INLINE __declspec(noinline) #else - #define JSON_HEDLEY_NEVER_INLINE +#define JSON_HEDLEY_NEVER_INLINE #endif #if defined(JSON_HEDLEY_PRIVATE) - #undef JSON_HEDLEY_PRIVATE +#undef JSON_HEDLEY_PRIVATE #endif #if defined(JSON_HEDLEY_PUBLIC) - #undef JSON_HEDLEY_PUBLIC +#undef JSON_HEDLEY_PUBLIC #endif #if defined(JSON_HEDLEY_IMPORT) - #undef JSON_HEDLEY_IMPORT +#undef JSON_HEDLEY_IMPORT #endif #if defined(_WIN32) || defined(__CYGWIN__) - #define JSON_HEDLEY_PRIVATE - #define JSON_HEDLEY_PUBLIC __declspec(dllexport) - #define JSON_HEDLEY_IMPORT __declspec(dllimport) +#define JSON_HEDLEY_PRIVATE +#define JSON_HEDLEY_PUBLIC __declspec(dllexport) +#define JSON_HEDLEY_IMPORT __declspec(dllimport) #else - #if \ - JSON_HEDLEY_HAS_ATTRIBUTE(visibility) || \ - JSON_HEDLEY_GCC_VERSION_CHECK(3,3,0) || \ - JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,11,0) || \ - JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ - JSON_HEDLEY_ARM_VERSION_CHECK(4,1,0) || \ - JSON_HEDLEY_IBM_VERSION_CHECK(13,1,0) || \ - JSON_HEDLEY_TI_VERSION_CHECK(8,0,0) || \ - (JSON_HEDLEY_TI_VERSION_CHECK(7,3,0) && defined(__TI_EABI__) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) - #define JSON_HEDLEY_PRIVATE __attribute__((__visibility__("hidden"))) - #define JSON_HEDLEY_PUBLIC __attribute__((__visibility__("default"))) - #else - #define JSON_HEDLEY_PRIVATE - #define JSON_HEDLEY_PUBLIC - #endif - #define JSON_HEDLEY_IMPORT extern +#if JSON_HEDLEY_HAS_ATTRIBUTE(visibility) || JSON_HEDLEY_GCC_VERSION_CHECK(3, 3, 0) || \ + JSON_HEDLEY_SUNPRO_VERSION_CHECK(5, 11, 0) || JSON_HEDLEY_INTEL_VERSION_CHECK(13, 0, 0) || \ + JSON_HEDLEY_ARM_VERSION_CHECK(4, 1, 0) || JSON_HEDLEY_IBM_VERSION_CHECK(13, 1, 0) || \ + JSON_HEDLEY_TI_VERSION_CHECK(8, 0, 0) || \ + (JSON_HEDLEY_TI_VERSION_CHECK(7, 3, 0) && defined(__TI_EABI__) && defined(__TI_GNU_ATTRIBUTE_SUPPORT__)) +#define JSON_HEDLEY_PRIVATE __attribute__((__visibility__("hidden"))) +#define JSON_HEDLEY_PUBLIC __attribute__((__visibility__("default"))) +#else +#define JSON_HEDLEY_PRIVATE +#define JSON_HEDLEY_PUBLIC +#endif +#define JSON_HEDLEY_IMPORT extern #endif #if defined(JSON_HEDLEY_NO_THROW) - #undef JSON_HEDLEY_NO_THROW +#undef JSON_HEDLEY_NO_THROW #endif -#if \ - JSON_HEDLEY_HAS_ATTRIBUTE(nothrow) || \ - JSON_HEDLEY_GCC_VERSION_CHECK(3,3,0) || \ - JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) - #define JSON_HEDLEY_NO_THROW __attribute__((__nothrow__)) -#elif \ - JSON_HEDLEY_MSVC_VERSION_CHECK(13,1,0) || \ - JSON_HEDLEY_ARM_VERSION_CHECK(4,1,0) - #define JSON_HEDLEY_NO_THROW __declspec(nothrow) +#if JSON_HEDLEY_HAS_ATTRIBUTE(nothrow) || JSON_HEDLEY_GCC_VERSION_CHECK(3, 3, 0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13, 0, 0) +#define JSON_HEDLEY_NO_THROW __attribute__((__nothrow__)) +#elif JSON_HEDLEY_MSVC_VERSION_CHECK(13, 1, 0) || JSON_HEDLEY_ARM_VERSION_CHECK(4, 1, 0) +#define JSON_HEDLEY_NO_THROW __declspec(nothrow) #else - #define JSON_HEDLEY_NO_THROW +#define JSON_HEDLEY_NO_THROW #endif #if defined(JSON_HEDLEY_FALL_THROUGH) - #undef JSON_HEDLEY_FALL_THROUGH +#undef JSON_HEDLEY_FALL_THROUGH #endif -#if \ - defined(__cplusplus) && \ - (!defined(JSON_HEDLEY_SUNPRO_VERSION) || JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,15,0)) && \ +#if defined(__cplusplus) && (!defined(JSON_HEDLEY_SUNPRO_VERSION) || JSON_HEDLEY_SUNPRO_VERSION_CHECK(5, 15, 0)) && \ !defined(JSON_HEDLEY_PGI_VERSION) - #if \ - (__cplusplus >= 201703L) || \ - ((__cplusplus >= 201103L) && JSON_HEDLEY_HAS_CPP_ATTRIBUTE(fallthrough)) - #define JSON_HEDLEY_FALL_THROUGH [[fallthrough]] - #elif (__cplusplus >= 201103L) && JSON_HEDLEY_HAS_CPP_ATTRIBUTE(clang::fallthrough) - #define JSON_HEDLEY_FALL_THROUGH [[clang::fallthrough]] - #elif (__cplusplus >= 201103L) && JSON_HEDLEY_GCC_VERSION_CHECK(7,0,0) - #define JSON_HEDLEY_FALL_THROUGH [[gnu::fallthrough]] - #endif +#if (__cplusplus >= 201703L) || ((__cplusplus >= 201103L) && JSON_HEDLEY_HAS_CPP_ATTRIBUTE(fallthrough)) +#define JSON_HEDLEY_FALL_THROUGH [[fallthrough]] +#elif (__cplusplus >= 201103L) && JSON_HEDLEY_HAS_CPP_ATTRIBUTE(clang::fallthrough) +#define JSON_HEDLEY_FALL_THROUGH [[clang::fallthrough]] +#elif (__cplusplus >= 201103L) && JSON_HEDLEY_GCC_VERSION_CHECK(7, 0, 0) +#define JSON_HEDLEY_FALL_THROUGH [[gnu::fallthrough]] +#endif #endif #if !defined(JSON_HEDLEY_FALL_THROUGH) - #if JSON_HEDLEY_GNUC_HAS_ATTRIBUTE(fallthrough,7,0,0) && !defined(JSON_HEDLEY_PGI_VERSION) - #define JSON_HEDLEY_FALL_THROUGH __attribute__((__fallthrough__)) - #elif defined(__fallthrough) /* SAL */ - #define JSON_HEDLEY_FALL_THROUGH __fallthrough - #else - #define JSON_HEDLEY_FALL_THROUGH - #endif +#if JSON_HEDLEY_GNUC_HAS_ATTRIBUTE(fallthrough, 7, 0, 0) && !defined(JSON_HEDLEY_PGI_VERSION) +#define JSON_HEDLEY_FALL_THROUGH __attribute__((__fallthrough__)) +#elif defined(__fallthrough) /* SAL */ +#define JSON_HEDLEY_FALL_THROUGH __fallthrough +#else +#define JSON_HEDLEY_FALL_THROUGH +#endif #endif #if defined(JSON_HEDLEY_RETURNS_NON_NULL) - #undef JSON_HEDLEY_RETURNS_NON_NULL +#undef JSON_HEDLEY_RETURNS_NON_NULL #endif -#if \ - JSON_HEDLEY_HAS_ATTRIBUTE(returns_nonnull) || \ - JSON_HEDLEY_GCC_VERSION_CHECK(4,9,0) - #define JSON_HEDLEY_RETURNS_NON_NULL __attribute__((__returns_nonnull__)) +#if JSON_HEDLEY_HAS_ATTRIBUTE(returns_nonnull) || JSON_HEDLEY_GCC_VERSION_CHECK(4, 9, 0) +#define JSON_HEDLEY_RETURNS_NON_NULL __attribute__((__returns_nonnull__)) #elif defined(_Ret_notnull_) /* SAL */ - #define JSON_HEDLEY_RETURNS_NON_NULL _Ret_notnull_ +#define JSON_HEDLEY_RETURNS_NON_NULL _Ret_notnull_ #else - #define JSON_HEDLEY_RETURNS_NON_NULL +#define JSON_HEDLEY_RETURNS_NON_NULL #endif #if defined(JSON_HEDLEY_ARRAY_PARAM) - #undef JSON_HEDLEY_ARRAY_PARAM +#undef JSON_HEDLEY_ARRAY_PARAM #endif -#if \ - defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) && \ - !defined(__STDC_NO_VLA__) && \ - !defined(__cplusplus) && \ - !defined(JSON_HEDLEY_PGI_VERSION) && \ - !defined(JSON_HEDLEY_TINYC_VERSION) - #define JSON_HEDLEY_ARRAY_PARAM(name) (name) +#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) && !defined(__STDC_NO_VLA__) && \ + !defined(__cplusplus) && !defined(JSON_HEDLEY_PGI_VERSION) && !defined(JSON_HEDLEY_TINYC_VERSION) +#define JSON_HEDLEY_ARRAY_PARAM(name) (name) #else - #define JSON_HEDLEY_ARRAY_PARAM(name) +#define JSON_HEDLEY_ARRAY_PARAM(name) #endif #if defined(JSON_HEDLEY_IS_CONSTANT) - #undef JSON_HEDLEY_IS_CONSTANT +#undef JSON_HEDLEY_IS_CONSTANT #endif #if defined(JSON_HEDLEY_REQUIRE_CONSTEXPR) - #undef JSON_HEDLEY_REQUIRE_CONSTEXPR +#undef JSON_HEDLEY_REQUIRE_CONSTEXPR #endif /* Note the double-underscore. For internal use only; no API * guarantees! */ #if defined(JSON_HEDLEY__IS_CONSTEXPR) - #undef JSON_HEDLEY__IS_CONSTEXPR +#undef JSON_HEDLEY__IS_CONSTEXPR #endif -#if \ - JSON_HEDLEY_HAS_BUILTIN(__builtin_constant_p) || \ - JSON_HEDLEY_GCC_VERSION_CHECK(3,4,0) || \ - JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ - JSON_HEDLEY_TINYC_VERSION_CHECK(0,9,19) || \ - JSON_HEDLEY_ARM_VERSION_CHECK(4,1,0) || \ - JSON_HEDLEY_IBM_VERSION_CHECK(13,1,0) || \ - JSON_HEDLEY_TI_VERSION_CHECK(6,1,0) || \ - JSON_HEDLEY_SUNPRO_VERSION_CHECK(5,10,0) || \ - JSON_HEDLEY_CRAY_VERSION_CHECK(8,1,0) - #define JSON_HEDLEY_IS_CONSTANT(expr) __builtin_constant_p(expr) +#if JSON_HEDLEY_HAS_BUILTIN(__builtin_constant_p) || JSON_HEDLEY_GCC_VERSION_CHECK(3, 4, 0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13, 0, 0) || JSON_HEDLEY_TINYC_VERSION_CHECK(0, 9, 19) || \ + JSON_HEDLEY_ARM_VERSION_CHECK(4, 1, 0) || JSON_HEDLEY_IBM_VERSION_CHECK(13, 1, 0) || \ + JSON_HEDLEY_TI_VERSION_CHECK(6, 1, 0) || JSON_HEDLEY_SUNPRO_VERSION_CHECK(5, 10, 0) || \ + JSON_HEDLEY_CRAY_VERSION_CHECK(8, 1, 0) +#define JSON_HEDLEY_IS_CONSTANT(expr) __builtin_constant_p(expr) #endif #if !defined(__cplusplus) -# if \ - JSON_HEDLEY_HAS_BUILTIN(__builtin_types_compatible_p) || \ - JSON_HEDLEY_GCC_VERSION_CHECK(3,4,0) || \ - JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ - JSON_HEDLEY_IBM_VERSION_CHECK(13,1,0) || \ - JSON_HEDLEY_CRAY_VERSION_CHECK(8,1,0) || \ - JSON_HEDLEY_ARM_VERSION_CHECK(5,4,0) || \ - JSON_HEDLEY_TINYC_VERSION_CHECK(0,9,24) +#if JSON_HEDLEY_HAS_BUILTIN(__builtin_types_compatible_p) || JSON_HEDLEY_GCC_VERSION_CHECK(3, 4, 0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13, 0, 0) || JSON_HEDLEY_IBM_VERSION_CHECK(13, 1, 0) || \ + JSON_HEDLEY_CRAY_VERSION_CHECK(8, 1, 0) || JSON_HEDLEY_ARM_VERSION_CHECK(5, 4, 0) || \ + JSON_HEDLEY_TINYC_VERSION_CHECK(0, 9, 24) #if defined(__INTPTR_TYPE__) - #define JSON_HEDLEY__IS_CONSTEXPR(expr) __builtin_types_compatible_p(__typeof__((1 ? (void*) ((__INTPTR_TYPE__) ((expr) * 0)) : (int*) 0)), int*) +#define JSON_HEDLEY__IS_CONSTEXPR(expr) \ + __builtin_types_compatible_p(__typeof__((1 ? (void*)((__INTPTR_TYPE__)((expr)*0)) : (int*)0)), int*) #else - #include - #define JSON_HEDLEY__IS_CONSTEXPR(expr) __builtin_types_compatible_p(__typeof__((1 ? (void*) ((intptr_t) ((expr) * 0)) : (int*) 0)), int*) +#include +#define JSON_HEDLEY__IS_CONSTEXPR(expr) \ + __builtin_types_compatible_p(__typeof__((1 ? (void*)((intptr_t)((expr)*0)) : (int*)0)), int*) #endif -# elif \ - (defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) && !defined(JSON_HEDLEY_SUNPRO_VERSION) && !defined(JSON_HEDLEY_PGI_VERSION)) || \ - JSON_HEDLEY_HAS_EXTENSION(c_generic_selections) || \ - JSON_HEDLEY_GCC_VERSION_CHECK(4,9,0) || \ - JSON_HEDLEY_INTEL_VERSION_CHECK(17,0,0) || \ - JSON_HEDLEY_IBM_VERSION_CHECK(12,1,0) || \ - JSON_HEDLEY_ARM_VERSION_CHECK(5,3,0) +#elif (defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) && !defined(JSON_HEDLEY_SUNPRO_VERSION) && \ + !defined(JSON_HEDLEY_PGI_VERSION)) || \ + JSON_HEDLEY_HAS_EXTENSION(c_generic_selections) || JSON_HEDLEY_GCC_VERSION_CHECK(4, 9, 0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(17, 0, 0) || JSON_HEDLEY_IBM_VERSION_CHECK(12, 1, 0) || \ + JSON_HEDLEY_ARM_VERSION_CHECK(5, 3, 0) #if defined(__INTPTR_TYPE__) - #define JSON_HEDLEY__IS_CONSTEXPR(expr) _Generic((1 ? (void*) ((__INTPTR_TYPE__) ((expr) * 0)) : (int*) 0), int*: 1, void*: 0) +#define JSON_HEDLEY__IS_CONSTEXPR(expr) \ + _Generic((1 ? (void*)((__INTPTR_TYPE__)((expr)*0)) : (int*)0), int* : 1, void* : 0) #else - #include - #define JSON_HEDLEY__IS_CONSTEXPR(expr) _Generic((1 ? (void*) ((intptr_t) * 0) : (int*) 0), int*: 1, void*: 0) +#include +#define JSON_HEDLEY__IS_CONSTEXPR(expr) _Generic((1 ? (void*)((intptr_t)*0) : (int*)0), int* : 1, void* : 0) +#endif +#elif defined(JSON_HEDLEY_GCC_VERSION) || defined(JSON_HEDLEY_INTEL_VERSION) || defined(JSON_HEDLEY_TINYC_VERSION) || \ + defined(JSON_HEDLEY_TI_VERSION) || defined(__clang__) +#define JSON_HEDLEY__IS_CONSTEXPR(expr) \ + (sizeof(void) != sizeof(*(1 ? ((void*)((expr)*0L)) : ((struct { char v[sizeof(void) * 2]; }*)1)))) #endif -# elif \ - defined(JSON_HEDLEY_GCC_VERSION) || \ - defined(JSON_HEDLEY_INTEL_VERSION) || \ - defined(JSON_HEDLEY_TINYC_VERSION) || \ - defined(JSON_HEDLEY_TI_VERSION) || \ - defined(__clang__) -# define JSON_HEDLEY__IS_CONSTEXPR(expr) ( \ - sizeof(void) != \ - sizeof(*( \ - 1 ? \ - ((void*) ((expr) * 0L) ) : \ -((struct { char v[sizeof(void) * 2]; } *) 1) \ - ) \ - ) \ - ) -# endif #endif #if defined(JSON_HEDLEY__IS_CONSTEXPR) - #if !defined(JSON_HEDLEY_IS_CONSTANT) - #define JSON_HEDLEY_IS_CONSTANT(expr) JSON_HEDLEY__IS_CONSTEXPR(expr) - #endif - #define JSON_HEDLEY_REQUIRE_CONSTEXPR(expr) (JSON_HEDLEY__IS_CONSTEXPR(expr) ? (expr) : (-1)) +#if !defined(JSON_HEDLEY_IS_CONSTANT) +#define JSON_HEDLEY_IS_CONSTANT(expr) JSON_HEDLEY__IS_CONSTEXPR(expr) +#endif +#define JSON_HEDLEY_REQUIRE_CONSTEXPR(expr) (JSON_HEDLEY__IS_CONSTEXPR(expr) ? (expr) : (-1)) #else - #if !defined(JSON_HEDLEY_IS_CONSTANT) - #define JSON_HEDLEY_IS_CONSTANT(expr) (0) - #endif - #define JSON_HEDLEY_REQUIRE_CONSTEXPR(expr) (expr) +#if !defined(JSON_HEDLEY_IS_CONSTANT) +#define JSON_HEDLEY_IS_CONSTANT(expr) (0) +#endif +#define JSON_HEDLEY_REQUIRE_CONSTEXPR(expr) (expr) #endif #if defined(JSON_HEDLEY_BEGIN_C_DECLS) - #undef JSON_HEDLEY_BEGIN_C_DECLS +#undef JSON_HEDLEY_BEGIN_C_DECLS #endif #if defined(JSON_HEDLEY_END_C_DECLS) - #undef JSON_HEDLEY_END_C_DECLS +#undef JSON_HEDLEY_END_C_DECLS #endif #if defined(JSON_HEDLEY_C_DECL) - #undef JSON_HEDLEY_C_DECL +#undef JSON_HEDLEY_C_DECL #endif #if defined(__cplusplus) - #define JSON_HEDLEY_BEGIN_C_DECLS extern "C" { - #define JSON_HEDLEY_END_C_DECLS } - #define JSON_HEDLEY_C_DECL extern "C" +#define JSON_HEDLEY_BEGIN_C_DECLS extern "C" { +#define JSON_HEDLEY_END_C_DECLS } +#define JSON_HEDLEY_C_DECL extern "C" #else - #define JSON_HEDLEY_BEGIN_C_DECLS - #define JSON_HEDLEY_END_C_DECLS - #define JSON_HEDLEY_C_DECL +#define JSON_HEDLEY_BEGIN_C_DECLS +#define JSON_HEDLEY_END_C_DECLS +#define JSON_HEDLEY_C_DECL #endif #if defined(JSON_HEDLEY_STATIC_ASSERT) - #undef JSON_HEDLEY_STATIC_ASSERT +#undef JSON_HEDLEY_STATIC_ASSERT #endif -#if \ - !defined(__cplusplus) && ( \ - (defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L)) || \ - JSON_HEDLEY_HAS_FEATURE(c_static_assert) || \ - JSON_HEDLEY_GCC_VERSION_CHECK(6,0,0) || \ - JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) || \ - defined(_Static_assert) \ - ) -# define JSON_HEDLEY_STATIC_ASSERT(expr, message) _Static_assert(expr, message) -#elif \ - (defined(__cplusplus) && (__cplusplus >= 201703L)) || \ - JSON_HEDLEY_MSVC_VERSION_CHECK(16,0,0) || \ - (defined(__cplusplus) && JSON_HEDLEY_TI_VERSION_CHECK(8,3,0)) -# define JSON_HEDLEY_STATIC_ASSERT(expr, message) static_assert(expr, message) +#if !defined(__cplusplus) && \ + ((defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L)) || JSON_HEDLEY_HAS_FEATURE(c_static_assert) || \ + JSON_HEDLEY_GCC_VERSION_CHECK(6, 0, 0) || JSON_HEDLEY_INTEL_VERSION_CHECK(13, 0, 0) || defined(_Static_assert)) +#define JSON_HEDLEY_STATIC_ASSERT(expr, message) _Static_assert(expr, message) +#elif (defined(__cplusplus) && (__cplusplus >= 201703L)) || JSON_HEDLEY_MSVC_VERSION_CHECK(16, 0, 0) || \ + (defined(__cplusplus) && JSON_HEDLEY_TI_VERSION_CHECK(8, 3, 0)) +#define JSON_HEDLEY_STATIC_ASSERT(expr, message) static_assert(expr, message) #elif defined(__cplusplus) && (__cplusplus >= 201103L) -# define JSON_HEDLEY_STATIC_ASSERT(expr, message) static_assert(expr) +#define JSON_HEDLEY_STATIC_ASSERT(expr, message) static_assert(expr) #else -# define JSON_HEDLEY_STATIC_ASSERT(expr, message) +#define JSON_HEDLEY_STATIC_ASSERT(expr, message) #endif #if defined(JSON_HEDLEY_CONST_CAST) - #undef JSON_HEDLEY_CONST_CAST +#undef JSON_HEDLEY_CONST_CAST #endif #if defined(__cplusplus) -# define JSON_HEDLEY_CONST_CAST(T, expr) (const_cast(expr)) -#elif \ - JSON_HEDLEY_HAS_WARNING("-Wcast-qual") || \ - JSON_HEDLEY_GCC_VERSION_CHECK(4,6,0) || \ - JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) -# define JSON_HEDLEY_CONST_CAST(T, expr) (__extension__ ({ \ - JSON_HEDLEY_DIAGNOSTIC_PUSH \ - JSON_HEDLEY_DIAGNOSTIC_DISABLE_CAST_QUAL \ - ((T) (expr)); \ - JSON_HEDLEY_DIAGNOSTIC_POP \ +#define JSON_HEDLEY_CONST_CAST(T, expr) (const_cast(expr)) +#elif JSON_HEDLEY_HAS_WARNING("-Wcast-qual") || JSON_HEDLEY_GCC_VERSION_CHECK(4, 6, 0) || \ + JSON_HEDLEY_INTEL_VERSION_CHECK(13, 0, 0) +#define JSON_HEDLEY_CONST_CAST(T, expr) \ + (__extension__({ \ + JSON_HEDLEY_DIAGNOSTIC_PUSH \ + JSON_HEDLEY_DIAGNOSTIC_DISABLE_CAST_QUAL((T)(expr)); \ + JSON_HEDLEY_DIAGNOSTIC_POP \ })) #else -# define JSON_HEDLEY_CONST_CAST(T, expr) ((T) (expr)) +#define JSON_HEDLEY_CONST_CAST(T, expr) ((T)(expr)) #endif #if defined(JSON_HEDLEY_REINTERPRET_CAST) - #undef JSON_HEDLEY_REINTERPRET_CAST +#undef JSON_HEDLEY_REINTERPRET_CAST #endif #if defined(__cplusplus) - #define JSON_HEDLEY_REINTERPRET_CAST(T, expr) (reinterpret_cast(expr)) +#define JSON_HEDLEY_REINTERPRET_CAST(T, expr) (reinterpret_cast(expr)) #else - #define JSON_HEDLEY_REINTERPRET_CAST(T, expr) (*((T*) &(expr))) +#define JSON_HEDLEY_REINTERPRET_CAST(T, expr) (*((T*)&(expr))) #endif #if defined(JSON_HEDLEY_STATIC_CAST) - #undef JSON_HEDLEY_STATIC_CAST +#undef JSON_HEDLEY_STATIC_CAST #endif #if defined(__cplusplus) - #define JSON_HEDLEY_STATIC_CAST(T, expr) (static_cast(expr)) +#define JSON_HEDLEY_STATIC_CAST(T, expr) (static_cast(expr)) #else - #define JSON_HEDLEY_STATIC_CAST(T, expr) ((T) (expr)) +#define JSON_HEDLEY_STATIC_CAST(T, expr) ((T)(expr)) #endif #if defined(JSON_HEDLEY_CPP_CAST) - #undef JSON_HEDLEY_CPP_CAST +#undef JSON_HEDLEY_CPP_CAST #endif #if defined(__cplusplus) - #define JSON_HEDLEY_CPP_CAST(T, expr) static_cast(expr) +#define JSON_HEDLEY_CPP_CAST(T, expr) static_cast(expr) #else - #define JSON_HEDLEY_CPP_CAST(T, expr) (expr) +#define JSON_HEDLEY_CPP_CAST(T, expr) (expr) #endif #if defined(JSON_HEDLEY_MESSAGE) - #undef JSON_HEDLEY_MESSAGE +#undef JSON_HEDLEY_MESSAGE #endif #if JSON_HEDLEY_HAS_WARNING("-Wunknown-pragmas") -# define JSON_HEDLEY_MESSAGE(msg) \ - JSON_HEDLEY_DIAGNOSTIC_PUSH \ +#define JSON_HEDLEY_MESSAGE(msg) \ + JSON_HEDLEY_DIAGNOSTIC_PUSH \ JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS \ - JSON_HEDLEY_PRAGMA(message msg) \ + JSON_HEDLEY_PRAGMA(message msg) \ JSON_HEDLEY_DIAGNOSTIC_POP -#elif \ - JSON_HEDLEY_GCC_VERSION_CHECK(4,4,0) || \ - JSON_HEDLEY_INTEL_VERSION_CHECK(13,0,0) -# define JSON_HEDLEY_MESSAGE(msg) JSON_HEDLEY_PRAGMA(message msg) -#elif JSON_HEDLEY_CRAY_VERSION_CHECK(5,0,0) -# define JSON_HEDLEY_MESSAGE(msg) JSON_HEDLEY_PRAGMA(_CRI message msg) -#elif JSON_HEDLEY_IAR_VERSION_CHECK(8,0,0) -# define JSON_HEDLEY_MESSAGE(msg) JSON_HEDLEY_PRAGMA(message(msg)) -#elif JSON_HEDLEY_PELLES_VERSION_CHECK(2,0,0) -# define JSON_HEDLEY_MESSAGE(msg) JSON_HEDLEY_PRAGMA(message(msg)) +#elif JSON_HEDLEY_GCC_VERSION_CHECK(4, 4, 0) || JSON_HEDLEY_INTEL_VERSION_CHECK(13, 0, 0) +#define JSON_HEDLEY_MESSAGE(msg) JSON_HEDLEY_PRAGMA(message msg) +#elif JSON_HEDLEY_CRAY_VERSION_CHECK(5, 0, 0) +#define JSON_HEDLEY_MESSAGE(msg) JSON_HEDLEY_PRAGMA(_CRI message msg) +#elif JSON_HEDLEY_IAR_VERSION_CHECK(8, 0, 0) +#define JSON_HEDLEY_MESSAGE(msg) JSON_HEDLEY_PRAGMA(message(msg)) +#elif JSON_HEDLEY_PELLES_VERSION_CHECK(2, 0, 0) +#define JSON_HEDLEY_MESSAGE(msg) JSON_HEDLEY_PRAGMA(message(msg)) #else -# define JSON_HEDLEY_MESSAGE(msg) +#define JSON_HEDLEY_MESSAGE(msg) #endif #if defined(JSON_HEDLEY_WARNING) - #undef JSON_HEDLEY_WARNING +#undef JSON_HEDLEY_WARNING #endif #if JSON_HEDLEY_HAS_WARNING("-Wunknown-pragmas") -# define JSON_HEDLEY_WARNING(msg) \ - JSON_HEDLEY_DIAGNOSTIC_PUSH \ +#define JSON_HEDLEY_WARNING(msg) \ + JSON_HEDLEY_DIAGNOSTIC_PUSH \ JSON_HEDLEY_DIAGNOSTIC_DISABLE_UNKNOWN_PRAGMAS \ - JSON_HEDLEY_PRAGMA(clang warning msg) \ + JSON_HEDLEY_PRAGMA(clang warning msg) \ JSON_HEDLEY_DIAGNOSTIC_POP -#elif \ - JSON_HEDLEY_GCC_VERSION_CHECK(4,8,0) || \ - JSON_HEDLEY_PGI_VERSION_CHECK(18,4,0) -# define JSON_HEDLEY_WARNING(msg) JSON_HEDLEY_PRAGMA(GCC warning msg) -#elif JSON_HEDLEY_MSVC_VERSION_CHECK(15,0,0) -# define JSON_HEDLEY_WARNING(msg) JSON_HEDLEY_PRAGMA(message(msg)) +#elif JSON_HEDLEY_GCC_VERSION_CHECK(4, 8, 0) || JSON_HEDLEY_PGI_VERSION_CHECK(18, 4, 0) +#define JSON_HEDLEY_WARNING(msg) JSON_HEDLEY_PRAGMA(GCC warning msg) +#elif JSON_HEDLEY_MSVC_VERSION_CHECK(15, 0, 0) +#define JSON_HEDLEY_WARNING(msg) JSON_HEDLEY_PRAGMA(message(msg)) #else -# define JSON_HEDLEY_WARNING(msg) JSON_HEDLEY_MESSAGE(msg) +#define JSON_HEDLEY_WARNING(msg) JSON_HEDLEY_MESSAGE(msg) #endif #if defined(JSON_HEDLEY_REQUIRE_MSG) - #undef JSON_HEDLEY_REQUIRE_MSG +#undef JSON_HEDLEY_REQUIRE_MSG #endif #if JSON_HEDLEY_HAS_ATTRIBUTE(diagnose_if) -# if JSON_HEDLEY_HAS_WARNING("-Wgcc-compat") -# define JSON_HEDLEY_REQUIRE_MSG(expr, msg) \ - JSON_HEDLEY_DIAGNOSTIC_PUSH \ - _Pragma("clang diagnostic ignored \"-Wgcc-compat\"") \ - __attribute__((__diagnose_if__(!(expr), msg, "error"))) \ - JSON_HEDLEY_DIAGNOSTIC_POP -# else -# define JSON_HEDLEY_REQUIRE_MSG(expr, msg) __attribute__((__diagnose_if__(!(expr), msg, "error"))) -# endif +#if JSON_HEDLEY_HAS_WARNING("-Wgcc-compat") +#define JSON_HEDLEY_REQUIRE_MSG(expr, msg) \ + JSON_HEDLEY_DIAGNOSTIC_PUSH \ + _Pragma("clang diagnostic ignored \"-Wgcc-compat\"") __attribute__((__diagnose_if__(!(expr), msg, "error"))) \ + JSON_HEDLEY_DIAGNOSTIC_POP #else -# define JSON_HEDLEY_REQUIRE_MSG(expr, msg) +#define JSON_HEDLEY_REQUIRE_MSG(expr, msg) __attribute__((__diagnose_if__(!(expr), msg, "error"))) +#endif +#else +#define JSON_HEDLEY_REQUIRE_MSG(expr, msg) #endif #if defined(JSON_HEDLEY_REQUIRE) - #undef JSON_HEDLEY_REQUIRE +#undef JSON_HEDLEY_REQUIRE #endif #define JSON_HEDLEY_REQUIRE(expr) JSON_HEDLEY_REQUIRE_MSG(expr, #expr) #if defined(JSON_HEDLEY_FLAGS) - #undef JSON_HEDLEY_FLAGS +#undef JSON_HEDLEY_FLAGS #endif #if JSON_HEDLEY_HAS_ATTRIBUTE(flag_enum) - #define JSON_HEDLEY_FLAGS __attribute__((__flag_enum__)) +#define JSON_HEDLEY_FLAGS __attribute__((__flag_enum__)) #endif #if defined(JSON_HEDLEY_FLAGS_CAST) - #undef JSON_HEDLEY_FLAGS_CAST +#undef JSON_HEDLEY_FLAGS_CAST #endif -#if JSON_HEDLEY_INTEL_VERSION_CHECK(19,0,0) -# define JSON_HEDLEY_FLAGS_CAST(T, expr) (__extension__ ({ \ - JSON_HEDLEY_DIAGNOSTIC_PUSH \ - _Pragma("warning(disable:188)") \ - ((T) (expr)); \ - JSON_HEDLEY_DIAGNOSTIC_POP \ +#if JSON_HEDLEY_INTEL_VERSION_CHECK(19, 0, 0) +#define JSON_HEDLEY_FLAGS_CAST(T, expr) \ + (__extension__({ \ + JSON_HEDLEY_DIAGNOSTIC_PUSH \ + _Pragma("warning(disable:188)")((T)(expr)); \ + JSON_HEDLEY_DIAGNOSTIC_POP \ })) #else -# define JSON_HEDLEY_FLAGS_CAST(T, expr) JSON_HEDLEY_STATIC_CAST(T, expr) +#define JSON_HEDLEY_FLAGS_CAST(T, expr) JSON_HEDLEY_STATIC_CAST(T, expr) #endif /* Remaining macros are deprecated. */ #if defined(JSON_HEDLEY_GCC_NOT_CLANG_VERSION_CHECK) - #undef JSON_HEDLEY_GCC_NOT_CLANG_VERSION_CHECK +#undef JSON_HEDLEY_GCC_NOT_CLANG_VERSION_CHECK #endif #if defined(__clang__) - #define JSON_HEDLEY_GCC_NOT_CLANG_VERSION_CHECK(major,minor,patch) (0) +#define JSON_HEDLEY_GCC_NOT_CLANG_VERSION_CHECK(major, minor, patch) (0) #else - #define JSON_HEDLEY_GCC_NOT_CLANG_VERSION_CHECK(major,minor,patch) JSON_HEDLEY_GCC_VERSION_CHECK(major,minor,patch) +#define JSON_HEDLEY_GCC_NOT_CLANG_VERSION_CHECK(major, minor, patch) JSON_HEDLEY_GCC_VERSION_CHECK(major, minor, patch) #endif #if defined(JSON_HEDLEY_CLANG_HAS_ATTRIBUTE) - #undef JSON_HEDLEY_CLANG_HAS_ATTRIBUTE +#undef JSON_HEDLEY_CLANG_HAS_ATTRIBUTE #endif #define JSON_HEDLEY_CLANG_HAS_ATTRIBUTE(attribute) JSON_HEDLEY_HAS_ATTRIBUTE(attribute) #if defined(JSON_HEDLEY_CLANG_HAS_CPP_ATTRIBUTE) - #undef JSON_HEDLEY_CLANG_HAS_CPP_ATTRIBUTE +#undef JSON_HEDLEY_CLANG_HAS_CPP_ATTRIBUTE #endif #define JSON_HEDLEY_CLANG_HAS_CPP_ATTRIBUTE(attribute) JSON_HEDLEY_HAS_CPP_ATTRIBUTE(attribute) #if defined(JSON_HEDLEY_CLANG_HAS_BUILTIN) - #undef JSON_HEDLEY_CLANG_HAS_BUILTIN +#undef JSON_HEDLEY_CLANG_HAS_BUILTIN #endif #define JSON_HEDLEY_CLANG_HAS_BUILTIN(builtin) JSON_HEDLEY_HAS_BUILTIN(builtin) #if defined(JSON_HEDLEY_CLANG_HAS_FEATURE) - #undef JSON_HEDLEY_CLANG_HAS_FEATURE +#undef JSON_HEDLEY_CLANG_HAS_FEATURE #endif #define JSON_HEDLEY_CLANG_HAS_FEATURE(feature) JSON_HEDLEY_HAS_FEATURE(feature) #if defined(JSON_HEDLEY_CLANG_HAS_EXTENSION) - #undef JSON_HEDLEY_CLANG_HAS_EXTENSION +#undef JSON_HEDLEY_CLANG_HAS_EXTENSION #endif #define JSON_HEDLEY_CLANG_HAS_EXTENSION(extension) JSON_HEDLEY_HAS_EXTENSION(extension) #if defined(JSON_HEDLEY_CLANG_HAS_DECLSPEC_DECLSPEC_ATTRIBUTE) - #undef JSON_HEDLEY_CLANG_HAS_DECLSPEC_DECLSPEC_ATTRIBUTE +#undef JSON_HEDLEY_CLANG_HAS_DECLSPEC_DECLSPEC_ATTRIBUTE #endif #define JSON_HEDLEY_CLANG_HAS_DECLSPEC_ATTRIBUTE(attribute) JSON_HEDLEY_HAS_DECLSPEC_ATTRIBUTE(attribute) #if defined(JSON_HEDLEY_CLANG_HAS_WARNING) - #undef JSON_HEDLEY_CLANG_HAS_WARNING +#undef JSON_HEDLEY_CLANG_HAS_WARNING #endif #define JSON_HEDLEY_CLANG_HAS_WARNING(warning) JSON_HEDLEY_HAS_WARNING(warning) #endif /* !defined(JSON_HEDLEY_VERSION) || (JSON_HEDLEY_VERSION < X) */ - // This file contains all internal macro definitions // You MUST include macro_unscope.hpp at the end of json.hpp to undef all of them // exclude unsupported compilers #if !defined(JSON_SKIP_UNSUPPORTED_COMPILER_CHECK) - #if defined(__clang__) - #if (__clang_major__ * 10000 + __clang_minor__ * 100 + __clang_patchlevel__) < 30400 - #error "unsupported Clang version - see https://github.com/nlohmann/json#supported-compilers" - #endif - #elif defined(__GNUC__) && !(defined(__ICC) || defined(__INTEL_COMPILER)) - #if (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) < 40800 - #error "unsupported GCC version - see https://github.com/nlohmann/json#supported-compilers" - #endif - #endif +#if defined(__clang__) +#if (__clang_major__ * 10000 + __clang_minor__ * 100 + __clang_patchlevel__) < 30400 +#error "unsupported Clang version - see https://github.com/nlohmann/json#supported-compilers" +#endif +#elif defined(__GNUC__) && !(defined(__ICC) || defined(__INTEL_COMPILER)) +#if (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) < 40800 +#error "unsupported GCC version - see https://github.com/nlohmann/json#supported-compilers" +#endif +#endif #endif // C++ language standard detection -#if (defined(__cplusplus) && __cplusplus >= 201703L) || (defined(_HAS_CXX17) && _HAS_CXX17 == 1) // fix for issue #464 - #define JSON_HAS_CPP_17 - #define JSON_HAS_CPP_14 +#if (defined(__cplusplus) && __cplusplus >= 201703L) || (defined(_HAS_CXX17) && _HAS_CXX17 == 1) // fix for issue #464 +#define JSON_HAS_CPP_17 +#define JSON_HAS_CPP_14 #elif (defined(__cplusplus) && __cplusplus >= 201402L) || (defined(_HAS_CXX14) && _HAS_CXX14 == 1) - #define JSON_HAS_CPP_14 +#define JSON_HAS_CPP_14 #endif // disable float-equal warnings on GCC/clang #if defined(__clang__) || defined(__GNUC__) || defined(__GNUG__) - #pragma GCC diagnostic push - #pragma GCC diagnostic ignored "-Wfloat-equal" +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wfloat-equal" #endif // disable documentation warnings on clang #if defined(__clang__) - #pragma GCC diagnostic push - #pragma GCC diagnostic ignored "-Wdocumentation" +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wdocumentation" #endif // allow to disable exceptions #if (defined(__cpp_exceptions) || defined(__EXCEPTIONS) || defined(_CPPUNWIND)) && !defined(JSON_NOEXCEPTION) - #define JSON_THROW(exception) throw exception - #define JSON_TRY try - #define JSON_CATCH(exception) catch(exception) - #define JSON_INTERNAL_CATCH(exception) catch(exception) +#define JSON_THROW(exception) throw exception +#define JSON_TRY try +#define JSON_CATCH(exception) catch (exception) +#define JSON_INTERNAL_CATCH(exception) catch (exception) #else - #include - #define JSON_THROW(exception) std::abort() - #define JSON_TRY if(true) - #define JSON_CATCH(exception) if(false) - #define JSON_INTERNAL_CATCH(exception) if(false) +#include +#define JSON_THROW(exception) std::abort() +#define JSON_TRY if (true) +#define JSON_CATCH(exception) if (false) +#define JSON_INTERNAL_CATCH(exception) if (false) #endif // override exception macros #if defined(JSON_THROW_USER) - #undef JSON_THROW - #define JSON_THROW JSON_THROW_USER +#undef JSON_THROW +#define JSON_THROW JSON_THROW_USER #endif #if defined(JSON_TRY_USER) - #undef JSON_TRY - #define JSON_TRY JSON_TRY_USER +#undef JSON_TRY +#define JSON_TRY JSON_TRY_USER #endif #if defined(JSON_CATCH_USER) - #undef JSON_CATCH - #define JSON_CATCH JSON_CATCH_USER - #undef JSON_INTERNAL_CATCH - #define JSON_INTERNAL_CATCH JSON_CATCH_USER +#undef JSON_CATCH +#define JSON_CATCH JSON_CATCH_USER +#undef JSON_INTERNAL_CATCH +#define JSON_INTERNAL_CATCH JSON_CATCH_USER #endif #if defined(JSON_INTERNAL_CATCH_USER) - #undef JSON_INTERNAL_CATCH - #define JSON_INTERNAL_CATCH JSON_INTERNAL_CATCH_USER +#undef JSON_INTERNAL_CATCH +#define JSON_INTERNAL_CATCH JSON_INTERNAL_CATCH_USER #endif /*! @@ -1692,53 +1594,41 @@ JSON_HEDLEY_DIAGNOSTIC_POP @def NLOHMANN_JSON_SERIALIZE_ENUM @since version 3.4.0 */ -#define NLOHMANN_JSON_SERIALIZE_ENUM(ENUM_TYPE, ...) \ - template \ - inline void to_json(BasicJsonType& j, const ENUM_TYPE& e) \ - { \ - static_assert(std::is_enum::value, #ENUM_TYPE " must be an enum!"); \ - static const std::pair m[] = __VA_ARGS__; \ - auto it = std::find_if(std::begin(m), std::end(m), \ - [e](const std::pair& ej_pair) -> bool \ - { \ - return ej_pair.first == e; \ - }); \ - j = ((it != std::end(m)) ? it : std::begin(m))->second; \ - } \ - template \ - inline void from_json(const BasicJsonType& j, ENUM_TYPE& e) \ - { \ - static_assert(std::is_enum::value, #ENUM_TYPE " must be an enum!"); \ - static const std::pair m[] = __VA_ARGS__; \ - auto it = std::find_if(std::begin(m), std::end(m), \ - [j](const std::pair& ej_pair) -> bool \ - { \ - return ej_pair.second == j; \ - }); \ - e = ((it != std::end(m)) ? it : std::begin(m))->first; \ +#define NLOHMANN_JSON_SERIALIZE_ENUM(ENUM_TYPE, ...) \ + template \ + inline void to_json(BasicJsonType& j, const ENUM_TYPE& e) { \ + static_assert(std::is_enum::value, #ENUM_TYPE " must be an enum!"); \ + static const std::pair m[] = __VA_ARGS__; \ + auto it = std::find_if( \ + std::begin(m), std::end(m), \ + [e](const std::pair& ej_pair) -> bool { return ej_pair.first == e; }); \ + j = ((it != std::end(m)) ? it : std::begin(m))->second; \ + } \ + template \ + inline void from_json(const BasicJsonType& j, ENUM_TYPE& e) { \ + static_assert(std::is_enum::value, #ENUM_TYPE " must be an enum!"); \ + static const std::pair m[] = __VA_ARGS__; \ + auto it = std::find_if( \ + std::begin(m), std::end(m), \ + [j](const std::pair& ej_pair) -> bool { return ej_pair.second == j; }); \ + e = ((it != std::end(m)) ? it : std::begin(m))->first; \ } // Ugly macros to avoid uglier copy-paste when specializing basic_json. They // may be removed in the future once the class is split. -#define NLOHMANN_BASIC_JSON_TPL_DECLARATION \ - template class ObjectType, \ - template class ArrayType, \ - class StringType, class BooleanType, class NumberIntegerType, \ - class NumberUnsignedType, class NumberFloatType, \ - template class AllocatorType, \ - template class JSONSerializer> +#define NLOHMANN_BASIC_JSON_TPL_DECLARATION \ + template