From a9bc655cfb5dbcd06131ca4fe70fc26816cb2c51 Mon Sep 17 00:00:00 2001 From: wxyu Date: Thu, 21 Nov 2019 17:26:11 +0800 Subject: [PATCH 1/9] Read gpu config only gpu_resource_config.enable=true fix #467 --- CHANGELOG.md | 1 + core/src/scheduler/SchedInst.cpp | 52 ++++++++++++++------------- core/src/scheduler/SchedInst.h | 15 +++++--- core/src/server/Config.cpp | 42 +++++++++++----------- core/src/wrapper/KnowhereResource.cpp | 12 +++++-- 5 files changed, 71 insertions(+), 51 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a8b243546e..c68f655077 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,7 @@ Please mark all change in change log and use the ticket from JIRA. - \#412 - Message returned is confused when partition created with null partition name - \#416 - Drop the same partition success repeatally - \#440 - Query API in customization still uses old version +- \#440 - Server cannot startup with gpu_resource_config.enable=false in GPU version ## Feature - \#12 - Pure CPU version for Milvus diff --git a/core/src/scheduler/SchedInst.cpp b/core/src/scheduler/SchedInst.cpp index 69d293f986..f86b6f44b3 100644 --- a/core/src/scheduler/SchedInst.cpp +++ b/core/src/scheduler/SchedInst.cpp @@ -54,36 +54,40 @@ load_simple_config() { // get resources #ifdef MILVUS_GPU_VERSION + bool enable_gpu = false; server::Config& config = server::Config::GetInstance(); - std::vector gpu_ids; - config.GetGpuResourceConfigSearchResources(gpu_ids); - std::vector build_gpu_ids; - config.GetGpuResourceConfigBuildIndexResources(build_gpu_ids); - auto pcie = Connection("pcie", 12000); + config.GetGpuResourceConfigEnable(enable_gpu); + if (enable_gpu) { + std::vector gpu_ids; + config.GetGpuResourceConfigSearchResources(gpu_ids); + std::vector build_gpu_ids; + config.GetGpuResourceConfigBuildIndexResources(build_gpu_ids); + auto pcie = Connection("pcie", 12000); - std::vector not_find_build_ids; - for (auto& build_id : build_gpu_ids) { - bool find_gpu_id = false; - for (auto& gpu_id : gpu_ids) { - if (gpu_id == build_id) { - find_gpu_id = true; - break; + std::vector not_find_build_ids; + for (auto& build_id : build_gpu_ids) { + bool find_gpu_id = false; + for (auto& gpu_id : gpu_ids) { + if (gpu_id == build_id) { + find_gpu_id = true; + break; + } + } + if (not find_gpu_id) { + not_find_build_ids.emplace_back(build_id); } } - if (not find_gpu_id) { - not_find_build_ids.emplace_back(build_id); + + for (auto& gpu_id : gpu_ids) { + ResMgrInst::GetInstance()->Add(ResourceFactory::Create(std::to_string(gpu_id), "GPU", gpu_id, true, true)); + ResMgrInst::GetInstance()->Connect("cpu", std::to_string(gpu_id), pcie); } - } - for (auto& gpu_id : gpu_ids) { - ResMgrInst::GetInstance()->Add(ResourceFactory::Create(std::to_string(gpu_id), "GPU", gpu_id, true, true)); - ResMgrInst::GetInstance()->Connect("cpu", std::to_string(gpu_id), pcie); - } - - for (auto& not_find_id : not_find_build_ids) { - ResMgrInst::GetInstance()->Add( - ResourceFactory::Create(std::to_string(not_find_id), "GPU", not_find_id, true, true)); - ResMgrInst::GetInstance()->Connect("cpu", std::to_string(not_find_id), pcie); + for (auto& not_find_id : not_find_build_ids) { + ResMgrInst::GetInstance()->Add( + ResourceFactory::Create(std::to_string(not_find_id), "GPU", not_find_id, true, true)); + ResMgrInst::GetInstance()->Connect("cpu", std::to_string(not_find_id), pcie); + } } #endif } diff --git a/core/src/scheduler/SchedInst.h b/core/src/scheduler/SchedInst.h index dc2d5ade35..6273af7a9f 100644 --- a/core/src/scheduler/SchedInst.h +++ b/core/src/scheduler/SchedInst.h @@ -102,11 +102,16 @@ class OptimizerInst { if (instance == nullptr) { std::vector pass_list; #ifdef MILVUS_GPU_VERSION - pass_list.push_back(std::make_shared()); - pass_list.push_back(std::make_shared()); - pass_list.push_back(std::make_shared()); - pass_list.push_back(std::make_shared()); - pass_list.push_back(std::make_shared()); + bool enable_gpu = false; + server::Config& config = server::Config::GetInstance(); + config.GetGpuResourceConfigEnable(enable_gpu); + if (enable_gpu) { + pass_list.push_back(std::make_shared()); + pass_list.push_back(std::make_shared()); + pass_list.push_back(std::make_shared()); + pass_list.push_back(std::make_shared()); + pass_list.push_back(std::make_shared()); + } #endif pass_list.push_back(std::make_shared()); instance = std::make_shared(pass_list); diff --git a/core/src/server/Config.cpp b/core/src/server/Config.cpp index f3efcff0cc..5465c6c505 100644 --- a/core/src/server/Config.cpp +++ b/core/src/server/Config.cpp @@ -189,35 +189,37 @@ Config::ValidateConfig() { } /* gpu resource config */ -#ifdef MILVUS_GPU_VERSION bool gpu_resource_enable; s = GetGpuResourceConfigEnable(gpu_resource_enable); if (!s.ok()) { return s; } - int64_t resource_cache_capacity; - s = GetGpuResourceConfigCacheCapacity(resource_cache_capacity); - if (!s.ok()) { - return s; - } +#ifdef MILVUS_GPU_VERSION + if (gpu_resource_enable) { + int64_t resource_cache_capacity; + s = GetGpuResourceConfigCacheCapacity(resource_cache_capacity); + if (!s.ok()) { + return s; + } - float resource_cache_threshold; - s = GetGpuResourceConfigCacheThreshold(resource_cache_threshold); - if (!s.ok()) { - return s; - } + float resource_cache_threshold; + s = GetGpuResourceConfigCacheThreshold(resource_cache_threshold); + if (!s.ok()) { + return s; + } - std::vector search_resources; - s = GetGpuResourceConfigSearchResources(search_resources); - if (!s.ok()) { - return s; - } + std::vector search_resources; + s = GetGpuResourceConfigSearchResources(search_resources); + if (!s.ok()) { + return s; + } - std::vector index_build_resources; - s = GetGpuResourceConfigBuildIndexResources(index_build_resources); - if (!s.ok()) { - return s; + std::vector index_build_resources; + s = GetGpuResourceConfigBuildIndexResources(index_build_resources); + if (!s.ok()) { + return s; + } } #endif diff --git a/core/src/wrapper/KnowhereResource.cpp b/core/src/wrapper/KnowhereResource.cpp index 5a2296b16e..42105777aa 100644 --- a/core/src/wrapper/KnowhereResource.cpp +++ b/core/src/wrapper/KnowhereResource.cpp @@ -37,6 +37,16 @@ constexpr int64_t M_BYTE = 1024 * 1024; Status KnowhereResource::Initialize() { #ifdef MILVUS_GPU_VERSION + Status s; + bool enable_gpu = false; + server::Config& config = server::Config::GetInstance(); + s = config.GetGpuResourceConfigEnable(enable_gpu); + if (!s.ok()) + return s; + + if (not enable_gpu) + return Status::OK(); + struct GpuResourceSetting { int64_t pinned_memory = 300 * M_BYTE; int64_t temp_memory = 300 * M_BYTE; @@ -44,10 +54,8 @@ KnowhereResource::Initialize() { }; using GpuResourcesArray = std::map; GpuResourcesArray gpu_resources; - Status s; // get build index gpu resource - server::Config& config = server::Config::GetInstance(); std::vector build_index_gpus; s = config.GetGpuResourceConfigBuildIndexResources(build_index_gpus); if (!s.ok()) From 29d9ac4954b88b087116706108ada657b548e0cd Mon Sep 17 00:00:00 2001 From: zhenwu Date: Fri, 22 Nov 2019 11:13:43 +0800 Subject: [PATCH 2/9] [skip ci] Add ann-dataset accuracy pipeline --- .../ci/function/file_transfer.groovy | 10 ++ .../ci/jenkinsfile/acc_test.groovy | 16 ++ .../ci/jenkinsfile/cleanup.groovy | 13 ++ .../jenkinsfile/deploy_default_server.groovy | 22 +++ .../ci/jenkinsfile/notify.groovy | 15 ++ tests/milvus_ann_acc/ci/main_jenkinsfile | 130 ++++++++++++++ .../pod_containers/milvus-testframework.yaml | 13 ++ tests/milvus_ann_acc/client.py | 33 ++-- tests/milvus_ann_acc/main.py | 65 +++++-- tests/milvus_ann_acc/requirements.txt | 5 + tests/milvus_ann_acc/runner.py | 162 ++++++++++++++++++ tests/milvus_ann_acc/suite.yaml | 29 ++++ tests/milvus_ann_acc/suite.yaml.bak | 11 ++ tests/milvus_ann_acc/suite_czr.yaml | 20 +++ tests/milvus_ann_acc/suite_debug.yaml | 10 ++ tests/milvus_ann_acc/test.py | 157 ++++------------- 16 files changed, 547 insertions(+), 164 deletions(-) create mode 100644 tests/milvus_ann_acc/ci/function/file_transfer.groovy create mode 100644 tests/milvus_ann_acc/ci/jenkinsfile/acc_test.groovy create mode 100644 tests/milvus_ann_acc/ci/jenkinsfile/cleanup.groovy create mode 100644 tests/milvus_ann_acc/ci/jenkinsfile/deploy_default_server.groovy create mode 100644 tests/milvus_ann_acc/ci/jenkinsfile/notify.groovy create mode 100644 tests/milvus_ann_acc/ci/main_jenkinsfile create mode 100644 tests/milvus_ann_acc/ci/pod_containers/milvus-testframework.yaml create mode 100644 tests/milvus_ann_acc/runner.py create mode 100644 tests/milvus_ann_acc/suite.yaml create mode 100644 tests/milvus_ann_acc/suite.yaml.bak create mode 100644 tests/milvus_ann_acc/suite_czr.yaml create mode 100644 tests/milvus_ann_acc/suite_debug.yaml diff --git a/tests/milvus_ann_acc/ci/function/file_transfer.groovy b/tests/milvus_ann_acc/ci/function/file_transfer.groovy new file mode 100644 index 0000000000..bebae14832 --- /dev/null +++ b/tests/milvus_ann_acc/ci/function/file_transfer.groovy @@ -0,0 +1,10 @@ +def FileTransfer (sourceFiles, remoteDirectory, remoteIP, protocol = "ftp", makeEmptyDirs = true) { + if (protocol == "ftp") { + ftpPublisher masterNodeName: '', paramPublish: [parameterName: ''], alwaysPublishFromMaster: false, continueOnError: false, failOnError: true, publishers: [ + [configName: "${remoteIP}", transfers: [ + [asciiMode: false, cleanRemote: false, excludes: '', flatten: false, makeEmptyDirs: "${makeEmptyDirs}", noDefaultExcludes: false, patternSeparator: '[, ]+', remoteDirectory: "${remoteDirectory}", remoteDirectorySDF: false, removePrefix: '', sourceFiles: "${sourceFiles}"]], usePromotionTimestamp: true, useWorkspaceInPromotion: false, verbose: true + ] + ] + } +} +return this diff --git a/tests/milvus_ann_acc/ci/jenkinsfile/acc_test.groovy b/tests/milvus_ann_acc/ci/jenkinsfile/acc_test.groovy new file mode 100644 index 0000000000..1ce327b802 --- /dev/null +++ b/tests/milvus_ann_acc/ci/jenkinsfile/acc_test.groovy @@ -0,0 +1,16 @@ +timeout(time: 1800, unit: 'MINUTES') { + try { + dir ("milvu_ann_acc") { + print "Git clone url: ${TEST_URL}:${TEST_BRANCH}" + checkout([$class: 'GitSCM', branches: [[name: "${TEST_BRANCH}"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "${TEST_URL}", name: 'origin', refspec: "+refs/heads/${TEST_BRANCH}:refs/remotes/origin/${TEST_BRANCH}"]]]) + print "Install requirements" + sh 'python3 -m pip install -r requirements.txt -i http://pypi.douban.com/simple --trusted-host pypi.douban.com' + // sleep(120000) + sh "python3 main.py --suite=${params.SUITE} --host=acc-test-${env.JOB_NAME}-${env.BUILD_NUMBER}-engine.milvus.svc.cluster.local --port=19530" + } + } catch (exc) { + echo 'Milvus Ann Accuracy Test Failed !' + throw exc + } +} + diff --git a/tests/milvus_ann_acc/ci/jenkinsfile/cleanup.groovy b/tests/milvus_ann_acc/ci/jenkinsfile/cleanup.groovy new file mode 100644 index 0000000000..2e9332fa6e --- /dev/null +++ b/tests/milvus_ann_acc/ci/jenkinsfile/cleanup.groovy @@ -0,0 +1,13 @@ +try { + def result = sh script: "helm status ${env.JOB_NAME}-${env.BUILD_NUMBER}", returnStatus: true + if (!result) { + sh "helm del --purge ${env.JOB_NAME}-${env.BUILD_NUMBER}" + } +} catch (exc) { + def result = sh script: "helm status ${env.JOB_NAME}-${env.BUILD_NUMBER}", returnStatus: true + if (!result) { + sh "helm del --purge ${env.JOB_NAME}-${env.BUILD_NUMBER}" + } + throw exc +} + diff --git a/tests/milvus_ann_acc/ci/jenkinsfile/deploy_default_server.groovy b/tests/milvus_ann_acc/ci/jenkinsfile/deploy_default_server.groovy new file mode 100644 index 0000000000..951bb69941 --- /dev/null +++ b/tests/milvus_ann_acc/ci/jenkinsfile/deploy_default_server.groovy @@ -0,0 +1,22 @@ +timeout(time: 30, unit: 'MINUTES') { + try { + dir ("milvus") { + sh 'helm init --client-only --skip-refresh --stable-repo-url https://kubernetes.oss-cn-hangzhou.aliyuncs.com/charts' + sh 'helm repo update' + checkout([$class: 'GitSCM', branches: [[name: "${HELM_BRANCH}"]], userRemoteConfigs: [[url: "${HELM_URL}", name: 'origin', refspec: "+refs/heads/${HELM_BRANCH}:refs/remotes/origin/${HELM_BRANCH}"]]]) + dir ("milvus") { + sh "helm install --wait --timeout 300 --set engine.image.tag=${IMAGE_TAG} --set expose.type=clusterIP --name acc-test-${env.JOB_NAME}-${env.BUILD_NUMBER} -f ci/db_backend/sqlite_${params.IMAGE_TYPE}_values.yaml -f ci/filebeat/values.yaml --namespace milvus --version ${HELM_BRANCH} ." + } + } + // dir ("milvus") { + // checkout([$class: 'GitSCM', branches: [[name: "${env.SERVER_BRANCH}"]], userRemoteConfigs: [[url: "${env.SERVER_URL}", name: 'origin', refspec: "+refs/heads/${env.SERVER_BRANCH}:refs/remotes/origin/${env.SERVER_BRANCH}"]]]) + // dir ("milvus") { + // load "ci/jenkins/step/deploySingle2Dev.groovy" + // } + // } + } catch (exc) { + echo 'Deploy Milvus Server Failed !' + throw exc + } +} + diff --git a/tests/milvus_ann_acc/ci/jenkinsfile/notify.groovy b/tests/milvus_ann_acc/ci/jenkinsfile/notify.groovy new file mode 100644 index 0000000000..0a257b8cd8 --- /dev/null +++ b/tests/milvus_ann_acc/ci/jenkinsfile/notify.groovy @@ -0,0 +1,15 @@ +def notify() { + if (!currentBuild.resultIsBetterOrEqualTo('SUCCESS')) { + // Send an email only if the build status has changed from green/unstable to red + emailext subject: '$DEFAULT_SUBJECT', + body: '$DEFAULT_CONTENT', + recipientProviders: [ + [$class: 'DevelopersRecipientProvider'], + [$class: 'RequesterRecipientProvider'] + ], + replyTo: '$DEFAULT_REPLYTO', + to: '$DEFAULT_RECIPIENTS' + } +} +return this + diff --git a/tests/milvus_ann_acc/ci/main_jenkinsfile b/tests/milvus_ann_acc/ci/main_jenkinsfile new file mode 100644 index 0000000000..9fdac4fc6e --- /dev/null +++ b/tests/milvus_ann_acc/ci/main_jenkinsfile @@ -0,0 +1,130 @@ +pipeline { + agent none + + options { + timestamps() + } + + parameters{ + choice choices: ['cpu', 'gpu'], description: 'cpu or gpu version', name: 'IMAGE_TYPE' + string defaultValue: '0.6.0', description: 'server image version', name: 'IMAGE_VERSION', trim: true + string defaultValue: 'suite.yaml', description: 'test suite config yaml', name: 'SUITE', trim: true + string defaultValue: '09509e53-9125-4f5d-9ce8-42855987ad67', description: 'git credentials', name: 'GIT_USER', trim: true + } + + environment { + IMAGE_TAG = "${params.IMAGE_VERSION}-${params.IMAGE_TYPE}-ubuntu18.04-release" + HELM_URL = "https://github.com/milvus-io/milvus-helm.git" + HELM_BRANCH = "0.6.0" + TEST_URL = "git@192.168.1.105:Test/milvus_ann_acc.git" + TEST_BRANCH = "0.6.0" + } + + stages { + stage("Setup env") { + agent { + kubernetes { + label 'dev-test' + defaultContainer 'jnlp' + yaml """ + apiVersion: v1 + kind: Pod + metadata: + labels: + app: milvus + componet: test + spec: + containers: + - name: milvus-testframework + image: registry.zilliz.com/milvus/milvus-test:v0.2 + command: + - cat + tty: true + volumeMounts: + - name: kubeconf + mountPath: /root/.kube/ + readOnly: true + - name: hdf5-path + mountPath: /test + readOnly: true + volumes: + - name: kubeconf + secret: + secretName: test-cluster-config + - name: hdf5-path + flexVolume: + driver: "fstab/cifs" + fsType: "cifs" + secretRef: + name: "cifs-test-secret" + options: + networkPath: "//192.168.1.126/test" + mountOptions: "vers=1.0" + """ + } + } + + stages { + stage("Deploy Default Server") { + steps { + gitlabCommitStatus(name: 'Accuracy Test') { + container('milvus-testframework') { + script { + print "In Deploy Default Server Stage" + load "${env.WORKSPACE}/ci/jenkinsfile/deploy_default_server.groovy" + } + } + } + } + } + stage("Acc Test") { + steps { + gitlabCommitStatus(name: 'Accuracy Test') { + container('milvus-testframework') { + script { + print "In Acc test stage" + load "${env.WORKSPACE}/ci/jenkinsfile/acc_test.groovy" + } + } + } + } + } + stage ("Cleanup Env") { + steps { + gitlabCommitStatus(name: 'Cleanup Env') { + container('milvus-testframework') { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/cleanup.groovy" + } + } + } + } + } + } + post { + always { + container('milvus-testframework') { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/cleanup.groovy" + } + } + } + success { + script { + echo "Milvus ann-accuracy test success !" + } + } + aborted { + script { + echo "Milvus ann-accuracy test aborted !" + } + } + failure { + script { + echo "Milvus ann-accuracy test failed !" + } + } + } + } + } +} diff --git a/tests/milvus_ann_acc/ci/pod_containers/milvus-testframework.yaml b/tests/milvus_ann_acc/ci/pod_containers/milvus-testframework.yaml new file mode 100644 index 0000000000..6b1d6c7dfd --- /dev/null +++ b/tests/milvus_ann_acc/ci/pod_containers/milvus-testframework.yaml @@ -0,0 +1,13 @@ +apiVersion: v1 +kind: Pod +metadata: + labels: + app: milvus + componet: testframework +spec: + containers: + - name: milvus-testframework + image: registry.zilliz.com/milvus/milvus-test:v0.2 + command: + - cat + tty: true diff --git a/tests/milvus_ann_acc/client.py b/tests/milvus_ann_acc/client.py index de4ef17cb6..6fec829612 100644 --- a/tests/milvus_ann_acc/client.py +++ b/tests/milvus_ann_acc/client.py @@ -8,7 +8,7 @@ import numpy import sklearn.preprocessing from milvus import Milvus, IndexType, MetricType -logger = logging.getLogger("milvus_ann_acc.client") +logger = logging.getLogger("milvus_acc.client") SERVER_HOST_DEFAULT = "127.0.0.1" SERVER_PORT_DEFAULT = 19530 @@ -28,17 +28,17 @@ def time_wrapper(func): class MilvusClient(object): - def __init__(self, table_name=None, ip=None, port=None): + def __init__(self, table_name=None, host=None, port=None): self._milvus = Milvus() self._table_name = table_name try: - if not ip: + if not host: self._milvus.connect( host = SERVER_HOST_DEFAULT, port = SERVER_PORT_DEFAULT) else: self._milvus.connect( - host = ip, + host = host, port = port) except Exception as e: raise e @@ -113,7 +113,6 @@ class MilvusClient(object): X = X.astype(numpy.float32) status, results = self._milvus.search_vectors(self._table_name, top_k, nprobe, X.tolist()) self.check_status(status) - # logger.info(results[0]) ids = [] for result in results: tmp_ids = [] @@ -125,24 +124,20 @@ class MilvusClient(object): def count(self): return self._milvus.get_table_row_count(self._table_name)[1] - def delete(self, timeout=60): - logger.info("Start delete table: %s" % self._table_name) - self._milvus.delete_table(self._table_name) - i = 0 - while i < timeout: - if self.count(): - time.sleep(1) - i = i + 1 - else: - break - if i >= timeout: - logger.error("Delete table timeout") + def delete(self, table_name): + logger.info("Start delete table: %s" % table_name) + return self._milvus.delete_table(table_name) def describe(self): return self._milvus.describe_table(self._table_name) - def exists_table(self): - return self._milvus.has_table(self._table_name) + def exists_table(self, table_name): + return self._milvus.has_table(table_name) + + def get_server_version(self): + status, res = self._milvus.server_version() + self.check_status(status) + return res @time_wrapper def preload_table(self): diff --git a/tests/milvus_ann_acc/main.py b/tests/milvus_ann_acc/main.py index 308e8246c7..703303232d 100644 --- a/tests/milvus_ann_acc/main.py +++ b/tests/milvus_ann_acc/main.py @@ -1,26 +1,57 @@ - +import os +import sys import argparse +from yaml import load, dump +import logging +from logging import handlers +from client import MilvusClient +import runner + +LOG_FOLDER = "logs" +logger = logging.getLogger("milvus_acc") +formatter = logging.Formatter('[%(asctime)s] [%(levelname)-4s] [%(pathname)s:%(lineno)d] %(message)s') +if not os.path.exists(LOG_FOLDER): + os.system('mkdir -p %s' % LOG_FOLDER) +fileTimeHandler = handlers.TimedRotatingFileHandler(os.path.join(LOG_FOLDER, 'acc'), "D", 1, 10) +fileTimeHandler.suffix = "%Y%m%d.log" +fileTimeHandler.setFormatter(formatter) +logging.basicConfig(level=logging.DEBUG) +fileTimeHandler.setFormatter(formatter) +logger.addHandler(fileTimeHandler) + def main(): parser = argparse.ArgumentParser( formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser.add_argument( - '--dataset', - metavar='NAME', - help='the dataset to load training points from', - default='glove-100-angular', - choices=DATASETS.keys()) + "--host", + default="127.0.0.1", + help="server host") parser.add_argument( - "-k", "--count", - default=10, - type=positive_int, - help="the number of near neighbours to search for") + "--port", + default=19530, + help="server port") parser.add_argument( - '--definitions', + '--suite', metavar='FILE', - help='load algorithm definitions from FILE', - default='algos.yaml') - parser.add_argument( - '--image-tag', - default=None, - help='pull image first') \ No newline at end of file + help='load config definitions from suite_czr' + '.yaml', + default='suite_czr.yaml') + args = parser.parse_args() + if args.suite: + with open(args.suite, "r") as f: + suite = load(f) + hdf5_path = suite["hdf5_path"] + dataset_configs = suite["datasets"] + if not hdf5_path or not dataset_configs: + logger.warning("No datasets given") + sys.exit() + f.close() + for dataset_config in dataset_configs: + logger.debug(dataset_config) + milvus_instance = MilvusClient(host=args.host, port=args.port) + runner.run(milvus_instance, dataset_config, hdf5_path) + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/tests/milvus_ann_acc/requirements.txt b/tests/milvus_ann_acc/requirements.txt index 8c10e71b1f..1f2b337423 100644 --- a/tests/milvus_ann_acc/requirements.txt +++ b/tests/milvus_ann_acc/requirements.txt @@ -2,3 +2,8 @@ numpy==1.16.3 pymilvus>=0.2.0 scikit-learn==0.19.1 h5py==2.7.1 +influxdb==5.2.2 +pyyaml==3.12 +tableprint==0.8.0 +ansicolors==1.1.8 +scipy==1.3.1 \ No newline at end of file diff --git a/tests/milvus_ann_acc/runner.py b/tests/milvus_ann_acc/runner.py new file mode 100644 index 0000000000..88a5d24016 --- /dev/null +++ b/tests/milvus_ann_acc/runner.py @@ -0,0 +1,162 @@ +import os +import pdb +import time +import random +import sys +import logging +import h5py +import numpy +from influxdb import InfluxDBClient + +INSERT_INTERVAL = 100000 +# s +DELETE_INTERVAL_TIME = 5 +INFLUXDB_HOST = "192.168.1.194" +INFLUXDB_PORT = 8086 +INFLUXDB_USER = "admin" +INFLUXDB_PASSWD = "admin" +INFLUXDB_NAME = "test_result" +influxdb_client = InfluxDBClient(host=INFLUXDB_HOST, port=INFLUXDB_PORT, username=INFLUXDB_USER, password=INFLUXDB_PASSWD, database=INFLUXDB_NAME) + +logger = logging.getLogger("milvus_acc.runner") + + +def parse_dataset_name(dataset_name): + data_type = dataset_name.split("-")[0] + dimension = int(dataset_name.split("-")[1]) + metric = dataset_name.split("-")[-1] + # metric = dataset.attrs['distance'] + # dimension = len(dataset["train"][0]) + if metric == "euclidean": + metric_type = "l2" + elif metric == "angular": + metric_type = "ip" + return ("ann"+data_type, dimension, metric_type) + + +def get_dataset(hdf5_path, dataset_name): + file_path = os.path.join(hdf5_path, '%s.hdf5' % dataset_name) + if not os.path.exists(file_path): + raise Exception("%s not existed" % file_path) + dataset = h5py.File(file_path) + return dataset + + +def get_table_name(hdf5_path, dataset_name, index_file_size): + data_type, dimension, metric_type = parse_dataset_name(dataset_name) + dataset = get_dataset(hdf5_path, dataset_name) + table_size = len(dataset["train"]) + table_size = str(table_size // 1000000)+"m" + table_name = data_type+'_'+table_size+'_'+str(index_file_size)+'_'+str(dimension)+'_'+metric_type + return table_name + + +def recall_calc(result_ids, true_ids, top_k, recall_k): + sum_intersect_num = 0 + recall = 0.0 + for index, result_item in enumerate(result_ids): + if len(set(true_ids[index][:top_k])) != len(set(result_item)): + logger.warning("Error happened: query result length is wrong") + continue + tmp = set(true_ids[index][:recall_k]).intersection(set(result_item)) + sum_intersect_num = sum_intersect_num + len(tmp) + recall = round(sum_intersect_num / (len(result_ids) * recall_k), 4) + return recall + + +def run(milvus, config, hdf5_path, force=True): + server_version = milvus.get_server_version() + logger.info(server_version) + + for dataset_name, config_value in config.items(): + dataset = get_dataset(hdf5_path, dataset_name) + index_file_sizes = config_value["index_file_sizes"] + index_types = config_value["index_types"] + nlists = config_value["nlists"] + search_param = config_value["search_param"] + top_ks = search_param["top_ks"] + nprobes = search_param["nprobes"] + nqs = search_param["nqs"] + + for index_file_size in index_file_sizes: + table_name = get_table_name(hdf5_path, dataset_name, index_file_size) + if milvus.exists_table(table_name): + if force is True: + logger.info("Re-create table: %s" % table_name) + milvus.delete(table_name) + time.sleep(DELETE_INTERVAL_TIME) + else: + logger.warning("Table name: %s existed" % table_name) + continue + data_type, dimension, metric_type = parse_dataset_name(dataset_name) + milvus.create_table(table_name, dimension, index_file_size, metric_type) + logger.info(milvus.describe()) + insert_vectors = numpy.array(dataset["train"]) + # milvus.insert(insert_vectors) + + loops = len(insert_vectors) // INSERT_INTERVAL + 1 + for i in range(loops): + start = i*INSERT_INTERVAL + end = min((i+1)*INSERT_INTERVAL, len(insert_vectors)) + tmp_vectors = insert_vectors[start:end] + if start < end: + milvus.insert(tmp_vectors, ids=[i for i in range(start, end)]) + time.sleep(20) + row_count = milvus.count() + logger.info("Table: %s, row count: %s" % (table_name, row_count)) + if milvus.count() != len(insert_vectors): + logger.error("Table row count is not equal to insert vectors") + return + for index_type in index_types: + for nlist in nlists: + milvus.create_index(index_type, nlist) + logger.info(milvus.describe_index()) + logger.info("Start preload table: %s, index_type: %s, nlist: %s" % (table_name, index_type, nlist)) + milvus.preload_table() + true_ids = numpy.array(dataset["neighbors"]) + for nprobe in nprobes: + for nq in nqs: + query_vectors = numpy.array(dataset["test"][:nq]) + for top_k in top_ks: + rec1 = 0.0 + rec10 = 0.0 + rec100 = 0.0 + result_ids = milvus.query(query_vectors, top_k, nprobe) + logger.info("Query result: %s" % len(result_ids)) + rec1 = recall_calc(result_ids, true_ids, top_k, 1) + if top_k == 10: + rec10 = recall_calc(result_ids, true_ids, top_k, 10) + if top_k == 100: + rec10 = recall_calc(result_ids, true_ids, top_k, 10) + rec100 = recall_calc(result_ids, true_ids, top_k, 100) + avg_radio = recall_calc(result_ids, true_ids, top_k, top_k) + logger.debug("Recall_1: %s" % rec1) + logger.debug("Recall_10: %s" % rec10) + logger.debug("Recall_100: %s" % rec100) + logger.debug("Accuracy: %s" % avg_radio) + acc_record = [{ + "measurement": "accuracy", + "tags": { + "server_version": server_version, + "dataset": dataset_name, + "index_file_size": index_file_size, + "index_type": index_type, + "nlist": nlist, + "search_nprobe": nprobe, + "top_k": top_k, + "nq": len(query_vectors) + }, + # "time": time.ctime(), + "time": time.strftime("%Y-%m-%dT%H:%M:%SZ"), + "fields": { + "recall1": rec1, + "recall10": rec10, + "recall100": rec100, + "avg_radio": avg_radio + } + }] + logger.info(acc_record) + try: + res = influxdb_client.write_points(acc_record) + except Exception as e: + logger.error("Insert infuxdb failed: %s" % str(e)) diff --git a/tests/milvus_ann_acc/suite.yaml b/tests/milvus_ann_acc/suite.yaml new file mode 100644 index 0000000000..1137ccfa64 --- /dev/null +++ b/tests/milvus_ann_acc/suite.yaml @@ -0,0 +1,29 @@ +datasets: + - sift-128-euclidean: + index_file_sizes: [50, 1024] + index_types: ['ivf_flat', 'ivf_sq8', 'ivf_sq8h'] + # index_types: ['ivf_sq8'] + nlists: [16384] + search_param: + nprobes: [1, 32, 128, 256] + top_ks: [10] + nqs: [10000] + - glove-25-angular: + index_file_sizes: [50, 1024] + index_types: ['ivf_flat', 'ivf_sq8', 'ivf_sq8h'] + # index_types: ['ivf_sq8'] + nlists: [16384] + search_param: + nprobes: [1, 32, 128, 256] + top_ks: [10] + nqs: [10000] + - glove-200-angular: + index_file_sizes: [50, 1024] + index_types: ['ivf_flat', 'ivf_sq8', 'ivf_sq8h'] + # index_types: ['ivf_sq8'] + nlists: [16384] + search_param: + nprobes: [1, 32, 128, 256] + top_ks: [10] + nqs: [10000] +hdf5_path: /test/milvus/ann_hdf5/ \ No newline at end of file diff --git a/tests/milvus_ann_acc/suite.yaml.bak b/tests/milvus_ann_acc/suite.yaml.bak new file mode 100644 index 0000000000..7736786d03 --- /dev/null +++ b/tests/milvus_ann_acc/suite.yaml.bak @@ -0,0 +1,11 @@ +datasets: + - glove-200-angular: + index_file_sizes: [1024] + index_types: ['ivf_sq8'] + # index_types: ['ivf_sq8'] + nlists: [16384] + search_param: + nprobes: [256, 400, 256] + top_ks: [100] + nqs: [10000] +hdf5_path: /test/milvus/ann_hdf5/ diff --git a/tests/milvus_ann_acc/suite_czr.yaml b/tests/milvus_ann_acc/suite_czr.yaml new file mode 100644 index 0000000000..7e2b0c8708 --- /dev/null +++ b/tests/milvus_ann_acc/suite_czr.yaml @@ -0,0 +1,20 @@ +datasets: + - sift-128-euclidean: + index_file_sizes: [1024] + index_types: ['ivf_sq8', 'ivf_sq8h'] + # index_types: ['ivf_sq8'] + nlists: [16384] + search_param: + nprobes: [16, 128, 1024] + top_ks: [1, 10, 100] + nqs: [10, 100, 1000] + - glove-200-angular: + index_file_sizes: [1024] + index_types: ['ivf_sq8', 'ivf_sq8h'] + # index_types: ['ivf_sq8'] + nlists: [16384] + search_param: + nprobes: [16, 128, 1024] + top_ks: [1, 10, 100] + nqs: [10, 100, 1000] +hdf5_path: /test/milvus/ann_hdf5/ \ No newline at end of file diff --git a/tests/milvus_ann_acc/suite_debug.yaml b/tests/milvus_ann_acc/suite_debug.yaml new file mode 100644 index 0000000000..ca463a9c40 --- /dev/null +++ b/tests/milvus_ann_acc/suite_debug.yaml @@ -0,0 +1,10 @@ +datasets: + - sift-128-euclidean: + index_file_sizes: [1024] + index_types: ['ivf_flat'] + nlists: [16384] + search_param: + nprobes: [1, 256] + top_ks: [10] + nqs: [10000] +hdf5_path: /test/milvus/ann_hdf5/ diff --git a/tests/milvus_ann_acc/test.py b/tests/milvus_ann_acc/test.py index c4fbc33195..44ffd53051 100644 --- a/tests/milvus_ann_acc/test.py +++ b/tests/milvus_ann_acc/test.py @@ -1,132 +1,33 @@ -import os -import pdb import time -import random -import sys -import h5py -import numpy -import logging -from logging import handlers +from influxdb import InfluxDBClient -from client import MilvusClient +INFLUXDB_HOST = "192.168.1.194" +INFLUXDB_PORT = 8086 +INFLUXDB_USER = "admin" +INFLUXDB_PASSWD = "admin" +INFLUXDB_NAME = "test_result" -LOG_FOLDER = "logs" -logger = logging.getLogger("milvus_ann_acc") +client = InfluxDBClient(host=INFLUXDB_HOST, port=INFLUXDB_PORT, username=INFLUXDB_USER, password=INFLUXDB_PASSWD, database=INFLUXDB_NAME) -formatter = logging.Formatter('[%(asctime)s] [%(levelname)-4s] [%(pathname)s:%(lineno)d] %(message)s') -if not os.path.exists(LOG_FOLDER): - os.system('mkdir -p %s' % LOG_FOLDER) -fileTimeHandler = handlers.TimedRotatingFileHandler(os.path.join(LOG_FOLDER, 'acc'), "D", 1, 10) -fileTimeHandler.suffix = "%Y%m%d.log" -fileTimeHandler.setFormatter(formatter) -logging.basicConfig(level=logging.DEBUG) -fileTimeHandler.setFormatter(formatter) -logger.addHandler(fileTimeHandler) - - -def get_dataset_fn(dataset_name): - file_path = "/test/milvus/ann_hdf5/" - if not os.path.exists(file_path): - raise Exception("%s not exists" % file_path) - return os.path.join(file_path, '%s.hdf5' % dataset_name) - - -def get_dataset(dataset_name): - hdf5_fn = get_dataset_fn(dataset_name) - hdf5_f = h5py.File(hdf5_fn) - return hdf5_f - - -def parse_dataset_name(dataset_name): - data_type = dataset_name.split("-")[0] - dimension = int(dataset_name.split("-")[1]) - metric = dataset_name.split("-")[-1] - # metric = dataset.attrs['distance'] - # dimension = len(dataset["train"][0]) - if metric == "euclidean": - metric_type = "l2" - elif metric == "angular": - metric_type = "ip" - return ("ann"+data_type, dimension, metric_type) - - -def get_table_name(dataset_name, index_file_size): - data_type, dimension, metric_type = parse_dataset_name(dataset_name) - dataset = get_dataset(dataset_name) - table_size = len(dataset["train"]) - table_size = str(table_size // 1000000)+"m" - table_name = data_type+'_'+table_size+'_'+str(index_file_size)+'_'+str(dimension)+'_'+metric_type - return table_name - - -def main(dataset_name, index_file_size, nlist=16384, force=False): - top_k = 10 - nprobes = [32, 128] - - dataset = get_dataset(dataset_name) - table_name = get_table_name(dataset_name, index_file_size) - m = MilvusClient(table_name) - if m.exists_table(): - if force is True: - logger.info("Re-create table: %s" % table_name) - m.delete() - time.sleep(10) - else: - logger.info("Table name: %s existed" % table_name) - return - data_type, dimension, metric_type = parse_dataset_name(dataset_name) - m.create_table(table_name, dimension, index_file_size, metric_type) - print(m.describe()) - vectors = numpy.array(dataset["train"]) - query_vectors = numpy.array(dataset["test"]) - # m.insert(vectors) - - interval = 100000 - loops = len(vectors) // interval + 1 - - for i in range(loops): - start = i*interval - end = min((i+1)*interval, len(vectors)) - tmp_vectors = vectors[start:end] - if start < end: - m.insert(tmp_vectors, ids=[i for i in range(start, end)]) - - time.sleep(60) - print(m.count()) - - for index_type in ["ivf_flat", "ivf_sq8", "ivf_sq8h"]: - m.create_index(index_type, nlist) - print(m.describe_index()) - if m.count() != len(vectors): - return - m.preload_table() - true_ids = numpy.array(dataset["neighbors"]) - for nprobe in nprobes: - print("nprobe: %s" % nprobe) - sum_radio = 0.0; avg_radio = 0.0 - result_ids = m.query(query_vectors, top_k, nprobe) - # print(result_ids[:10]) - for index, result_item in enumerate(result_ids): - if len(set(true_ids[index][:top_k])) != len(set(result_item)): - logger.info("Error happened") - # logger.info(query_vectors[index]) - # logger.info(true_ids[index][:top_k], result_item) - tmp = set(true_ids[index][:top_k]).intersection(set(result_item)) - sum_radio = sum_radio + (len(tmp) / top_k) - avg_radio = round(sum_radio / len(result_ids), 4) - logger.info(avg_radio) - m.drop_index() - - -if __name__ == "__main__": - print("glove-25-angular") - # main("sift-128-euclidean", 1024, force=True) - for index_file_size in [50, 1024]: - print("Index file size: %d" % index_file_size) - main("glove-25-angular", index_file_size, force=True) - - print("sift-128-euclidean") - for index_file_size in [50, 1024]: - print("Index file size: %d" % index_file_size) - main("sift-128-euclidean", index_file_size, force=True) - # m = MilvusClient() \ No newline at end of file +print(client.get_list_database()) +acc_record = [{ + "measurement": "accuracy", + "tags": { + "server_version": "0.4.3", + "dataset": "test", + "index_type": "test", + "nlist": 12, + "search_nprobe": 12, + "top_k": 1, + "nq": 1 + }, + "time": time.ctime(), + "fields": { + "accuracy": 0.1 + } +}] +try: + res = client.write_points(acc_record) + print(res) +except Exception as e: + print(str(e)) \ No newline at end of file From 54e113ba31763b2cec4370c38558641e10802ed3 Mon Sep 17 00:00:00 2001 From: zhenwu Date: Fri, 22 Nov 2019 14:12:57 +0800 Subject: [PATCH 3/9] Disable multiprocess cases --- tests/milvus_python_test/test_connect.py | 3 ++- tests/milvus_python_test/test_mix.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/milvus_python_test/test_connect.py b/tests/milvus_python_test/test_connect.py index 143ac4d8bf..b5b55f3ee1 100644 --- a/tests/milvus_python_test/test_connect.py +++ b/tests/milvus_python_test/test_connect.py @@ -149,7 +149,8 @@ class TestConnect: milvus.connect(uri=uri_value, timeout=1) assert not milvus.connected() - def test_connect_with_multiprocess(self, args): + # disable + def _test_connect_with_multiprocess(self, args): ''' target: test uri connect with multiprocess method: set correct uri, test with multiprocessing connecting diff --git a/tests/milvus_python_test/test_mix.py b/tests/milvus_python_test/test_mix.py index 5ef9ba2cde..d9331151dd 100644 --- a/tests/milvus_python_test/test_mix.py +++ b/tests/milvus_python_test/test_mix.py @@ -25,7 +25,8 @@ index_params = {'index_type': IndexType.IVFLAT, 'nlist': 16384} class TestMixBase: - def test_search_during_createIndex(self, args): + # disable + def _test_search_during_createIndex(self, args): loops = 10000 table = gen_unique_str() query_vecs = [vectors[0], vectors[1]] From 5268a1628ccfc021bc425915bb7151a28e775d34 Mon Sep 17 00:00:00 2001 From: zhenwu Date: Fri, 22 Nov 2019 14:18:33 +0800 Subject: [PATCH 4/9] Update caser run timeout --- tests/milvus_ann_acc/ci/jenkinsfile/acc_test.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/milvus_ann_acc/ci/jenkinsfile/acc_test.groovy b/tests/milvus_ann_acc/ci/jenkinsfile/acc_test.groovy index 1ce327b802..c1b2b2ed64 100644 --- a/tests/milvus_ann_acc/ci/jenkinsfile/acc_test.groovy +++ b/tests/milvus_ann_acc/ci/jenkinsfile/acc_test.groovy @@ -1,4 +1,4 @@ -timeout(time: 1800, unit: 'MINUTES') { +timeout(time: 7200, unit: 'MINUTES') { try { dir ("milvu_ann_acc") { print "Git clone url: ${TEST_URL}:${TEST_BRANCH}" From f39eb065461fc9640409978af9c86f60b3361b34 Mon Sep 17 00:00:00 2001 From: quicksilver Date: Fri, 22 Nov 2019 15:01:57 +0800 Subject: [PATCH 5/9] clean up old jenkins ci --- ci/function/file_transfer.groovy | 10 - ci/jenkinsfile/cleanup_dev.groovy | 13 - ci/jenkinsfile/cleanup_staging.groovy | 13 - ci/jenkinsfile/cluster_cleanup_dev.groovy | 13 - ci/jenkinsfile/cluster_deploy2dev.groovy | 24 - ci/jenkinsfile/cluster_dev_test.groovy | 12 - ci/jenkinsfile/deploy2dev.groovy | 16 - ci/jenkinsfile/deploy2staging.groovy | 16 - ci/jenkinsfile/dev_test.groovy | 28 - ci/jenkinsfile/dev_test_all.groovy | 29 -- ci/jenkinsfile/milvus_build.groovy | 30 -- ci/jenkinsfile/milvus_build_no_ut.groovy | 28 - ci/jenkinsfile/nightly_publish_docker.groovy | 38 -- ci/jenkinsfile/notify.groovy | 15 - ci/jenkinsfile/packaged_milvus.groovy | 44 -- ci/jenkinsfile/packaged_milvus_no_ut.groovy | 26 - ci/jenkinsfile/publish_docker.groovy | 35 -- ci/jenkinsfile/staging_test.groovy | 31 -- .../upload_dev_cluster_test_out.groovy | 14 - ci/jenkinsfile/upload_dev_test_out.groovy | 13 - ci/jenkinsfile/upload_staging_test_out.groovy | 13 - ci/main_jenkinsfile | 396 --------------- ci/main_jenkinsfile_no_ut | 396 --------------- ci/nightly_main_jenkinsfile | 478 ------------------ ci/pod_containers/milvus-engine-build.yaml | 13 - ci/pod_containers/milvus-testframework.yaml | 13 - ci/pod_containers/publish-docker.yaml | 22 - 27 files changed, 1779 deletions(-) delete mode 100644 ci/function/file_transfer.groovy delete mode 100644 ci/jenkinsfile/cleanup_dev.groovy delete mode 100644 ci/jenkinsfile/cleanup_staging.groovy delete mode 100644 ci/jenkinsfile/cluster_cleanup_dev.groovy delete mode 100644 ci/jenkinsfile/cluster_deploy2dev.groovy delete mode 100644 ci/jenkinsfile/cluster_dev_test.groovy delete mode 100644 ci/jenkinsfile/deploy2dev.groovy delete mode 100644 ci/jenkinsfile/deploy2staging.groovy delete mode 100644 ci/jenkinsfile/dev_test.groovy delete mode 100644 ci/jenkinsfile/dev_test_all.groovy delete mode 100644 ci/jenkinsfile/milvus_build.groovy delete mode 100644 ci/jenkinsfile/milvus_build_no_ut.groovy delete mode 100644 ci/jenkinsfile/nightly_publish_docker.groovy delete mode 100644 ci/jenkinsfile/notify.groovy delete mode 100644 ci/jenkinsfile/packaged_milvus.groovy delete mode 100644 ci/jenkinsfile/packaged_milvus_no_ut.groovy delete mode 100644 ci/jenkinsfile/publish_docker.groovy delete mode 100644 ci/jenkinsfile/staging_test.groovy delete mode 100644 ci/jenkinsfile/upload_dev_cluster_test_out.groovy delete mode 100644 ci/jenkinsfile/upload_dev_test_out.groovy delete mode 100644 ci/jenkinsfile/upload_staging_test_out.groovy delete mode 100644 ci/main_jenkinsfile delete mode 100644 ci/main_jenkinsfile_no_ut delete mode 100644 ci/nightly_main_jenkinsfile delete mode 100644 ci/pod_containers/milvus-engine-build.yaml delete mode 100644 ci/pod_containers/milvus-testframework.yaml delete mode 100644 ci/pod_containers/publish-docker.yaml diff --git a/ci/function/file_transfer.groovy b/ci/function/file_transfer.groovy deleted file mode 100644 index bebae14832..0000000000 --- a/ci/function/file_transfer.groovy +++ /dev/null @@ -1,10 +0,0 @@ -def FileTransfer (sourceFiles, remoteDirectory, remoteIP, protocol = "ftp", makeEmptyDirs = true) { - if (protocol == "ftp") { - ftpPublisher masterNodeName: '', paramPublish: [parameterName: ''], alwaysPublishFromMaster: false, continueOnError: false, failOnError: true, publishers: [ - [configName: "${remoteIP}", transfers: [ - [asciiMode: false, cleanRemote: false, excludes: '', flatten: false, makeEmptyDirs: "${makeEmptyDirs}", noDefaultExcludes: false, patternSeparator: '[, ]+', remoteDirectory: "${remoteDirectory}", remoteDirectorySDF: false, removePrefix: '', sourceFiles: "${sourceFiles}"]], usePromotionTimestamp: true, useWorkspaceInPromotion: false, verbose: true - ] - ] - } -} -return this diff --git a/ci/jenkinsfile/cleanup_dev.groovy b/ci/jenkinsfile/cleanup_dev.groovy deleted file mode 100644 index 2e9332fa6e..0000000000 --- a/ci/jenkinsfile/cleanup_dev.groovy +++ /dev/null @@ -1,13 +0,0 @@ -try { - def result = sh script: "helm status ${env.JOB_NAME}-${env.BUILD_NUMBER}", returnStatus: true - if (!result) { - sh "helm del --purge ${env.JOB_NAME}-${env.BUILD_NUMBER}" - } -} catch (exc) { - def result = sh script: "helm status ${env.JOB_NAME}-${env.BUILD_NUMBER}", returnStatus: true - if (!result) { - sh "helm del --purge ${env.JOB_NAME}-${env.BUILD_NUMBER}" - } - throw exc -} - diff --git a/ci/jenkinsfile/cleanup_staging.groovy b/ci/jenkinsfile/cleanup_staging.groovy deleted file mode 100644 index 2e9332fa6e..0000000000 --- a/ci/jenkinsfile/cleanup_staging.groovy +++ /dev/null @@ -1,13 +0,0 @@ -try { - def result = sh script: "helm status ${env.JOB_NAME}-${env.BUILD_NUMBER}", returnStatus: true - if (!result) { - sh "helm del --purge ${env.JOB_NAME}-${env.BUILD_NUMBER}" - } -} catch (exc) { - def result = sh script: "helm status ${env.JOB_NAME}-${env.BUILD_NUMBER}", returnStatus: true - if (!result) { - sh "helm del --purge ${env.JOB_NAME}-${env.BUILD_NUMBER}" - } - throw exc -} - diff --git a/ci/jenkinsfile/cluster_cleanup_dev.groovy b/ci/jenkinsfile/cluster_cleanup_dev.groovy deleted file mode 100644 index e57988fefe..0000000000 --- a/ci/jenkinsfile/cluster_cleanup_dev.groovy +++ /dev/null @@ -1,13 +0,0 @@ -try { - def result = sh script: "helm status ${env.JOB_NAME}-${env.BUILD_NUMBER}-cluster", returnStatus: true - if (!result) { - sh "helm del --purge ${env.JOB_NAME}-${env.BUILD_NUMBER}-cluster" - } -} catch (exc) { - def result = sh script: "helm status ${env.JOB_NAME}-${env.BUILD_NUMBER}-cluster", returnStatus: true - if (!result) { - sh "helm del --purge ${env.JOB_NAME}-${env.BUILD_NUMBER}-cluster" - } - throw exc -} - diff --git a/ci/jenkinsfile/cluster_deploy2dev.groovy b/ci/jenkinsfile/cluster_deploy2dev.groovy deleted file mode 100644 index 7f2a584256..0000000000 --- a/ci/jenkinsfile/cluster_deploy2dev.groovy +++ /dev/null @@ -1,24 +0,0 @@ -try { - sh 'helm init --client-only --skip-refresh --stable-repo-url https://kubernetes.oss-cn-hangzhou.aliyuncs.com/charts' - sh 'helm repo add milvus https://registry.zilliz.com/chartrepo/milvus' - sh 'helm repo update' - dir ("milvus-helm") { - checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:megasearch/milvus-helm.git", name: 'origin', refspec: "+refs/heads/${SEMVER}:refs/remotes/origin/${SEMVER}"]]]) - dir ("milvus/milvus-cluster") { - sh "helm install --wait --timeout 300 --set roServers.image.tag=${DOCKER_VERSION} --set woServers.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP -f ci/values.yaml --name ${env.JOB_NAME}-${env.BUILD_NUMBER}-cluster --namespace milvus-cluster --version 0.5.0 . " - } - } - /* - timeout(time: 2, unit: 'MINUTES') { - waitUntil { - def result = sh script: "nc -z -w 3 ${env.JOB_NAME}-${env.BUILD_NUMBER}-cluster-milvus-cluster-proxy.milvus-cluster.svc.cluster.local 19530", returnStatus: true - return !result - } - } - */ -} catch (exc) { - echo 'Helm running failed!' - sh "helm del --purge ${env.JOB_NAME}-${env.BUILD_NUMBER}-cluster" - throw exc -} - diff --git a/ci/jenkinsfile/cluster_dev_test.groovy b/ci/jenkinsfile/cluster_dev_test.groovy deleted file mode 100644 index 4a15b926cf..0000000000 --- a/ci/jenkinsfile/cluster_dev_test.groovy +++ /dev/null @@ -1,12 +0,0 @@ -timeout(time: 25, unit: 'MINUTES') { - try { - dir ("${PROJECT_NAME}_test") { - checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:Test/milvus_test.git", name: 'origin', refspec: "+refs/heads/${SEMVER}:refs/remotes/origin/${SEMVER}"]]]) - sh 'python3 -m pip install -r requirements_cluster.txt' - sh "pytest . --alluredir=cluster_test_out --ip ${env.JOB_NAME}-${env.BUILD_NUMBER}-cluster-milvus-cluster-proxy.milvus-cluster.svc.cluster.local" - } - } catch (exc) { - echo 'Milvus Cluster Test Failed !' - throw exc - } -} diff --git a/ci/jenkinsfile/deploy2dev.groovy b/ci/jenkinsfile/deploy2dev.groovy deleted file mode 100644 index b00c8fa335..0000000000 --- a/ci/jenkinsfile/deploy2dev.groovy +++ /dev/null @@ -1,16 +0,0 @@ -try { - sh 'helm init --client-only --skip-refresh --stable-repo-url https://kubernetes.oss-cn-hangzhou.aliyuncs.com/charts' - sh 'helm repo add milvus https://registry.zilliz.com/chartrepo/milvus' - sh 'helm repo update' - dir ("milvus-helm") { - checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:megasearch/milvus-helm.git", name: 'origin', refspec: "+refs/heads/${SEMVER}:refs/remotes/origin/${SEMVER}"]]]) - dir ("milvus/milvus-gpu") { - sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.JOB_NAME}-${env.BUILD_NUMBER} -f ci/values.yaml --namespace milvus-1 --version 0.5.0 ." - } - } -} catch (exc) { - echo 'Helm running failed!' - sh "helm del --purge ${env.JOB_NAME}-${env.BUILD_NUMBER}" - throw exc -} - diff --git a/ci/jenkinsfile/deploy2staging.groovy b/ci/jenkinsfile/deploy2staging.groovy deleted file mode 100644 index 42ccfda71a..0000000000 --- a/ci/jenkinsfile/deploy2staging.groovy +++ /dev/null @@ -1,16 +0,0 @@ -try { - sh 'helm init --client-only --skip-refresh --stable-repo-url https://kubernetes.oss-cn-hangzhou.aliyuncs.com/charts' - sh 'helm repo add milvus https://registry.zilliz.com/chartrepo/milvus' - sh 'helm repo update' - dir ("milvus-helm") { - checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:megasearch/milvus-helm.git", name: 'origin', refspec: "+refs/heads/${SEMVER}:refs/remotes/origin/${SEMVER}"]]]) - dir ("milvus/milvus-gpu") { - sh "helm install --wait --timeout 300 --set engine.image.repository=\"zilliz.azurecr.cn/milvus/engine\" --set engine.image.tag=${DOCKER_VERSION} --set expose.type=loadBalancer --name ${env.JOB_NAME}-${env.BUILD_NUMBER} -f ci/values.yaml --namespace milvus-1 --version 0.5.0 ." - } - } -} catch (exc) { - echo 'Helm running failed!' - sh "helm del --purge ${env.JOB_NAME}-${env.BUILD_NUMBER}" - throw exc -} - diff --git a/ci/jenkinsfile/dev_test.groovy b/ci/jenkinsfile/dev_test.groovy deleted file mode 100644 index 9b265ac401..0000000000 --- a/ci/jenkinsfile/dev_test.groovy +++ /dev/null @@ -1,28 +0,0 @@ -timeout(time: 30, unit: 'MINUTES') { - try { - dir ("${PROJECT_NAME}_test") { - checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:Test/milvus_test.git", name: 'origin', refspec: "+refs/heads/${SEMVER}:refs/remotes/origin/${SEMVER}"]]]) - sh 'python3 -m pip install -r requirements.txt -i http://pypi.douban.com/simple --trusted-host pypi.douban.com' - sh "pytest . --alluredir=\"test_out/dev/single/sqlite\" --level=1 --ip ${env.JOB_NAME}-${env.BUILD_NUMBER}-milvus-gpu-engine.milvus-1.svc.cluster.local --internal=true" - } - // mysql database backend test - load "${env.WORKSPACE}/ci/jenkinsfile/cleanup_dev.groovy" - - if (!fileExists('milvus-helm')) { - dir ("milvus-helm") { - checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:megasearch/milvus-helm.git", name: 'origin', refspec: "+refs/heads/${SEMVER}:refs/remotes/origin/${SEMVER}"]]]) - } - } - dir ("milvus-helm") { - dir ("milvus/milvus-gpu") { - sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.JOB_NAME}-${env.BUILD_NUMBER} -f ci/db_backend/mysql_values.yaml --namespace milvus-2 --version 0.5.0 ." - } - } - dir ("${PROJECT_NAME}_test") { - sh "pytest . --alluredir=\"test_out/dev/single/mysql\" --level=1 --ip ${env.JOB_NAME}-${env.BUILD_NUMBER}-milvus-gpu-engine.milvus-2.svc.cluster.local --internal=true" - } - } catch (exc) { - echo 'Milvus Test Failed !' - throw exc - } -} diff --git a/ci/jenkinsfile/dev_test_all.groovy b/ci/jenkinsfile/dev_test_all.groovy deleted file mode 100644 index b82d995d8c..0000000000 --- a/ci/jenkinsfile/dev_test_all.groovy +++ /dev/null @@ -1,29 +0,0 @@ -timeout(time: 60, unit: 'MINUTES') { - try { - dir ("${PROJECT_NAME}_test") { - checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:Test/milvus_test.git", name: 'origin', refspec: "+refs/heads/${SEMVER}:refs/remotes/origin/${SEMVER}"]]]) - sh 'python3 -m pip install -r requirements.txt -i http://pypi.douban.com/simple --trusted-host pypi.douban.com' - sh "pytest . --alluredir=\"test_out/dev/single/sqlite\" --ip ${env.JOB_NAME}-${env.BUILD_NUMBER}-milvus-gpu-engine.milvus-1.svc.cluster.local --internal=true" - } - - // mysql database backend test - load "${env.WORKSPACE}/ci/jenkinsfile/cleanup_dev.groovy" - - if (!fileExists('milvus-helm')) { - dir ("milvus-helm") { - checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:megasearch/milvus-helm.git", name: 'origin', refspec: "+refs/heads/${SEMVER}:refs/remotes/origin/${SEMVER}"]]]) - } - } - dir ("milvus-helm") { - dir ("milvus/milvus-gpu") { - sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.JOB_NAME}-${env.BUILD_NUMBER} -f ci/db_backend/mysql_values.yaml --namespace milvus-2 --version 0.4.0 ." - } - } - dir ("${PROJECT_NAME}_test") { - sh "pytest . --alluredir=\"test_out/dev/single/mysql\" --ip ${env.JOB_NAME}-${env.BUILD_NUMBER}-milvus-gpu-engine.milvus-2.svc.cluster.local --internal=true" - } - } catch (exc) { - echo 'Milvus Test Failed !' - throw exc - } -} diff --git a/ci/jenkinsfile/milvus_build.groovy b/ci/jenkinsfile/milvus_build.groovy deleted file mode 100644 index 27e50c64c4..0000000000 --- a/ci/jenkinsfile/milvus_build.groovy +++ /dev/null @@ -1,30 +0,0 @@ -container('milvus-build-env') { - timeout(time: 120, unit: 'MINUTES') { - gitlabCommitStatus(name: 'Build Engine') { - dir ("milvus_engine") { - try { - checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [[$class: 'SubmoduleOption',disableSubmodules: false,parentCredentials: true,recursiveSubmodules: true,reference: '',trackingSubmodules: false]], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:megasearch/milvus.git", name: 'origin', refspec: "+refs/heads/${SEMVER}:refs/remotes/origin/${SEMVER}"]]]) - - dir ("core") { - sh "git config --global user.email \"test@zilliz.com\"" - sh "git config --global user.name \"test\"" - withCredentials([usernamePassword(credentialsId: "${params.JFROG_USER}", usernameVariable: 'USERNAME', passwordVariable: 'PASSWORD')]) { - sh "./build.sh -l" - sh "rm -rf cmake_build" - sh "export JFROG_ARTFACTORY_URL='${params.JFROG_ARTFACTORY_URL}' \ - && export JFROG_USER_NAME='${USERNAME}' \ - && export JFROG_PASSWORD='${PASSWORD}' \ - && export FAISS_URL='http://192.168.1.105:6060/jinhai/faiss/-/archive/branch-0.3.0/faiss-branch-0.3.0.tar.gz' \ - && ./build.sh -t ${params.BUILD_TYPE} -d /opt/milvus -j -u -c" - - sh "./coverage.sh -u root -p 123456 -t \$POD_IP" - } - } - } catch (exc) { - updateGitlabCommitStatus name: 'Build Engine', state: 'failed' - throw exc - } - } - } - } -} diff --git a/ci/jenkinsfile/milvus_build_no_ut.groovy b/ci/jenkinsfile/milvus_build_no_ut.groovy deleted file mode 100644 index 3f221b8947..0000000000 --- a/ci/jenkinsfile/milvus_build_no_ut.groovy +++ /dev/null @@ -1,28 +0,0 @@ -container('milvus-build-env') { - timeout(time: 120, unit: 'MINUTES') { - gitlabCommitStatus(name: 'Build Engine') { - dir ("milvus_engine") { - try { - checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [[$class: 'SubmoduleOption',disableSubmodules: false,parentCredentials: true,recursiveSubmodules: true,reference: '',trackingSubmodules: false]], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:megasearch/milvus.git", name: 'origin', refspec: "+refs/heads/${SEMVER}:refs/remotes/origin/${SEMVER}"]]]) - - dir ("core") { - sh "git config --global user.email \"test@zilliz.com\"" - sh "git config --global user.name \"test\"" - withCredentials([usernamePassword(credentialsId: "${params.JFROG_USER}", usernameVariable: 'USERNAME', passwordVariable: 'PASSWORD')]) { - sh "./build.sh -l" - sh "rm -rf cmake_build" - sh "export JFROG_ARTFACTORY_URL='${params.JFROG_ARTFACTORY_URL}' \ - && export JFROG_USER_NAME='${USERNAME}' \ - && export JFROG_PASSWORD='${PASSWORD}' \ - && export FAISS_URL='http://192.168.1.105:6060/jinhai/faiss/-/archive/branch-0.3.0/faiss-branch-0.3.0.tar.gz' \ - && ./build.sh -t ${params.BUILD_TYPE} -j -d /opt/milvus" - } - } - } catch (exc) { - updateGitlabCommitStatus name: 'Build Engine', state: 'failed' - throw exc - } - } - } - } -} diff --git a/ci/jenkinsfile/nightly_publish_docker.groovy b/ci/jenkinsfile/nightly_publish_docker.groovy deleted file mode 100644 index 8c6121bec8..0000000000 --- a/ci/jenkinsfile/nightly_publish_docker.groovy +++ /dev/null @@ -1,38 +0,0 @@ -container('publish-docker') { - timeout(time: 15, unit: 'MINUTES') { - gitlabCommitStatus(name: 'Publish Engine Docker') { - try { - dir ("${PROJECT_NAME}_build") { - checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:build/milvus_build.git", name: 'origin', refspec: "+refs/heads/${SEMVER}:refs/remotes/origin/${SEMVER}"]]]) - dir ("docker/deploy/ubuntu16.04/free_version") { - sh "curl -O -u anonymous: ftp://192.168.1.126/data/${PROJECT_NAME}/engine/${JOB_NAME}-${BUILD_ID}/${PROJECT_NAME}-engine-${PACKAGE_VERSION}.tar.gz" - sh "tar zxvf ${PROJECT_NAME}-engine-${PACKAGE_VERSION}.tar.gz" - try { - def customImage = docker.build("${PROJECT_NAME}/engine:${DOCKER_VERSION}") - docker.withRegistry('https://registry.zilliz.com', "${params.DOCKER_PUBLISH_USER}") { - customImage.push() - } - docker.withRegistry('https://zilliz.azurecr.cn', "${params.AZURE_DOCKER_PUBLISH_USER}") { - customImage.push() - } - if (currentBuild.resultIsBetterOrEqualTo('SUCCESS')) { - updateGitlabCommitStatus name: 'Publish Engine Docker', state: 'success' - echo "Docker Pull Command: docker pull registry.zilliz.com/${PROJECT_NAME}/engine:${DOCKER_VERSION}" - } - } catch (exc) { - updateGitlabCommitStatus name: 'Publish Engine Docker', state: 'canceled' - throw exc - } finally { - sh "docker rmi ${PROJECT_NAME}/engine:${DOCKER_VERSION}" - } - } - } - } catch (exc) { - updateGitlabCommitStatus name: 'Publish Engine Docker', state: 'failed' - echo 'Publish docker failed!' - throw exc - } - } - } -} - diff --git a/ci/jenkinsfile/notify.groovy b/ci/jenkinsfile/notify.groovy deleted file mode 100644 index 0a257b8cd8..0000000000 --- a/ci/jenkinsfile/notify.groovy +++ /dev/null @@ -1,15 +0,0 @@ -def notify() { - if (!currentBuild.resultIsBetterOrEqualTo('SUCCESS')) { - // Send an email only if the build status has changed from green/unstable to red - emailext subject: '$DEFAULT_SUBJECT', - body: '$DEFAULT_CONTENT', - recipientProviders: [ - [$class: 'DevelopersRecipientProvider'], - [$class: 'RequesterRecipientProvider'] - ], - replyTo: '$DEFAULT_REPLYTO', - to: '$DEFAULT_RECIPIENTS' - } -} -return this - diff --git a/ci/jenkinsfile/packaged_milvus.groovy b/ci/jenkinsfile/packaged_milvus.groovy deleted file mode 100644 index 1d30e21910..0000000000 --- a/ci/jenkinsfile/packaged_milvus.groovy +++ /dev/null @@ -1,44 +0,0 @@ -container('milvus-build-env') { - timeout(time: 5, unit: 'MINUTES') { - dir ("milvus_engine") { - dir ("core") { - gitlabCommitStatus(name: 'Packaged Engine') { - if (fileExists('milvus')) { - try { - sh "tar -zcvf ./${PROJECT_NAME}-engine-${PACKAGE_VERSION}.tar.gz ./milvus" - def fileTransfer = load "${env.WORKSPACE}/ci/function/file_transfer.groovy" - fileTransfer.FileTransfer("${PROJECT_NAME}-engine-${PACKAGE_VERSION}.tar.gz", "${PROJECT_NAME}/engine/${JOB_NAME}-${BUILD_ID}", 'nas storage') - if (currentBuild.resultIsBetterOrEqualTo('SUCCESS')) { - echo "Download Milvus Engine Binary Viewer \"http://192.168.1.126:8080/${PROJECT_NAME}/engine/${JOB_NAME}-${BUILD_ID}/${PROJECT_NAME}-engine-${PACKAGE_VERSION}.tar.gz\"" - } - } catch (exc) { - updateGitlabCommitStatus name: 'Packaged Engine', state: 'failed' - throw exc - } - } else { - updateGitlabCommitStatus name: 'Packaged Engine', state: 'failed' - error("Milvus binary directory don't exists!") - } - } - - gitlabCommitStatus(name: 'Packaged Engine lcov') { - if (fileExists('lcov_out')) { - try { - def fileTransfer = load "${env.WORKSPACE}/ci/function/file_transfer.groovy" - fileTransfer.FileTransfer("lcov_out/", "${PROJECT_NAME}/lcov/${JOB_NAME}-${BUILD_ID}", 'nas storage') - if (currentBuild.resultIsBetterOrEqualTo('SUCCESS')) { - echo "Milvus lcov out Viewer \"http://192.168.1.126:8080/${PROJECT_NAME}/lcov/${JOB_NAME}-${BUILD_ID}/lcov_out/\"" - } - } catch (exc) { - updateGitlabCommitStatus name: 'Packaged Engine lcov', state: 'failed' - throw exc - } - } else { - updateGitlabCommitStatus name: 'Packaged Engine lcov', state: 'failed' - error("Milvus lcov out directory don't exists!") - } - } - } - } - } -} diff --git a/ci/jenkinsfile/packaged_milvus_no_ut.groovy b/ci/jenkinsfile/packaged_milvus_no_ut.groovy deleted file mode 100644 index bc68be374a..0000000000 --- a/ci/jenkinsfile/packaged_milvus_no_ut.groovy +++ /dev/null @@ -1,26 +0,0 @@ -container('milvus-build-env') { - timeout(time: 5, unit: 'MINUTES') { - dir ("milvus_engine") { - dir ("core") { - gitlabCommitStatus(name: 'Packaged Engine') { - if (fileExists('milvus')) { - try { - sh "tar -zcvf ./${PROJECT_NAME}-engine-${PACKAGE_VERSION}.tar.gz ./milvus" - def fileTransfer = load "${env.WORKSPACE}/ci/function/file_transfer.groovy" - fileTransfer.FileTransfer("${PROJECT_NAME}-engine-${PACKAGE_VERSION}.tar.gz", "${PROJECT_NAME}/engine/${JOB_NAME}-${BUILD_ID}", 'nas storage') - if (currentBuild.resultIsBetterOrEqualTo('SUCCESS')) { - echo "Download Milvus Engine Binary Viewer \"http://192.168.1.126:8080/${PROJECT_NAME}/engine/${JOB_NAME}-${BUILD_ID}/${PROJECT_NAME}-engine-${PACKAGE_VERSION}.tar.gz\"" - } - } catch (exc) { - updateGitlabCommitStatus name: 'Packaged Engine', state: 'failed' - throw exc - } - } else { - updateGitlabCommitStatus name: 'Packaged Engine', state: 'failed' - error("Milvus binary directory don't exists!") - } - } - } - } - } -} diff --git a/ci/jenkinsfile/publish_docker.groovy b/ci/jenkinsfile/publish_docker.groovy deleted file mode 100644 index ef31eba9a4..0000000000 --- a/ci/jenkinsfile/publish_docker.groovy +++ /dev/null @@ -1,35 +0,0 @@ -container('publish-docker') { - timeout(time: 15, unit: 'MINUTES') { - gitlabCommitStatus(name: 'Publish Engine Docker') { - try { - dir ("${PROJECT_NAME}_build") { - checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:build/milvus_build.git", name: 'origin', refspec: "+refs/heads/${SEMVER}:refs/remotes/origin/${SEMVER}"]]]) - dir ("docker/deploy/ubuntu16.04/free_version") { - sh "curl -O -u anonymous: ftp://192.168.1.126/data/${PROJECT_NAME}/engine/${JOB_NAME}-${BUILD_ID}/${PROJECT_NAME}-engine-${PACKAGE_VERSION}.tar.gz" - sh "tar zxvf ${PROJECT_NAME}-engine-${PACKAGE_VERSION}.tar.gz" - try { - def customImage = docker.build("${PROJECT_NAME}/engine:${DOCKER_VERSION}") - docker.withRegistry('https://registry.zilliz.com', "${params.DOCKER_PUBLISH_USER}") { - customImage.push() - } - if (currentBuild.resultIsBetterOrEqualTo('SUCCESS')) { - updateGitlabCommitStatus name: 'Publish Engine Docker', state: 'success' - echo "Docker Pull Command: docker pull registry.zilliz.com/${PROJECT_NAME}/engine:${DOCKER_VERSION}" - } - } catch (exc) { - updateGitlabCommitStatus name: 'Publish Engine Docker', state: 'canceled' - throw exc - } finally { - sh "docker rmi ${PROJECT_NAME}/engine:${DOCKER_VERSION}" - } - } - } - } catch (exc) { - updateGitlabCommitStatus name: 'Publish Engine Docker', state: 'failed' - echo 'Publish docker failed!' - throw exc - } - } - } -} - diff --git a/ci/jenkinsfile/staging_test.groovy b/ci/jenkinsfile/staging_test.groovy deleted file mode 100644 index dcf1787103..0000000000 --- a/ci/jenkinsfile/staging_test.groovy +++ /dev/null @@ -1,31 +0,0 @@ -timeout(time: 40, unit: 'MINUTES') { - try { - dir ("${PROJECT_NAME}_test") { - checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:Test/milvus_test.git", name: 'origin', refspec: "+refs/heads/${SEMVER}:refs/remotes/origin/${SEMVER}"]]]) - sh 'python3 -m pip install -r requirements.txt' - def service_ip = sh (script: "kubectl get svc --namespace milvus-1 ${env.JOB_NAME}-${env.BUILD_NUMBER}-milvus-gpu-engine --template \"{{range .status.loadBalancer.ingress}}{{.ip}}{{end}}\"",returnStdout: true).trim() - sh "pytest . --alluredir=\"test_out/staging/single/sqlite\" --ip ${service_ip}" - } - - // mysql database backend test - load "${env.WORKSPACE}/ci/jenkinsfile/cleanup_staging.groovy" - - if (!fileExists('milvus-helm')) { - dir ("milvus-helm") { - checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:megasearch/milvus-helm.git", name: 'origin', refspec: "+refs/heads/${SEMVER}:refs/remotes/origin/${SEMVER}"]]]) - } - } - dir ("milvus-helm") { - dir ("milvus/milvus-gpu") { - sh "helm install --wait --timeout 300 --set engine.image.repository=\"zilliz.azurecr.cn/milvus/engine\" --set engine.image.tag=${DOCKER_VERSION} --set expose.type=loadBalancer --name ${env.JOB_NAME}-${env.BUILD_NUMBER} -f ci/db_backend/mysql_values.yaml --namespace milvus-2 --version 0.5.0 ." - } - } - dir ("${PROJECT_NAME}_test") { - def service_ip = sh (script: "kubectl get svc --namespace milvus-2 ${env.JOB_NAME}-${env.BUILD_NUMBER}-milvus-gpu-engine --template \"{{range .status.loadBalancer.ingress}}{{.ip}}{{end}}\"",returnStdout: true).trim() - sh "pytest . --alluredir=\"test_out/staging/single/mysql\" --ip ${service_ip}" - } - } catch (exc) { - echo 'Milvus Test Failed !' - throw exc - } -} diff --git a/ci/jenkinsfile/upload_dev_cluster_test_out.groovy b/ci/jenkinsfile/upload_dev_cluster_test_out.groovy deleted file mode 100644 index 6bbd8a649f..0000000000 --- a/ci/jenkinsfile/upload_dev_cluster_test_out.groovy +++ /dev/null @@ -1,14 +0,0 @@ -timeout(time: 5, unit: 'MINUTES') { - dir ("${PROJECT_NAME}_test") { - if (fileExists('cluster_test_out')) { - def fileTransfer = load "${env.WORKSPACE}/ci/function/file_transfer.groovy" - fileTransfer.FileTransfer("cluster_test_out/", "${PROJECT_NAME}/test/${JOB_NAME}-${BUILD_ID}", 'nas storage') - if (currentBuild.resultIsBetterOrEqualTo('SUCCESS')) { - echo "Milvus Dev Test Out Viewer \"ftp://192.168.1.126/data/${PROJECT_NAME}/test/${JOB_NAME}-${BUILD_ID}\"" - } - } else { - error("Milvus Dev Test Out directory don't exists!") - } - } -} - diff --git a/ci/jenkinsfile/upload_dev_test_out.groovy b/ci/jenkinsfile/upload_dev_test_out.groovy deleted file mode 100644 index 017b887334..0000000000 --- a/ci/jenkinsfile/upload_dev_test_out.groovy +++ /dev/null @@ -1,13 +0,0 @@ -timeout(time: 5, unit: 'MINUTES') { - dir ("${PROJECT_NAME}_test") { - if (fileExists('test_out/dev')) { - def fileTransfer = load "${env.WORKSPACE}/ci/function/file_transfer.groovy" - fileTransfer.FileTransfer("test_out/dev/", "${PROJECT_NAME}/test/${JOB_NAME}-${BUILD_ID}", 'nas storage') - if (currentBuild.resultIsBetterOrEqualTo('SUCCESS')) { - echo "Milvus Dev Test Out Viewer \"ftp://192.168.1.126/data/${PROJECT_NAME}/test/${JOB_NAME}-${BUILD_ID}\"" - } - } else { - error("Milvus Dev Test Out directory don't exists!") - } - } -} diff --git a/ci/jenkinsfile/upload_staging_test_out.groovy b/ci/jenkinsfile/upload_staging_test_out.groovy deleted file mode 100644 index 1f1e66ab1b..0000000000 --- a/ci/jenkinsfile/upload_staging_test_out.groovy +++ /dev/null @@ -1,13 +0,0 @@ -timeout(time: 5, unit: 'MINUTES') { - dir ("${PROJECT_NAME}_test") { - if (fileExists('test_out/staging')) { - def fileTransfer = load "${env.WORKSPACE}/ci/function/file_transfer.groovy" - fileTransfer.FileTransfer("test_out/staging/", "${PROJECT_NAME}/test/${JOB_NAME}-${BUILD_ID}", 'nas storage') - if (currentBuild.resultIsBetterOrEqualTo('SUCCESS')) { - echo "Milvus Dev Test Out Viewer \"ftp://192.168.1.126/data/${PROJECT_NAME}/test/${JOB_NAME}-${BUILD_ID}\"" - } - } else { - error("Milvus Dev Test Out directory don't exists!") - } - } -} diff --git a/ci/main_jenkinsfile b/ci/main_jenkinsfile deleted file mode 100644 index 0c3fc32e5b..0000000000 --- a/ci/main_jenkinsfile +++ /dev/null @@ -1,396 +0,0 @@ -pipeline { - agent none - - options { - timestamps() - } - - environment { - PROJECT_NAME = "milvus" - LOWER_BUILD_TYPE = BUILD_TYPE.toLowerCase() - SEMVER = "${env.gitlabSourceBranch == null ? params.ENGINE_BRANCH.substring(params.ENGINE_BRANCH.lastIndexOf('/') + 1) : env.gitlabSourceBranch}" - GITLAB_AFTER_COMMIT = "${env.gitlabAfter == null ? null : env.gitlabAfter}" - SUFFIX_VERSION_NAME = "${env.gitlabAfter == null ? null : env.gitlabAfter.substring(0, 6)}" - DOCKER_VERSION_STR = "${env.gitlabAfter == null ? "${SEMVER}-${LOWER_BUILD_TYPE}" : "${SEMVER}-${LOWER_BUILD_TYPE}-${SUFFIX_VERSION_NAME}"}" - } - - stages { - stage("Ubuntu 16.04") { - environment { - PACKAGE_VERSION = VersionNumber([ - versionNumberString : '${SEMVER}-${LOWER_BUILD_TYPE}-${BUILD_DATE_FORMATTED, "yyyyMMdd"}' - ]); - - DOCKER_VERSION = VersionNumber([ - versionNumberString : '${DOCKER_VERSION_STR}' - ]); - } - - stages { - stage("Run Build") { - agent { - kubernetes { - cloud 'build-kubernetes' - label 'build' - defaultContainer 'jnlp' - yaml """ -apiVersion: v1 -kind: Pod -metadata: - name: milvus-build-env - labels: - app: milvus - componet: build-env -spec: - containers: - - name: milvus-build-env - image: registry.zilliz.com/milvus/milvus-build-env:v0.13 - env: - - name: POD_IP - valueFrom: - fieldRef: - fieldPath: status.podIP - command: - - cat - tty: true - resources: - limits: - memory: "28Gi" - cpu: "10.0" - nvidia.com/gpu: 1 - requests: - memory: "14Gi" - cpu: "5.0" - - name: milvus-mysql - image: mysql:5.6 - env: - - name: MYSQL_ROOT_PASSWORD - value: 123456 - ports: - - containerPort: 3306 - name: mysql -""" - } - } - stages { - stage('Build') { - steps { - gitlabCommitStatus(name: 'Build') { - script { - load "${env.WORKSPACE}/ci/jenkinsfile/milvus_build.groovy" - load "${env.WORKSPACE}/ci/jenkinsfile/packaged_milvus.groovy" - } - } - } - } - } - post { - aborted { - script { - updateGitlabCommitStatus name: 'Build', state: 'canceled' - echo "Milvus Build aborted !" - } - } - - failure { - script { - updateGitlabCommitStatus name: 'Build', state: 'failed' - echo "Milvus Build failure !" - } - } - } - } - - stage("Publish docker and helm") { - agent { - kubernetes { - label 'publish' - defaultContainer 'jnlp' - yaml """ -apiVersion: v1 -kind: Pod -metadata: - labels: - app: publish - componet: docker -spec: - containers: - - name: publish-docker - image: registry.zilliz.com/library/zilliz_docker:v1.0.0 - securityContext: - privileged: true - command: - - cat - tty: true - volumeMounts: - - name: docker-sock - mountPath: /var/run/docker.sock - volumes: - - name: docker-sock - hostPath: - path: /var/run/docker.sock -""" - } - } - stages { - stage('Publish Docker') { - steps { - gitlabCommitStatus(name: 'Publish Docker') { - script { - load "${env.WORKSPACE}/ci/jenkinsfile/publish_docker.groovy" - } - } - } - } - } - post { - aborted { - script { - updateGitlabCommitStatus name: 'Publish Docker', state: 'canceled' - echo "Milvus Publish Docker aborted !" - } - } - - failure { - script { - updateGitlabCommitStatus name: 'Publish Docker', state: 'failed' - echo "Milvus Publish Docker failure !" - } - } - } - } - - stage("Deploy to Development") { - parallel { - stage("Single Node") { - agent { - kubernetes { - label 'dev-test' - defaultContainer 'jnlp' - yaml """ -apiVersion: v1 -kind: Pod -metadata: - labels: - app: milvus - componet: test -spec: - containers: - - name: milvus-testframework - image: registry.zilliz.com/milvus/milvus-test:v0.2 - command: - - cat - tty: true - volumeMounts: - - name: kubeconf - mountPath: /root/.kube/ - readOnly: true - volumes: - - name: kubeconf - secret: - secretName: test-cluster-config -""" - } - } - - stages { - stage("Deploy to Dev") { - steps { - gitlabCommitStatus(name: 'Deloy to Dev') { - container('milvus-testframework') { - script { - load "${env.WORKSPACE}/ci/jenkinsfile/deploy2dev.groovy" - } - } - } - } - } - stage("Dev Test") { - steps { - gitlabCommitStatus(name: 'Deloy Test') { - container('milvus-testframework') { - script { - load "${env.WORKSPACE}/ci/jenkinsfile/dev_test.groovy" - load "${env.WORKSPACE}/ci/jenkinsfile/upload_dev_test_out.groovy" - } - } - } - } - } - stage ("Cleanup Dev") { - steps { - gitlabCommitStatus(name: 'Cleanup Dev') { - container('milvus-testframework') { - script { - load "${env.WORKSPACE}/ci/jenkinsfile/cleanup_dev.groovy" - } - } - } - } - } - } - post { - always { - container('milvus-testframework') { - script { - load "${env.WORKSPACE}/ci/jenkinsfile/cleanup_dev.groovy" - } - } - } - success { - script { - echo "Milvus Single Node CI/CD success !" - } - } - aborted { - script { - echo "Milvus Single Node CI/CD aborted !" - } - } - failure { - script { - echo "Milvus Single Node CI/CD failure !" - } - } - } - } - -// stage("Cluster") { -// agent { -// kubernetes { -// label 'dev-test' -// defaultContainer 'jnlp' -// yaml """ -// apiVersion: v1 -// kind: Pod -// metadata: -// labels: -// app: milvus -// componet: test -// spec: -// containers: -// - name: milvus-testframework -// image: registry.zilliz.com/milvus/milvus-test:v0.2 -// command: -// - cat -// tty: true -// volumeMounts: -// - name: kubeconf -// mountPath: /root/.kube/ -// readOnly: true -// volumes: -// - name: kubeconf -// secret: -// secretName: test-cluster-config -// """ -// } -// } -// stages { -// stage("Deploy to Dev") { -// steps { -// gitlabCommitStatus(name: 'Deloy to Dev') { -// container('milvus-testframework') { -// script { -// load "${env.WORKSPACE}/ci/jenkinsfile/cluster_deploy2dev.groovy" -// } -// } -// } -// } -// } -// stage("Dev Test") { -// steps { -// gitlabCommitStatus(name: 'Deloy Test') { -// container('milvus-testframework') { -// script { -// load "${env.WORKSPACE}/ci/jenkinsfile/cluster_dev_test.groovy" -// load "${env.WORKSPACE}/ci/jenkinsfile/upload_dev_cluster_test_out.groovy" -// } -// } -// } -// } -// } -// stage ("Cleanup Dev") { -// steps { -// gitlabCommitStatus(name: 'Cleanup Dev') { -// container('milvus-testframework') { -// script { -// load "${env.WORKSPACE}/ci/jenkinsfile/cluster_cleanup_dev.groovy" -// } -// } -// } -// } -// } -// } -// post { -// always { -// container('milvus-testframework') { -// script { -// load "${env.WORKSPACE}/ci/jenkinsfile/cluster_cleanup_dev.groovy" -// } -// } -// } -// success { -// script { -// echo "Milvus Cluster CI/CD success !" -// } -// } -// aborted { -// script { -// echo "Milvus Cluster CI/CD aborted !" -// } -// } -// failure { -// script { -// echo "Milvus Cluster CI/CD failure !" -// } -// } -// } -// } - } - } - } - } - } - - post { - always { - script { - if (env.gitlabAfter != null) { - if (!currentBuild.resultIsBetterOrEqualTo('SUCCESS')) { - // Send an email only if the build status has changed from green/unstable to red - emailext subject: '$DEFAULT_SUBJECT', - body: '$DEFAULT_CONTENT', - recipientProviders: [ - [$class: 'DevelopersRecipientProvider'], - [$class: 'RequesterRecipientProvider'] - ], - replyTo: '$DEFAULT_REPLYTO', - to: '$DEFAULT_RECIPIENTS' - } - } - } - } - - success { - script { - updateGitlabCommitStatus name: 'CI/CD', state: 'success' - echo "Milvus CI/CD success !" - } - } - - aborted { - script { - updateGitlabCommitStatus name: 'CI/CD', state: 'canceled' - echo "Milvus CI/CD aborted !" - } - } - - failure { - script { - updateGitlabCommitStatus name: 'CI/CD', state: 'failed' - echo "Milvus CI/CD failure !" - } - } - } -} - diff --git a/ci/main_jenkinsfile_no_ut b/ci/main_jenkinsfile_no_ut deleted file mode 100644 index e8d7dae75a..0000000000 --- a/ci/main_jenkinsfile_no_ut +++ /dev/null @@ -1,396 +0,0 @@ -pipeline { - agent none - - options { - timestamps() - } - - environment { - PROJECT_NAME = "milvus" - LOWER_BUILD_TYPE = BUILD_TYPE.toLowerCase() - SEMVER = "${env.gitlabSourceBranch == null ? params.ENGINE_BRANCH.substring(params.ENGINE_BRANCH.lastIndexOf('/') + 1) : env.gitlabSourceBranch}" - GITLAB_AFTER_COMMIT = "${env.gitlabAfter == null ? null : env.gitlabAfter}" - SUFFIX_VERSION_NAME = "${env.gitlabAfter == null ? null : env.gitlabAfter.substring(0, 6)}" - DOCKER_VERSION_STR = "${env.gitlabAfter == null ? "${SEMVER}-${LOWER_BUILD_TYPE}" : "${SEMVER}-${LOWER_BUILD_TYPE}-${SUFFIX_VERSION_NAME}"}" - } - - stages { - stage("Ubuntu 16.04") { - environment { - PACKAGE_VERSION = VersionNumber([ - versionNumberString : '${SEMVER}-${LOWER_BUILD_TYPE}-${BUILD_DATE_FORMATTED, "yyyyMMdd"}' - ]); - - DOCKER_VERSION = VersionNumber([ - versionNumberString : '${DOCKER_VERSION_STR}' - ]); - } - - stages { - stage("Run Build") { - agent { - kubernetes { - cloud 'build-kubernetes' - label 'build' - defaultContainer 'jnlp' - yaml """ -apiVersion: v1 -kind: Pod -metadata: - name: milvus-build-env - labels: - app: milvus - componet: build-env -spec: - containers: - - name: milvus-build-env - image: registry.zilliz.com/milvus/milvus-build-env:v0.13 - env: - - name: POD_IP - valueFrom: - fieldRef: - fieldPath: status.podIP - command: - - cat - tty: true - resources: - limits: - memory: "28Gi" - cpu: "10.0" - nvidia.com/gpu: 1 - requests: - memory: "14Gi" - cpu: "5.0" - - name: milvus-mysql - image: mysql:5.6 - env: - - name: MYSQL_ROOT_PASSWORD - value: 123456 - ports: - - containerPort: 3306 - name: mysql -""" - } - } - stages { - stage('Build') { - steps { - gitlabCommitStatus(name: 'Build') { - script { - load "${env.WORKSPACE}/ci/jenkinsfile/milvus_build_no_ut.groovy" - load "${env.WORKSPACE}/ci/jenkinsfile/packaged_milvus_no_ut.groovy" - } - } - } - } - } - post { - aborted { - script { - updateGitlabCommitStatus name: 'Build', state: 'canceled' - echo "Milvus Build aborted !" - } - } - - failure { - script { - updateGitlabCommitStatus name: 'Build', state: 'failed' - echo "Milvus Build failure !" - } - } - } - } - - stage("Publish docker and helm") { - agent { - kubernetes { - label 'publish' - defaultContainer 'jnlp' - yaml """ -apiVersion: v1 -kind: Pod -metadata: - labels: - app: publish - componet: docker -spec: - containers: - - name: publish-docker - image: registry.zilliz.com/library/zilliz_docker:v1.0.0 - securityContext: - privileged: true - command: - - cat - tty: true - volumeMounts: - - name: docker-sock - mountPath: /var/run/docker.sock - volumes: - - name: docker-sock - hostPath: - path: /var/run/docker.sock -""" - } - } - stages { - stage('Publish Docker') { - steps { - gitlabCommitStatus(name: 'Publish Docker') { - script { - load "${env.WORKSPACE}/ci/jenkinsfile/publish_docker.groovy" - } - } - } - } - } - post { - aborted { - script { - updateGitlabCommitStatus name: 'Publish Docker', state: 'canceled' - echo "Milvus Publish Docker aborted !" - } - } - - failure { - script { - updateGitlabCommitStatus name: 'Publish Docker', state: 'failed' - echo "Milvus Publish Docker failure !" - } - } - } - } - - stage("Deploy to Development") { - parallel { - stage("Single Node") { - agent { - kubernetes { - label 'dev-test' - defaultContainer 'jnlp' - yaml """ -apiVersion: v1 -kind: Pod -metadata: - labels: - app: milvus - componet: test -spec: - containers: - - name: milvus-testframework - image: registry.zilliz.com/milvus/milvus-test:v0.2 - command: - - cat - tty: true - volumeMounts: - - name: kubeconf - mountPath: /root/.kube/ - readOnly: true - volumes: - - name: kubeconf - secret: - secretName: test-cluster-config -""" - } - } - - stages { - stage("Deploy to Dev") { - steps { - gitlabCommitStatus(name: 'Deloy to Dev') { - container('milvus-testframework') { - script { - load "${env.WORKSPACE}/ci/jenkinsfile/deploy2dev.groovy" - } - } - } - } - } - stage("Dev Test") { - steps { - gitlabCommitStatus(name: 'Deloy Test') { - container('milvus-testframework') { - script { - load "${env.WORKSPACE}/ci/jenkinsfile/dev_test.groovy" - load "${env.WORKSPACE}/ci/jenkinsfile/upload_dev_test_out.groovy" - } - } - } - } - } - stage ("Cleanup Dev") { - steps { - gitlabCommitStatus(name: 'Cleanup Dev') { - container('milvus-testframework') { - script { - load "${env.WORKSPACE}/ci/jenkinsfile/cleanup_dev.groovy" - } - } - } - } - } - } - post { - always { - container('milvus-testframework') { - script { - load "${env.WORKSPACE}/ci/jenkinsfile/cleanup_dev.groovy" - } - } - } - success { - script { - echo "Milvus Single Node CI/CD success !" - } - } - aborted { - script { - echo "Milvus Single Node CI/CD aborted !" - } - } - failure { - script { - echo "Milvus Single Node CI/CD failure !" - } - } - } - } - -// stage("Cluster") { -// agent { -// kubernetes { -// label 'dev-test' -// defaultContainer 'jnlp' -// yaml """ -// apiVersion: v1 -// kind: Pod -// metadata: -// labels: -// app: milvus -// componet: test -// spec: -// containers: -// - name: milvus-testframework -// image: registry.zilliz.com/milvus/milvus-test:v0.2 -// command: -// - cat -// tty: true -// volumeMounts: -// - name: kubeconf -// mountPath: /root/.kube/ -// readOnly: true -// volumes: -// - name: kubeconf -// secret: -// secretName: test-cluster-config -// """ -// } -// } -// stages { -// stage("Deploy to Dev") { -// steps { -// gitlabCommitStatus(name: 'Deloy to Dev') { -// container('milvus-testframework') { -// script { -// load "${env.WORKSPACE}/ci/jenkinsfile/cluster_deploy2dev.groovy" -// } -// } -// } -// } -// } -// stage("Dev Test") { -// steps { -// gitlabCommitStatus(name: 'Deloy Test') { -// container('milvus-testframework') { -// script { -// load "${env.WORKSPACE}/ci/jenkinsfile/cluster_dev_test.groovy" -// load "${env.WORKSPACE}/ci/jenkinsfile/upload_dev_cluster_test_out.groovy" -// } -// } -// } -// } -// } -// stage ("Cleanup Dev") { -// steps { -// gitlabCommitStatus(name: 'Cleanup Dev') { -// container('milvus-testframework') { -// script { -// load "${env.WORKSPACE}/ci/jenkinsfile/cluster_cleanup_dev.groovy" -// } -// } -// } -// } -// } -// } -// post { -// always { -// container('milvus-testframework') { -// script { -// load "${env.WORKSPACE}/ci/jenkinsfile/cluster_cleanup_dev.groovy" -// } -// } -// } -// success { -// script { -// echo "Milvus Cluster CI/CD success !" -// } -// } -// aborted { -// script { -// echo "Milvus Cluster CI/CD aborted !" -// } -// } -// failure { -// script { -// echo "Milvus Cluster CI/CD failure !" -// } -// } -// } -// } - } - } - } - } - } - - post { - always { - script { - if (env.gitlabAfter != null) { - if (!currentBuild.resultIsBetterOrEqualTo('SUCCESS')) { - // Send an email only if the build status has changed from green/unstable to red - emailext subject: '$DEFAULT_SUBJECT', - body: '$DEFAULT_CONTENT', - recipientProviders: [ - [$class: 'DevelopersRecipientProvider'], - [$class: 'RequesterRecipientProvider'] - ], - replyTo: '$DEFAULT_REPLYTO', - to: '$DEFAULT_RECIPIENTS' - } - } - } - } - - success { - script { - updateGitlabCommitStatus name: 'CI/CD', state: 'success' - echo "Milvus CI/CD success !" - } - } - - aborted { - script { - updateGitlabCommitStatus name: 'CI/CD', state: 'canceled' - echo "Milvus CI/CD aborted !" - } - } - - failure { - script { - updateGitlabCommitStatus name: 'CI/CD', state: 'failed' - echo "Milvus CI/CD failure !" - } - } - } -} - diff --git a/ci/nightly_main_jenkinsfile b/ci/nightly_main_jenkinsfile deleted file mode 100644 index add9e00fb4..0000000000 --- a/ci/nightly_main_jenkinsfile +++ /dev/null @@ -1,478 +0,0 @@ -pipeline { - agent none - - options { - timestamps() - } - - environment { - PROJECT_NAME = "milvus" - LOWER_BUILD_TYPE = BUILD_TYPE.toLowerCase() - SEMVER = "${env.gitlabSourceBranch == null ? params.ENGINE_BRANCH.substring(params.ENGINE_BRANCH.lastIndexOf('/') + 1) : env.gitlabSourceBranch}" - GITLAB_AFTER_COMMIT = "${env.gitlabAfter == null ? null : env.gitlabAfter}" - SUFFIX_VERSION_NAME = "${env.gitlabAfter == null ? null : env.gitlabAfter.substring(0, 6)}" - DOCKER_VERSION_STR = "${env.gitlabAfter == null ? '${SEMVER}-${LOWER_BUILD_TYPE}-${BUILD_DATE_FORMATTED, \"yyyyMMdd\"}' : '${SEMVER}-${LOWER_BUILD_TYPE}-${SUFFIX_VERSION_NAME}'}" - } - - stages { - stage("Ubuntu 16.04") { - environment { - PACKAGE_VERSION = VersionNumber([ - versionNumberString : '${SEMVER}-${LOWER_BUILD_TYPE}-${BUILD_DATE_FORMATTED, "yyyyMMdd"}' - ]); - - DOCKER_VERSION = VersionNumber([ - versionNumberString : '${DOCKER_VERSION_STR}' - ]); - } - - stages { - stage("Run Build") { - agent { - kubernetes { - cloud 'build-kubernetes' - label 'build' - defaultContainer 'jnlp' - yaml """ -apiVersion: v1 -kind: Pod -metadata: - name: milvus-build-env - labels: - app: milvus - componet: build-env -spec: - containers: - - name: milvus-build-env - image: registry.zilliz.com/milvus/milvus-build-env:v0.13 - command: - - cat - tty: true - resources: - limits: - memory: "28Gi" - cpu: "10.0" - nvidia.com/gpu: 1 - requests: - memory: "14Gi" - cpu: "5.0" -""" - } - } - stages { - stage('Build') { - steps { - gitlabCommitStatus(name: 'Build') { - script { - load "${env.WORKSPACE}/ci/jenkinsfile/milvus_build.groovy" - load "${env.WORKSPACE}/ci/jenkinsfile/packaged_milvus.groovy" - } - } - } - } - } - post { - aborted { - script { - updateGitlabCommitStatus name: 'Build', state: 'canceled' - echo "Milvus Build aborted !" - } - } - - failure { - script { - updateGitlabCommitStatus name: 'Build', state: 'failed' - echo "Milvus Build failure !" - } - } - } - } - - stage("Publish docker and helm") { - agent { - kubernetes { - label 'publish' - defaultContainer 'jnlp' - yaml """ -apiVersion: v1 -kind: Pod -metadata: - labels: - app: publish - componet: docker -spec: - containers: - - name: publish-docker - image: registry.zilliz.com/library/zilliz_docker:v1.0.0 - securityContext: - privileged: true - command: - - cat - tty: true - volumeMounts: - - name: docker-sock - mountPath: /var/run/docker.sock - volumes: - - name: docker-sock - hostPath: - path: /var/run/docker.sock -""" - } - } - stages { - stage('Publish Docker') { - steps { - gitlabCommitStatus(name: 'Publish Docker') { - script { - load "${env.WORKSPACE}/ci/jenkinsfile/nightly_publish_docker.groovy" - } - } - } - } - } - post { - aborted { - script { - updateGitlabCommitStatus name: 'Publish Docker', state: 'canceled' - echo "Milvus Publish Docker aborted !" - } - } - - failure { - script { - updateGitlabCommitStatus name: 'Publish Docker', state: 'failed' - echo "Milvus Publish Docker failure !" - } - } - } - } - - stage("Deploy to Development") { - parallel { - stage("Single Node") { - agent { - kubernetes { - label 'dev-test' - defaultContainer 'jnlp' - yaml """ -apiVersion: v1 -kind: Pod -metadata: - labels: - app: milvus - componet: test -spec: - containers: - - name: milvus-testframework - image: registry.zilliz.com/milvus/milvus-test:v0.2 - command: - - cat - tty: true - volumeMounts: - - name: kubeconf - mountPath: /root/.kube/ - readOnly: true - volumes: - - name: kubeconf - secret: - secretName: test-cluster-config -""" - } - } - - stages { - stage("Deploy to Dev") { - steps { - gitlabCommitStatus(name: 'Deloy to Dev') { - container('milvus-testframework') { - script { - load "${env.WORKSPACE}/ci/jenkinsfile/deploy2dev.groovy" - } - } - } - } - } - stage("Dev Test") { - steps { - gitlabCommitStatus(name: 'Deloy Test') { - container('milvus-testframework') { - script { - load "${env.WORKSPACE}/ci/jenkinsfile/dev_test_all.groovy" - load "${env.WORKSPACE}/ci/jenkinsfile/upload_dev_test_out.groovy" - } - } - } - } - } - stage ("Cleanup Dev") { - steps { - gitlabCommitStatus(name: 'Cleanup Dev') { - container('milvus-testframework') { - script { - load "${env.WORKSPACE}/ci/jenkinsfile/cleanup_dev.groovy" - } - } - } - } - } - } - post { - always { - container('milvus-testframework') { - script { - load "${env.WORKSPACE}/ci/jenkinsfile/cleanup_dev.groovy" - } - } - } - success { - script { - echo "Milvus Deploy to Dev Single Node CI/CD success !" - } - } - aborted { - script { - echo "Milvus Deploy to Dev Single Node CI/CD aborted !" - } - } - failure { - script { - echo "Milvus Deploy to Dev Single Node CI/CD failure !" - } - } - } - } - -// stage("Cluster") { -// agent { -// kubernetes { -// label 'dev-test' -// defaultContainer 'jnlp' -// yaml """ -// apiVersion: v1 -// kind: Pod -// metadata: -// labels: -// app: milvus -// componet: test -// spec: -// containers: -// - name: milvus-testframework -// image: registry.zilliz.com/milvus/milvus-test:v0.2 -// command: -// - cat -// tty: true -// volumeMounts: -// - name: kubeconf -// mountPath: /root/.kube/ -// readOnly: true -// volumes: -// - name: kubeconf -// secret: -// secretName: test-cluster-config -// """ -// } -// } -// stages { -// stage("Deploy to Dev") { -// steps { -// gitlabCommitStatus(name: 'Deloy to Dev') { -// container('milvus-testframework') { -// script { -// load "${env.WORKSPACE}/ci/jenkinsfile/cluster_deploy2dev.groovy" -// } -// } -// } -// } -// } -// stage("Dev Test") { -// steps { -// gitlabCommitStatus(name: 'Deloy Test') { -// container('milvus-testframework') { -// script { -// load "${env.WORKSPACE}/ci/jenkinsfile/cluster_dev_test.groovy" -// load "${env.WORKSPACE}/ci/jenkinsfile/upload_dev_cluster_test_out.groovy" -// } -// } -// } -// } -// } -// stage ("Cleanup Dev") { -// steps { -// gitlabCommitStatus(name: 'Cleanup Dev') { -// container('milvus-testframework') { -// script { -// load "${env.WORKSPACE}/ci/jenkinsfile/cluster_cleanup_dev.groovy" -// } -// } -// } -// } -// } -// } -// post { -// always { -// container('milvus-testframework') { -// script { -// load "${env.WORKSPACE}/ci/jenkinsfile/cluster_cleanup_dev.groovy" -// } -// } -// } -// success { -// script { -// echo "Milvus Deploy to Dev Cluster CI/CD success !" -// } -// } -// aborted { -// script { -// echo "Milvus Deploy to Dev Cluster CI/CD aborted !" -// } -// } -// failure { -// script { -// echo "Milvus Deploy to Dev Cluster CI/CD failure !" -// } -// } -// } -// } - } - } - - stage("Deploy to Staging") { - parallel { - stage("Single Node") { - agent { - kubernetes { - label 'dev-test' - defaultContainer 'jnlp' - yaml """ -apiVersion: v1 -kind: Pod -metadata: - labels: - app: milvus - componet: test -spec: - containers: - - name: milvus-testframework - image: registry.zilliz.com/milvus/milvus-test:v0.2 - command: - - cat - tty: true - volumeMounts: - - name: kubeconf - mountPath: /root/.kube/ - readOnly: true - volumes: - - name: kubeconf - secret: - secretName: aks-gpu-cluster-config -""" - } - } - - stages { - stage("Deploy to Staging") { - steps { - gitlabCommitStatus(name: 'Deloy to Staging') { - container('milvus-testframework') { - script { - load "${env.WORKSPACE}/ci/jenkinsfile/deploy2staging.groovy" - } - } - } - } - } - stage("Staging Test") { - steps { - gitlabCommitStatus(name: 'Staging Test') { - container('milvus-testframework') { - script { - load "${env.WORKSPACE}/ci/jenkinsfile/staging_test.groovy" - load "${env.WORKSPACE}/ci/jenkinsfile/upload_staging_test_out.groovy" - } - } - } - } - } - stage ("Cleanup Staging") { - steps { - gitlabCommitStatus(name: 'Cleanup Staging') { - container('milvus-testframework') { - script { - load "${env.WORKSPACE}/ci/jenkinsfile/cleanup_staging.groovy" - } - } - } - } - } - } - post { - always { - container('milvus-testframework') { - script { - load "${env.WORKSPACE}/ci/jenkinsfile/cleanup_staging.groovy" - } - } - } - success { - script { - echo "Milvus Deploy to Staging Single Node CI/CD success !" - } - } - aborted { - script { - echo "Milvus Deploy to Staging Single Node CI/CD aborted !" - } - } - failure { - script { - echo "Milvus Deploy to Staging Single Node CI/CD failure !" - } - } - } - } - } - } - } - } - } - - post { - always { - script { - if (!currentBuild.resultIsBetterOrEqualTo('SUCCESS')) { - // Send an email only if the build status has changed from green/unstable to red - emailext subject: '$DEFAULT_SUBJECT', - body: '$DEFAULT_CONTENT', - recipientProviders: [ - [$class: 'DevelopersRecipientProvider'], - [$class: 'RequesterRecipientProvider'] - ], - replyTo: '$DEFAULT_REPLYTO', - to: '$DEFAULT_RECIPIENTS' - } - } - } - - success { - script { - updateGitlabCommitStatus name: 'CI/CD', state: 'success' - echo "Milvus CI/CD success !" - } - } - - aborted { - script { - updateGitlabCommitStatus name: 'CI/CD', state: 'canceled' - echo "Milvus CI/CD aborted !" - } - } - - failure { - script { - updateGitlabCommitStatus name: 'CI/CD', state: 'failed' - echo "Milvus CI/CD failure !" - } - } - } -} - diff --git a/ci/pod_containers/milvus-engine-build.yaml b/ci/pod_containers/milvus-engine-build.yaml deleted file mode 100644 index cd5352ffef..0000000000 --- a/ci/pod_containers/milvus-engine-build.yaml +++ /dev/null @@ -1,13 +0,0 @@ -apiVersion: v1 -kind: Pod -metadata: - labels: - app: milvus - componet: build-env -spec: - containers: - - name: milvus-build-env - image: registry.zilliz.com/milvus/milvus-build-env:v0.9 - command: - - cat - tty: true diff --git a/ci/pod_containers/milvus-testframework.yaml b/ci/pod_containers/milvus-testframework.yaml deleted file mode 100644 index 7a98fbca8e..0000000000 --- a/ci/pod_containers/milvus-testframework.yaml +++ /dev/null @@ -1,13 +0,0 @@ -apiVersion: v1 -kind: Pod -metadata: - labels: - app: milvus - componet: testframework -spec: - containers: - - name: milvus-testframework - image: registry.zilliz.com/milvus/milvus-test:v0.1 - command: - - cat - tty: true diff --git a/ci/pod_containers/publish-docker.yaml b/ci/pod_containers/publish-docker.yaml deleted file mode 100644 index 268afb1331..0000000000 --- a/ci/pod_containers/publish-docker.yaml +++ /dev/null @@ -1,22 +0,0 @@ -apiVersion: v1 -kind: Pod -metadata: - labels: - app: publish - componet: docker -spec: - containers: - - name: publish-docker - image: registry.zilliz.com/library/zilliz_docker:v1.0.0 - securityContext: - privileged: true - command: - - cat - tty: true - volumeMounts: - - name: docker-sock - mountPath: /var/run/docker.sock - volumes: - - name: docker-sock - hostPath: - path: /var/run/docker.sock From 91c33882b54d6f41f1b14f257d7518e6308740b2 Mon Sep 17 00:00:00 2001 From: zerowe-seven Date: Fri, 22 Nov 2019 06:56:53 -0800 Subject: [PATCH 6/9] #329 The error message in TableNotExistMsg has a grammatical error --- core/src/server/grpc_impl/request/GrpcBaseRequest.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/server/grpc_impl/request/GrpcBaseRequest.cpp b/core/src/server/grpc_impl/request/GrpcBaseRequest.cpp index e1eb07af5c..2aa96c5331 100644 --- a/core/src/server/grpc_impl/request/GrpcBaseRequest.cpp +++ b/core/src/server/grpc_impl/request/GrpcBaseRequest.cpp @@ -90,7 +90,7 @@ GrpcBaseRequest::SetStatus(ErrorCode error_code, const std::string& error_msg) { std::string GrpcBaseRequest::TableNotExistMsg(const std::string& table_name) { return "Table " + table_name + - " not exist. Use milvus.has_table to verify whether the table exists. You also can check if the table name " + " does not exist. Use milvus.has_table to verify whether the table exists. You also can check whether the table name " "exists."; } From 0ba7797833361f7b45822cabd7084876a8d3e980 Mon Sep 17 00:00:00 2001 From: quicksilver Date: Fri, 22 Nov 2019 18:37:09 +0800 Subject: [PATCH 7/9] update log_config.conf --- ci/jenkins/step/package.groovy | 1 + 1 file changed, 1 insertion(+) diff --git a/ci/jenkins/step/package.groovy b/ci/jenkins/step/package.groovy index c07dcd9d88..80128b3b74 100644 --- a/ci/jenkins/step/package.groovy +++ b/ci/jenkins/step/package.groovy @@ -2,6 +2,7 @@ timeout(time: 5, unit: 'MINUTES') { dir ("ci/jenkins/scripts") { sh "pip3 install -r requirements.txt" sh "./yaml_processor.py merge -f /opt/milvus/conf/server_config.yaml -m ../yaml/update_server_config.yaml -i && rm /opt/milvus/conf/server_config.yaml.bak" + sh "sed -i 's/\/tmp\/milvus/\/opt\/milvus/g' /opt/milvus/conf/log_config.conf" } sh "tar -zcvf ./${PROJECT_NAME}-${PACKAGE_VERSION}.tar.gz -C /opt/ milvus" withCredentials([usernamePassword(credentialsId: "${params.JFROG_CREDENTIALS_ID}", usernameVariable: 'JFROG_USERNAME', passwordVariable: 'JFROG_PASSWORD')]) { From 8164bb031cd71fb8b21842d1fed8fbda0544ea8d Mon Sep 17 00:00:00 2001 From: quicksilver Date: Fri, 22 Nov 2019 18:46:06 +0800 Subject: [PATCH 8/9] disable untar verbose --- core/cmake/BuildUtils.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/cmake/BuildUtils.cmake b/core/cmake/BuildUtils.cmake index 68cd22ae58..6332d29d74 100644 --- a/core/cmake/BuildUtils.cmake +++ b/core/cmake/BuildUtils.cmake @@ -74,7 +74,7 @@ function(ExternalProject_Use_Cache project_name package_file install_path) ${CMAKE_COMMAND} -E echo "Extracting ${package_file} to ${install_path}" COMMAND - ${CMAKE_COMMAND} -E tar xzvf ${package_file} ${install_path} + ${CMAKE_COMMAND} -E tar xzf ${package_file} ${install_path} WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} ) From 97807df0af0e527a205593d28497866e79003e54 Mon Sep 17 00:00:00 2001 From: JinHai-CN Date: Fri, 22 Nov 2019 20:30:44 +0800 Subject: [PATCH 9/9] Fix lint --- core/src/server/grpc_impl/request/GrpcBaseRequest.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/src/server/grpc_impl/request/GrpcBaseRequest.cpp b/core/src/server/grpc_impl/request/GrpcBaseRequest.cpp index 2aa96c5331..0f46217057 100644 --- a/core/src/server/grpc_impl/request/GrpcBaseRequest.cpp +++ b/core/src/server/grpc_impl/request/GrpcBaseRequest.cpp @@ -90,8 +90,8 @@ GrpcBaseRequest::SetStatus(ErrorCode error_code, const std::string& error_msg) { std::string GrpcBaseRequest::TableNotExistMsg(const std::string& table_name) { return "Table " + table_name + - " does not exist. Use milvus.has_table to verify whether the table exists. You also can check whether the table name " - "exists."; + " does not exist. Use milvus.has_table to verify whether the table exists. " + "You also can check whether the table name exists."; } Status