From 6f5ff61ac3411dc579c3cc965ab5cd3624437a22 Mon Sep 17 00:00:00 2001 From: Jenny Li Date: Tue, 18 Oct 2022 18:49:26 +0800 Subject: [PATCH] [skip e2e] Add publish migration image (#19869) Signed-off-by: Jenny Li Signed-off-by: Jenny Li --- Makefile | 3 +- build/docker/meta-migration/Dockerfile | 23 +++++++ .../docker/meta-migration/builder/Dockerfile | 2 + ci/jenkins/MetaMigrationBuilder.groovy | 48 +++++++++++++ ci/jenkins/PublishMigrationImage.groovy | 68 +++++++++++++++++++ ci/jenkins/pod/meta-builder.yaml | 57 ++++++++++++++++ ci/jenkins/pod/meta-migration.yaml | 62 +++++++++++++++++ 7 files changed, 261 insertions(+), 2 deletions(-) create mode 100644 build/docker/meta-migration/Dockerfile create mode 100644 build/docker/meta-migration/builder/Dockerfile create mode 100644 ci/jenkins/MetaMigrationBuilder.groovy create mode 100644 ci/jenkins/PublishMigrationImage.groovy create mode 100644 ci/jenkins/pod/meta-builder.yaml create mode 100644 ci/jenkins/pod/meta-migration.yaml diff --git a/Makefile b/Makefile index aa735e6988..cbbd5f945e 100644 --- a/Makefile +++ b/Makefile @@ -90,8 +90,7 @@ binlog: MIGRATION_PATH = $(PWD)/cmd/tools/migration meta-migration: @echo "Building migration tool ..." - @source $(PWD)/scripts/setenv.sh && \ - mkdir -p $(INSTALL_PATH) && go env -w CGO_ENABLED="1" && GO111MODULE=on $(GO) build -o $(INSTALL_PATH)/meta-migration $(MIGRATION_PATH)/main.go 1>/dev/null + @mkdir -p $(INSTALL_PATH) && go env -w CGO_ENABLED="1" && GO111MODULE=on $(GO) build -o $(INSTALL_PATH)/meta-migration $(MIGRATION_PATH)/main.go 1>/dev/null BUILD_TAGS = $(shell git describe --tags --always --dirty="-dev") BUILD_TIME = $(shell date -u) diff --git a/build/docker/meta-migration/Dockerfile b/build/docker/meta-migration/Dockerfile new file mode 100644 index 0000000000..9cc7abc28d --- /dev/null +++ b/build/docker/meta-migration/Dockerfile @@ -0,0 +1,23 @@ +# Copyright (C) 2019-2022 Zilliz. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software distributed under the License +# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +# or implied. See the License for the specific language governing permissions and limitations under the License. + + +#FROM alpine +FROM alpine:3.16 + + +COPY ./bin/ /milvus/bin/ + +ENV PATH=/milvus/bin:$PATH + +ENTRYPOINT ["/bin/sh"] + +WORKDIR /milvus/ diff --git a/build/docker/meta-migration/builder/Dockerfile b/build/docker/meta-migration/builder/Dockerfile new file mode 100644 index 0000000000..f9797b0f2e --- /dev/null +++ b/build/docker/meta-migration/builder/Dockerfile @@ -0,0 +1,2 @@ +FROM golang:1.18.3-alpine3.16 +RUN apk add --no-cache make bash \ No newline at end of file diff --git a/ci/jenkins/MetaMigrationBuilder.groovy b/ci/jenkins/MetaMigrationBuilder.groovy new file mode 100644 index 0000000000..1f44bcbf44 --- /dev/null +++ b/ci/jenkins/MetaMigrationBuilder.groovy @@ -0,0 +1,48 @@ +#!/usr/bin/env groovy +def app="meta-migration-builder" +def date="" +def gitShortCommit="" +pipeline { + agent { + kubernetes { + defaultContainer 'main' + yamlFile "ci/jenkins/pod/meta-builder.yaml" + customWorkspace '/home/jenkins/agent/workspace' + } + } + + options { + timestamps() + timeout(time: 36, unit: 'MINUTES') + disableConcurrentBuilds(abortPrevious: true) + } + + environment { + HARBOR_REPO = "harbor.milvus.io" + CI_DOCKER_CREDENTIAL_ID="harbor-milvus-io-registry" + } + + stages { + stage('Publish Meta Migration builder Images') { + steps { + container('main'){ + script{ + date=sh(returnStdout: true, script: 'date +%Y%m%d').trim() + gitShortCommit=sh(returnStdout: true, script: 'git rev-parse --short HEAD').trim() + sh './build/set_docker_mirror.sh' + def tag="${date}-${gitShortCommit}" + def image="${env.HARBOR_REPO}/milvus/${app}:${tag}" + withCredentials([usernamePassword(credentialsId: "${env.CI_DOCKER_CREDENTIAL_ID}", usernameVariable: 'CI_REGISTRY_USERNAME', passwordVariable: 'CI_REGISTRY_PASSWORD')]){ + sh "docker login ${env.HARBOR_REPO} -u '${CI_REGISTRY_USERNAME}' -p '${CI_REGISTRY_PASSWORD}'" + sh """ + docker build -t ${image} -f build/docker/meta-migration/builder/Dockerfile . + docker push ${image} + docker logout + """ + } + } + } + } + } + } +} diff --git a/ci/jenkins/PublishMigrationImage.groovy b/ci/jenkins/PublishMigrationImage.groovy new file mode 100644 index 0000000000..b97c00f95e --- /dev/null +++ b/ci/jenkins/PublishMigrationImage.groovy @@ -0,0 +1,68 @@ +#!/usr/bin/env groovy +def app="meta-migration" +def date="" +def gitShortCommit="" +pipeline { + agent { + kubernetes { + defaultContainer 'main' + yamlFile "ci/jenkins/pod/meta-migration.yaml" + customWorkspace '/home/jenkins/agent/workspace' + } + } + + options { + timestamps() + timeout(time: 36, unit: 'MINUTES') + disableConcurrentBuilds() + } + + environment { + HARBOR_REPO = "harbor.milvus.io/milvus" + CI_DOCKER_CREDENTIAL_ID="harbor-milvus-io-registry" + } + + stages { + stage('Publish Meta Migration Images') { + steps { + container(name: 'build',shell: '/bin/sh') { + script { + sh "make meta-migration" + } + } + container('main'){ + script { + date=sh(returnStdout: true, script: 'date +%Y%m%d').trim() + gitShortCommit=sh(returnStdout: true, script: 'git rev-parse --short HEAD').trim() + sh 'printenv' + def tag="${date}-${gitShortCommit}" + def image="${env.HARBOR_REPO}/milvus/${app}:${tag}" + withCredentials([usernamePassword(credentialsId: "${env.CI_DOCKER_CREDENTIAL_ID}", usernameVariable: 'CI_REGISTRY_USERNAME', passwordVariable: 'CI_REGISTRY_PASSWORD')]){ + sh "docker login ${env.HARBOR_REPO} -u '${CI_REGISTRY_USERNAME}' -p '${CI_REGISTRY_PASSWORD}'" + sh """ + docker build -t ${image} -f build/docker/meta-migration/Dockerfile . + docker push ${image} + docker logout + """ + } + } + } + } + + } + } + post { + unsuccessful { + container('jnlp') { + script { + emailext subject: '$DEFAULT_SUBJECT', + body: '$DEFAULT_CONTENT', + recipientProviders: [developers(), culprits()], + replyTo: '$DEFAULT_REPLYTO', + // to: "${authorEmail},qa@zilliz.com,devops@zilliz.com" + to: "jing.li@zilliz.com" + } + } + } + } +} \ No newline at end of file diff --git a/ci/jenkins/pod/meta-builder.yaml b/ci/jenkins/pod/meta-builder.yaml new file mode 100644 index 0000000000..60bdbc3e08 --- /dev/null +++ b/ci/jenkins/pod/meta-builder.yaml @@ -0,0 +1,57 @@ +apiVersion: v1 +kind: Pod +metadata: + labels: + app: meta-migration-builder + namespace: milvus-ci +spec: + enableServiceLinks: false + containers: + - name: main + image: milvusdb/krte:20211213-dcc15e9 + env: + - name: DOCKER_IN_DOCKER_ENABLED + value: "true" + - name: DOCKER_VOLUME_DIRECTORY + value: "/mnt/disk/.docker" + tty: true + securityContext: + privileged: true + args: ["cat"] + resources: + limits: + cpu: "6" + memory: 12Gi + requests: + cpu: "0.5" + memory: 512Mi + volumeMounts: + - mountPath: /docker-graph + name: docker-graph + - mountPath: /var/lib/docker + name: docker-root + - mountPath: /lib/modules + name: modules + readOnly: true + - mountPath: /sys/fs/cgroup + name: cgroup + - mountPath: /mnt/disk/.docker + name: build-cache + subPath: docker-volume + volumes: + - emptyDir: {} + name: docker-graph + - emptyDir: {} + name: docker-root + - hostPath: + path: /tmp/krte/cache + type: DirectoryOrCreate + name: build-cache + - hostPath: + path: /lib/modules + type: Directory + name: modules + - hostPath: + path: /sys/fs/cgroup + type: Directory + name: cgroup \ No newline at end of file diff --git a/ci/jenkins/pod/meta-migration.yaml b/ci/jenkins/pod/meta-migration.yaml new file mode 100644 index 0000000000..dfe7e233bd --- /dev/null +++ b/ci/jenkins/pod/meta-migration.yaml @@ -0,0 +1,62 @@ +apiVersion: v1 +kind: Pod +metadata: + labels: + app: meta-migration + namespace: milvus-ci +spec: + enableServiceLinks: false + containers: + - name: main + image: milvusdb/krte:20211213-dcc15e9 + env: + - name: DOCKER_IN_DOCKER_ENABLED + value: "true" + - name: DOCKER_VOLUME_DIRECTORY + value: "/mnt/disk/.docker" + tty: true + securityContext: + privileged: true + args: ["cat"] + resources: + limits: + cpu: "6" + memory: 12Gi + requests: + cpu: "0.5" + memory: 512Mi + volumeMounts: + - mountPath: /docker-graph + name: docker-graph + - mountPath: /var/lib/docker + name: docker-root + - mountPath: /lib/modules + name: modules + readOnly: true + - mountPath: /sys/fs/cgroup + name: cgroup + - mountPath: /mnt/disk/.docker + name: build-cache + subPath: docker-volume + - name: build + image: harbor.milvus.io/milvus/meta-migration-builder:20221018-fc6fe33f8 + command: ["/bin/sh"] + args: ["-c","cat"] + tty: true + volumes: + - emptyDir: {} + name: docker-graph + - emptyDir: {} + name: docker-root + - hostPath: + path: /tmp/krte/cache + type: DirectoryOrCreate + name: build-cache + - hostPath: + path: /lib/modules + type: Directory + name: modules + - hostPath: + path: /sys/fs/cgroup + type: Directory + name: cgroup \ No newline at end of file