[skip e2e] Add publish migration image (#19869)

Signed-off-by: Jenny Li <jing.li@zilliz.com>

Signed-off-by: Jenny Li <jing.li@zilliz.com>

Makefile

@@ -90,8 +90,7 @@ binlog:
 MIGRATION_PATH = $(PWD)/cmd/tools/migration
 meta-migration:
 	@echo "Building migration tool ..."
-	@source $(PWD)/scripts/setenv.sh && \
+	@mkdir -p $(INSTALL_PATH) && go env -w CGO_ENABLED="1" && GO111MODULE=on $(GO) build -o $(INSTALL_PATH)/meta-migration $(MIGRATION_PATH)/main.go 1>/dev/null
-		mkdir -p $(INSTALL_PATH) && go env -w CGO_ENABLED="1" && GO111MODULE=on $(GO) build -o $(INSTALL_PATH)/meta-migration $(MIGRATION_PATH)/main.go 1>/dev/null
 
 BUILD_TAGS = $(shell git describe --tags --always --dirty="-dev")
 BUILD_TIME = $(shell date -u)

build/docker/meta-migration/Dockerfile

@@ -0,0 +1,23 @@
+# Copyright (C) 2019-2022 Zilliz. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+# or implied. See the License for the specific language governing permissions and limitations under the License.
+
+
+#FROM alpine
+FROM alpine:3.16
+
+
+COPY ./bin/ /milvus/bin/
+
+ENV PATH=/milvus/bin:$PATH
+
+ENTRYPOINT ["/bin/sh"]
+
+WORKDIR /milvus/

build/docker/meta-migration/builder/Dockerfile

@@ -0,0 +1,2 @@
+FROM golang:1.18.3-alpine3.16
+RUN apk add --no-cache make bash

ci/jenkins/MetaMigrationBuilder.groovy

@@ -0,0 +1,48 @@
+#!/usr/bin/env groovy
+def app="meta-migration-builder"
+def date=""
+def gitShortCommit=""
+pipeline {
+    agent {
+        kubernetes {
+            defaultContainer 'main'
+            yamlFile "ci/jenkins/pod/meta-builder.yaml"
+            customWorkspace '/home/jenkins/agent/workspace'
+        }
+    }
+
+    options {
+        timestamps()
+        timeout(time: 36, unit: 'MINUTES')
+        disableConcurrentBuilds(abortPrevious: true)
+    }
+
+    environment {
+        HARBOR_REPO = "harbor.milvus.io"
+        CI_DOCKER_CREDENTIAL_ID="harbor-milvus-io-registry"
+    }
+
+    stages {
+        stage('Publish Meta Migration builder Images') {
+            steps {
+                container('main'){
+                    script{
+                        date=sh(returnStdout: true, script: 'date +%Y%m%d').trim()
+                        gitShortCommit=sh(returnStdout: true, script: 'git rev-parse --short HEAD').trim()
+                        sh './build/set_docker_mirror.sh'
+                        def tag="${date}-${gitShortCommit}"
+                        def image="${env.HARBOR_REPO}/milvus/${app}:${tag}"
+                        withCredentials([usernamePassword(credentialsId: "${env.CI_DOCKER_CREDENTIAL_ID}", usernameVariable: 'CI_REGISTRY_USERNAME', passwordVariable: 'CI_REGISTRY_PASSWORD')]){
+                                sh "docker login ${env.HARBOR_REPO} -u '${CI_REGISTRY_USERNAME}' -p '${CI_REGISTRY_PASSWORD}'"
+                                sh """
+                                    docker build -t  ${image} -f build/docker/meta-migration/builder/Dockerfile .
+                                    docker push ${image}
+                                    docker logout
+                                """
+                            }
+                        }
+                }
+            }
+        }
+    }
+}

ci/jenkins/PublishMigrationImage.groovy

@@ -0,0 +1,68 @@
+#!/usr/bin/env groovy
+def app="meta-migration"
+def date=""
+def gitShortCommit=""
+pipeline {
+    agent {
+        kubernetes {
+            defaultContainer 'main'
+            yamlFile "ci/jenkins/pod/meta-migration.yaml"
+            customWorkspace '/home/jenkins/agent/workspace'
+        }
+    }
+
+    options {
+        timestamps()
+        timeout(time: 36, unit: 'MINUTES')
+        disableConcurrentBuilds()
+    }
+
+    environment {
+        HARBOR_REPO = "harbor.milvus.io/milvus"
+        CI_DOCKER_CREDENTIAL_ID="harbor-milvus-io-registry"
+    }
+
+    stages {
+            stage('Publish Meta Migration Images') {
+                steps {
+                    container(name: 'build',shell: '/bin/sh') {
+                        script {
+                            sh "make meta-migration"
+                        }
+                    }
+                    container('main'){
+                        script {
+                            date=sh(returnStdout: true, script: 'date +%Y%m%d').trim()
+                            gitShortCommit=sh(returnStdout: true, script: 'git rev-parse --short HEAD').trim()
+                            sh 'printenv'
+                            def tag="${date}-${gitShortCommit}"
+                            def image="${env.HARBOR_REPO}/milvus/${app}:${tag}"
+                            withCredentials([usernamePassword(credentialsId: "${env.CI_DOCKER_CREDENTIAL_ID}", usernameVariable: 'CI_REGISTRY_USERNAME', passwordVariable: 'CI_REGISTRY_PASSWORD')]){
+                                    sh "docker login ${env.HARBOR_REPO} -u '${CI_REGISTRY_USERNAME}' -p '${CI_REGISTRY_PASSWORD}'"
+                                    sh """
+                                        docker build -t  ${image} -f build/docker/meta-migration/Dockerfile .
+                                        docker push ${image}
+                                        docker logout
+                                    """
+                            }
+                        }
+                    }
+                }
+
+            }
+    }
+    post {
+        unsuccessful {
+            container('jnlp') {
+                script {
+                    emailext subject: '$DEFAULT_SUBJECT',
+                    body: '$DEFAULT_CONTENT',
+                    recipientProviders: [developers(), culprits()],
+                    replyTo: '$DEFAULT_REPLYTO',
+                    // to: "${authorEmail},qa@zilliz.com,devops@zilliz.com"
+                    to: "jing.li@zilliz.com"
+                }
+            }
+        }
+    }
+}

ci/jenkins/pod/meta-builder.yaml

@@ -0,0 +1,57 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  labels:
+    app: meta-migration-builder
+  namespace: milvus-ci
+spec:
+  enableServiceLinks: false
+  containers:
+  - name: main
+    image: milvusdb/krte:20211213-dcc15e9
+    env:
+    - name: DOCKER_IN_DOCKER_ENABLED
+      value: "true"
+    - name: DOCKER_VOLUME_DIRECTORY
+      value: "/mnt/disk/.docker"
+    tty: true
+    securityContext:
+      privileged: true
+    args: ["cat"]
+    resources:
+      limits:
+        cpu: "6"
+        memory: 12Gi
+      requests:
+        cpu: "0.5"
+        memory: 512Mi
+    volumeMounts:
+    - mountPath: /docker-graph
+      name: docker-graph
+    - mountPath: /var/lib/docker
+      name: docker-root
+    - mountPath: /lib/modules
+      name: modules
+      readOnly: true
+    - mountPath: /sys/fs/cgroup
+      name: cgroup
+    - mountPath: /mnt/disk/.docker
+      name: build-cache
+      subPath: docker-volume
+  volumes:
+  - emptyDir: {}
+    name: docker-graph
+  - emptyDir: {}
+    name: docker-root
+  - hostPath:
+      path: /tmp/krte/cache
+      type: DirectoryOrCreate
+    name: build-cache
+  - hostPath:
+      path: /lib/modules
+      type: Directory
+    name: modules
+  - hostPath:
+      path: /sys/fs/cgroup
+      type: Directory
+    name: cgroup

ci/jenkins/pod/meta-migration.yaml

@@ -0,0 +1,62 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  labels:
+    app: meta-migration
+  namespace: milvus-ci
+spec:
+  enableServiceLinks: false
+  containers:
+  - name: main
+    image: milvusdb/krte:20211213-dcc15e9
+    env:
+    - name: DOCKER_IN_DOCKER_ENABLED
+      value: "true"
+    - name: DOCKER_VOLUME_DIRECTORY
+      value: "/mnt/disk/.docker"
+    tty: true
+    securityContext:
+      privileged: true
+    args: ["cat"]
+    resources:
+      limits:
+        cpu: "6"
+        memory: 12Gi
+      requests:
+        cpu: "0.5"
+        memory: 512Mi
+    volumeMounts:
+    - mountPath: /docker-graph
+      name: docker-graph
+    - mountPath: /var/lib/docker
+      name: docker-root
+    - mountPath: /lib/modules
+      name: modules
+      readOnly: true
+    - mountPath: /sys/fs/cgroup
+      name: cgroup
+    - mountPath: /mnt/disk/.docker
+      name: build-cache
+      subPath: docker-volume
+  - name: build
+    image: harbor.milvus.io/milvus/meta-migration-builder:20221018-fc6fe33f8
+    command: ["/bin/sh"]
+    args: ["-c","cat"]
+    tty: true
+  volumes:
+  - emptyDir: {}
+    name: docker-graph
+  - emptyDir: {}
+    name: docker-root
+  - hostPath:
+      path: /tmp/krte/cache
+      type: DirectoryOrCreate
+    name: build-cache
+  - hostPath:
+      path: /lib/modules
+      type: Directory
+    name: modules
+  - hostPath:
+      path: /sys/fs/cgroup
+      type: Directory
+    name: cgroup