From c9d4f10ce5624bd1254cf8dc4013da677d96386d Mon Sep 17 00:00:00 2001
From: "edward.zeng" <jie.zeng@zilliz.com>
Date: Thu, 20 Oct 2022 08:57:30 +0800
Subject: [PATCH] Add meta migration script for milvus installation in
 kubernetes (#19861)

Signed-off-by: Edward Zeng <jie.zeng@zilliz.com>

Signed-off-by: Edward Zeng <jie.zeng@zilliz.com>
---
 deployments/migrate-meta/README.md  |  66 +++++
 deployments/migrate-meta/migrate.sh | 434 ++++++++++++++++++++++++++++
 2 files changed, 500 insertions(+)
 create mode 100644 deployments/migrate-meta/README.md
 create mode 100755 deployments/migrate-meta/migrate.sh

diff --git a/deployments/migrate-meta/README.md b/deployments/migrate-meta/README.md
new file mode 100644
index 0000000000..4b4b0ea168
--- /dev/null
+++ b/deployments/migrate-meta/README.md
@@ -0,0 +1,66 @@
+# README
+
+## Overview
+
+Milvus 2.2 has changed the meta structure for segment index. If you have installed a milvus cluster of version v2.1.x, you'd run this script to migrate the meta and upgrade the milvus image version.
+
+> Note: 
+> 1. This script only works with Milvus installed on k8s cluster.
+> 2. You can only upgrade from v2.1.x to v2.2.
+
+## Parameter Description
+
+| Parameters | Description                                       | Default                      | Required                |
+| ---------- | ------------------------------------------------- | ---------------------------- | ----------------------- |
+| i          | Specify the milvus instance name                  | None                         | true                    |
+| n          | Specify the namespace that milvus is installed in | default                      | false                   |
+| s          | Specify the milvus source version                 | None                         | true                    |
+| t          | Specify the milvus target version                 | None                         | true                    |
+| r          | Specify the milvus meta root path                 | by-dev                       | false                   |
+| w          | Specify the milvus new image tag                  | milvusdb/milvus:v2.2.0       | false                   |
+| o          | Specify the meta migration operation              | migrate                      | false                   |
+> By default, the script only migrate from v2.1.x to v2.2.x. If there is anything wrong, you'd first rollback to the older version using the `rollback` operation.
+
+## Migrate Procedures
+The migration will take four steps:
+1. Stop the milvus components; if there are any live session in milvus etcd, the migration will fail.
+2. Backup the milvus meta;
+3. Migrate the milvus meta;
+4. Startup milvus components with provided new image;
+
+> Note:
+> 1. if step 1) or 2) are failed, you can rerun the migration again
+> 2. if step 3) or 4) are failed, you'd first rollback and migrate again
+
+## Usage
+
+1. Milvus instance name, milvus source version, milvus target vresion are required to be specified.
+
+```shell
+./migrate.sh -i my-release -s 2.1.1 -t 2.2.0
+```
+
+2. If your milvus is not installed in the k8s default namespace, please specify namespace with `-n`.
+
+```shell
+./migrate.sh -i my-release -n milvus -s 2.1.1 -t 2.2.0
+```
+
+3. If your milvus is installed with custom `rootpath`, please specify rootpath with `-r`.
+
+```shell
+./migrate.sh -i my-release -n milvus -s 2.1.1 -t 2.2.0 -r by-dev
+```
+
+4. If your milvus is installed with custom `image`, please specify the image tag with `-w`.
+
+```shell
+./migrate.sh -i my-release -n milvus -s 2.1.1 -t 2.2.0 -r by-dev -w milvusdb/milvus:master-20221016-15878781
+```
+
+5. If the migrate is failed, you'd first rollback migration and rerun migrate again.
+
+```
+./migrate.sh -i my-release -n milvus -s 2.1.1 -t 2.2.0 -r by-dev -o rollback -w <milvus-2-1-1-image>
+./migrate.sh -i my-release -n milvus -s 2.1.1 -t 2.2.0 -r by-dev -o migrate -w <milvus-2-2-0-image>
+```
diff --git a/deployments/migrate-meta/migrate.sh b/deployments/migrate-meta/migrate.sh
new file mode 100755
index 0000000000..be95224bf1
--- /dev/null
+++ b/deployments/migrate-meta/migrate.sh
@@ -0,0 +1,434 @@
+#!/bin/bash
+
+namespace="default"
+root_path="by-dev"
+operation="migrate"
+image_tag="milvusdb/milvus:v2.2.0"
+
+#-n namespace: The namespace that Milvus is installed in.
+#-i milvus_instance: The name of milvus instance.
+#-s source_version: The milvus source version.
+#-t target_version: The milvus target version.
+#-r root_path: The milvus meta root path.
+#-w image_tag: The new milvus image tag.
+#-o operation: The operation: migrate/rollback
+while getopts "n:i:s:t:r:w:o" opt_name
+do
+  case $opt_name in
+    n) namespace=$OPTARG;;
+    i) instance_name=$OPTARG;;
+    s) source_version=$OPTARG;;
+    t) target_version=$OPTARG;;
+    r) root_path=$OPTARG;;
+    w) image_tag=$OPTARG;;
+    o) operation=$OPTARG;;
+    *) echo "Unkonwen parameters";;
+  esac
+done
+
+if [ ! $instance_name ]; then
+  echo "Missing argument instance_name, please add it. For example:'./migrate.sh -i milvus-instance-name'"
+  exit 1
+fi
+
+if [ ! $source_version ]; then
+  echo "Missing argument source_version, please add it. For example:'./migrate.sh -s 2.1.4'"
+  exit 1
+fi
+
+if [ ! $target_version ]; then
+  echo "Missing argument target_version, please add it. For example:'./migrate.sh -t 2.2.0'"
+  exit 1
+fi
+
+if [ ! $image_tag ]; then
+  echo "Missing argument image_tag, please add it. For example:'./migrate.sh -w milvusdb/milvus:v2.2.0'"
+  exit 1
+fi
+
+echo "Migration milvus meta will take four steps:"
+echo "1. Stop the milvus components"
+echo "2. Backup the milvus meta"
+echo "3. Migrate the milvus meta"
+echo "4. Startup milvus components in new image version"
+echo
+
+
+replicas=()
+function fetch_deploy_replica(){
+  for deploy in $1;
+  do
+    replica=$(kubectl get deploy $deploy -n $namespace --output=jsonpath={.spec.replicas})
+    replicas+=("$deploy:$replica")
+  done
+}
+
+function stop_milvus_deploy(){
+  echo "Stop milvus deployments: $1"
+  kubectl scale deploy -n $namespace $1 --replicas=0 > /dev/null
+  wait_for_milvus_stopped $1
+  echo "Stopped..."
+}
+
+function wait_for_milvus_stopped(){
+  while true
+  do
+    total=0
+    for deploy in $1;
+    do 
+      count=$(kubectl get deploy $deploy -n $namespace --output=jsonpath={.spec.replicas})
+      total=`expr $total + $count`
+    done
+    if [ $total -eq 0 ]; then
+      break
+    fi
+    sleep 5
+  done
+}
+
+function wait_for_backup_done(){
+  backup_pod_name="milvus-meta-migration-backup-${instance_name}"
+  configmap_name="milvus-meta-migration-config-${instance_name}"
+  while true
+  do
+    status=$(kubectl get pod -n $namespace $backup_pod_name --output=jsonpath={.status.phase})
+    case $status in
+      Succeeded)
+        echo "Meta backup is done..."
+        kubectl annotate configmap $configmap_name -n $namespace backup=succeed
+        break
+        ;;
+      Failed)
+        echo "Meta backup is failed..."
+        echo "Here is the log:"
+        kubectl logs $backup_pod_name -n $namespace
+        echo
+        exit 2
+        ;;
+      *)
+        sleep 10
+        continue
+        ;;
+    esac
+  done
+}
+
+function wait_for_migrate_done(){
+  migrate_pod_name="milvus-meta-migration-${instance_name}"
+  while true
+  do
+    status=$(kubectl get pod -n $namespace $migrate_pod_name --output=jsonpath={.status.phase})
+    case $status in
+      Succeeded)
+        echo "Migration is done..."
+        echo
+        break
+        ;;
+      Failed)
+        echo "Migration is failed..."
+        echo "Here is the log:"
+        kubectl logs $migrate_pod_name -n $namespace
+        echo
+        exit 3
+        ;;
+      *)
+        sleep 10
+        continue
+        ;;
+    esac
+  done
+}
+
+function wait_for_rollback_done(){
+  rollback_pod_name="milvus-meta-migration-rollback-${instance_name}"
+  while true
+  do
+    status=$(kubectl get pod -n $namespace $rollback_pod_name --output=jsonpath={.status.phase})
+    case $status in
+      Succeeded)
+        echo "Rollback is done..."
+        echo
+        break
+        ;;
+      Failed)
+        echo "Rollback is failed..."
+        echo "Here is the log:"
+        kubectl logs $rollback_pod_name -n $namespace
+        echo
+        exit 5
+        ;;
+      *)
+        sleep 10
+        continue
+        ;;
+    esac
+  done
+}
+function generate_migrate_meta_cm_and_pvc(){
+  etcd_svc=$(kubectl get svc -n $namespace -l app.kubernetes.io/instance=$instance_name,app.kubernetes.io/name=etcd -o name | grep -v headless | cut -d '/' -f 2)
+  cat <<EOF | kubectl apply -f -
+  apiVersion: v1
+  kind: PersistentVolumeClaim
+  metadata:
+    name: milvus-meta-migration-backup-${instance_name}
+  spec:
+    accessModes:
+    - ReadWriteOnce
+    resources:
+      requests:
+        storage: 10Gi
+EOF
+  cat <<EOF | kubectl apply -f -
+  apiVersion: v1
+  kind: ConfigMap
+  metadata:
+    name: milvus-meta-migration-config-${instance_name}
+  data:
+    backup.yaml: |+
+      cmd:
+        type: backup
+
+      config:
+        sourceVersion: $source_version
+        targetVersion: $target_version
+        backupFilePath: /milvus/data/migration.bak
+
+      metastore:
+        type: etcd
+
+      etcd:
+        endpoints:
+          - $etcd_svc:2379
+        rootPath: $root_path
+        metaSubPath: meta
+        kvSubPath: kv
+    migration.yaml: |+
+      cmd:
+        type: run
+
+      config:
+        sourceVersion: $source_version
+        targetVersion: $target_version
+        backupFilePath: /milvus/data/migration.bak
+
+      metastore:
+        type: etcd
+
+      etcd:
+        endpoints:
+          - $etcd_svc:2379
+        rootPath: $root_path
+        metaSubPath: meta
+        kvSubPath: kv
+    rollback.yaml: |+
+      cmd:
+        type: rollback
+
+      config:
+        sourceVersion: $source_version
+        targetVersion: $target_version
+        backupFilePath: /milvus/data/migration.bak
+
+      metastore:
+        type: etcd
+
+      etcd:
+        endpoints:
+          - $etcd_svc:2379
+        rootPath: $root_path
+        metaSubPath: meta
+        kvSubPath: kv
+EOF
+}
+
+function backup_meta(){
+  echo "Backuping meta..."
+  echo "Checking whether backup exists..."
+  configmap_name="milvus-meta-migration-config-${instance_name}"
+  backup_exists=$(kubectl get configmap $configmap_name -n $namespace --output=jsonpath={.metadata.annotations.backup})
+  if [ "$backup_exists" == "succeed" ]; then
+    echo "Found previous backups, skip meta backup..."
+  else
+    cat <<EOF | kubectl apply -f -
+    apiVersion: v1
+    kind: Pod
+    metadata:
+      name: milvus-meta-migration-backup-${instance_name}
+    spec:
+      restartPolicy: Never
+      containers:
+      - name: meta-migration
+        image: harbor.milvus.io/milvus/meta-migration:20221019-b79687687
+        command: ["/bin/sh"]
+        args:
+        - -c
+        - /milvus/bin/meta-migration -config=/milvus/configs/meta/backup.yaml
+        volumeMounts:
+        - name: backup
+          mountPath: /milvus/data
+        - name: config
+          mountPath: /milvus/configs/meta
+      volumes:
+      - name: backup
+        persistentVolumeClaim:
+          claimName: milvus-meta-migration-backup-${instance_name}
+      - name: config
+        configMap:
+          name: milvus-meta-migration-config-${instance_name}
+EOF
+    wait_for_backup_done
+  fi
+}
+
+function rollback_meta(){
+  generate_migrate_meta_cm_and_pvc
+  echo "Checking whether backup exists..."
+  backup_exists=$(kubectl get configmap milvus-meta-migration-config -n $namespace --output=jsonpath={.metadata.annotations.backup})
+  if [ "$backup_exists" == "succeed" ]; then
+    echo "Found previous backups, start meta rollback..."
+  cat <<EOF | kubectl apply -f -
+  apiVersion: v1
+  kind: Pod
+  metadata:
+    name: milvus-meta-migration-rollback-${instance_name}
+  spec:
+    restartPolicy: Never
+    containers:
+    - name: meta-migration
+      image: harbor.milvus.io/milvus/meta-migration:20221019-b79687687
+      command: ["/bin/sh"]
+      args:
+      - -c
+      - /milvus/bin/meta-migration -config=/milvus/configs/meta/rollback.yaml
+      volumeMounts:
+      - name: backup
+        mountPath: /milvus/data
+      - name: config
+        mountPath: /milvus/configs/meta
+    volumes:
+    - name: backup
+      persistentVolumeClaim:
+        claimName: milvus-meta-migration-backup-${instance_name}
+    - name: config
+      configMap:
+        name: milvus-meta-migration-config-${instance_name}
+EOF
+    wait_for_rollback_done
+  else
+    echo "No backup exists, abort..."
+    exit 4
+  fi
+}
+
+function migrate_meta(){
+  generate_migrate_meta_cm_and_pvc
+
+  backup_meta
+  echo "Migrating meta..."
+  cat <<EOF | kubectl apply -f -
+  apiVersion: v1
+  kind: Pod
+  metadata:
+    name: milvus-meta-migration-${instance_name}
+  spec:
+    restartPolicy: Never
+    containers:
+    - name: meta-migration
+      image: harbor.milvus.io/milvus/meta-migration:20221019-b79687687
+      command: ["/bin/sh"]
+      args:
+      - -c
+      - /milvus/bin/meta-migration -config=/milvus/configs/meta/migration.yaml
+      volumeMounts:
+      - name: backup
+        mountPath: /milvus/data
+      - name: config
+        mountPath: /milvus/configs/meta
+    volumes:
+    - name: backup
+      persistentVolumeClaim:
+        claimName: milvus-meta-migration-backup-${instance_name}
+    - name: config
+      configMap:
+        name: milvus-meta-migration-config-${instance_name}
+EOF
+  wait_for_migrate_done
+}
+
+function start_milvus_deploy(){
+  echo "Starting milvus components..."
+  for deploy in "${replicas[@]}"
+  do
+    IFS=':'
+    read -a item <<< "$deploy"
+    deploy_name="${item[0]}"
+    replica_count="${item[1]}"
+    echo "Starting $deploy_name..."
+    component=$(kubectl get deploy -n $namespace $deploy_name --output=jsonpath={.metadata.labels.component})
+    kubectl patch deployment/$deploy_name -n $namespace --patch-file=/dev/stdin <<-EOF
+    spec:
+      replicas: $replica_count
+      template:
+        spec:
+          containers:
+          - name: $component
+            image: $image_tag
+EOF
+  done
+}
+
+function wait_for_milvus_ready(){
+  total_component_count=${#replicas[@]}
+  while true
+  do
+    ready_count=0
+    for deploy in "${replicas[@]}"
+    do 
+      IFS=':'
+      read -a item <<< "$deploy"
+      deploy_name="${item[0]}"
+      replica_count="${item[1]}"
+      count=$(kubectl get deploy $deploy_name -n $namespace --output=jsonpath={.status.readyReplicas})
+      if [ "$count" == "$replica_count" ] ; then
+        ready_count=`expr $ready_count + 1` 
+      fi
+    done
+    if [ "$total_component_count" == "$ready_count" ]; then
+      break
+    fi
+    sleep 5
+  done
+}
+
+deployments=$(kubectl get deploy -n $namespace -l app.kubernetes.io/instance=$instance_name,app.kubernetes.io/name=milvus --output=jsonpath={.items..metadata.name})
+if [ ${#deployments} == 0 ]; then
+	echo "There is no Milvus instance $instance_name in the namespace $namespace"
+else
+	 fetch_deploy_replica "${deployments[*]}"
+   stop_milvus_deploy "$deployments"
+   echo 
+   case $operation in
+     migrate)
+       echo "Starting to migrate milvus meta..."
+       migrate_meta
+       start_milvus_deploy
+       echo
+       echo "Upgrading is done. Waiting for milvus components to be ready again..."
+       wait_for_milvus_ready
+       echo "All milvus components are running. Enjoy your vector search..."
+       ;;
+     rollback)
+       echo "Starting to rollback milvus meta..."
+       rollback_meta
+       start_milvus_deploy
+       echo
+       echo "Rollbacking is done. Waiting for milvus components to be ready again..."
+       wait_for_milvus_ready
+       echo "All milvus components are running. Enjoy your vector search..."
+       ;;
+     *)
+       echo "Invalid operation..."
+       exit 6
+       ;;
+   esac
+fi