[skip e2e] Update scale test image tag to dynamic (#15040)

Signed-off-by: ThreadDao <yufen.zong@zilliz.com>

tests/python_client/scale/constants.py

@@ -1,5 +1,6 @@
 # scale object
-IMAGE_REPOSITORY = "registry.milvus.io/milvus/milvus"  # repository of milvus image
+# IMAGE_REPOSITORY = "registry.milvus.io/milvus/milvus"  # repository of milvus image
+IMAGE_REPOSITORY = "milvusdb/milvus-dev"
 IMAGE_TAG = "master-20211227-b022615"  # tag of milvus image
 NAMESPACE = "chaos-testing"  # namespace
 IF_NOT_PRESENT = "IfNotPresent"  # image pullPolicy IfNotPresent

tests/python_client/scale/test_data_node_scale.py

@@ -11,6 +11,7 @@ from scale import constants
 from pymilvus import connections
 from utils.util_log import test_log as log
 from utils.util_k8s import wait_pods_ready
+from utils.util_pymilvus import get_latest_tag
 
 prefix = "data_scale"
 default_schema = cf.gen_default_collection_schema()
@@ -35,7 +36,9 @@ class TestDataNodeScale:
                   Average dataNode memory usage
         """
         release_name = "scale-data"
-        image = f'{constants.IMAGE_REPOSITORY}:{constants.IMAGE_TAG}'
+        image_tag = get_latest_tag()
+        image = f'{constants.IMAGE_REPOSITORY}:{image_tag}'
+
         data_config = {
             'metadata.namespace': constants.NAMESPACE,
             'metadata.name': release_name,

tests/python_client/scale/test_index_node_scale.py

@@ -11,6 +11,7 @@ from scale import constants
 from common import common_func as cf
 from common import common_type as ct
 from utils.util_log import test_log as log
+from utils.util_pymilvus import get_latest_tag
 
 nb = 5000
 default_index_params = {"index_type": "IVF_SQ8", "metric_type": "L2", "params": {"nlist": 128}}
@@ -29,7 +30,8 @@ class TestIndexNodeScale:
         expected: The cost of one indexNode is about twice that of two indexNodes
         """
         release_name = "expand-index"
-        image = f'{constants.IMAGE_REPOSITORY}:{constants.IMAGE_TAG}'
+        image_tag = get_latest_tag()
+        image = f'{constants.IMAGE_REPOSITORY}:{image_tag}'
         init_replicas = 1
         expand_replicas = 2
         data_config = {
@@ -108,7 +110,8 @@ class TestIndexNodeScale:
         expected: The cost of one indexNode is about twice that of two indexNodes
         """
         release_name = "shrink-index"
-        image = f'{constants.IMAGE_REPOSITORY}:{constants.IMAGE_TAG}'
+        image_tag = get_latest_tag()
+        image = f'{constants.IMAGE_REPOSITORY}:{image_tag}'
         data_config = {
             'metadata.namespace': constants.NAMESPACE,
             'metadata.name': release_name,

tests/python_client/scale/test_proxy_scale.py

@@ -7,6 +7,7 @@ from common.common_type import CaseLabel
 from scale import scale_common as sc, constants
 from utils.util_log import test_log as log
 from utils.util_k8s import wait_pods_ready
+from utils.util_pymilvus import get_latest_tag
 
 prefix = "proxy_scale"
 
@@ -36,7 +37,8 @@ class TestProxyScale:
         """
         # deploy milvus cluster with one proxy
         release_name = "scale-proxy"
-        image = f'{constants.IMAGE_REPOSITORY}:{constants.IMAGE_TAG}'
+        image_tag = get_latest_tag()
+        image = f'{constants.IMAGE_REPOSITORY}:{image_tag}'
         data_config = {
             'metadata.namespace': constants.NAMESPACE,
             'metadata.name': release_name,

tests/python_client/scale/test_query_node_scale.py

@@ -12,6 +12,7 @@ from scale import constants
 from pymilvus import Index, connections
 from utils.util_log import test_log as log
 from utils.util_k8s import wait_pods_ready
+from utils.util_pymilvus import get_latest_tag
 
 prefix = "search_scale"
 nb = 5000
@@ -36,7 +37,8 @@ class TestQueryNodeScale:
         expected: Verify milvus remains healthy and search successfully during scale
         """
         release_name = "scale-query"
-        image = f'{constants.IMAGE_REPOSITORY}:{constants.IMAGE_TAG}'
+        image_tag = get_latest_tag()
+        image = f'{constants.IMAGE_REPOSITORY}:{image_tag}'
         query_config = {
             'metadata.namespace': constants.NAMESPACE,
             'metadata.name': release_name,

tests/python_client/utils/util_pymilvus.py

@@ -1,3 +1,4 @@
+import json
 import random
 import string
 import threading
@@ -5,6 +6,7 @@ import traceback
 import time
 import copy
 import numpy as np
+import requests
 from sklearn import preprocessing
 from pymilvus import Milvus, DataType
 from utils.util_log import test_log as log
@@ -1007,6 +1009,99 @@ def compare_list_elements(_first, _second):
     return True
 
 
+def get_token(url):
+    rep = requests.get(url)
+    data = json.loads(rep.text)
+    if 'token' in data:
+        token = data['token']
+    else:
+        token = ''
+        print("Can not get token.")
+    return token
+
+
+def get_tags(url, token):
+    headers = {'Content-type': "application/json",
+               "charset": "UTF-8",
+               "Accept": "application/vnd.docker.distribution.manifest.v2+json",
+               "Authorization": "Bearer %s" % token}
+    try:
+        rep = requests.get(url, headers=headers)
+        data = json.loads(rep.text)
+
+        tags = []
+        if 'tags' in data:
+            tags = data["tags"]
+        else:
+            print("Can not get the tag list")
+        return tags
+    except:
+        print("Can not get the tag list")
+        return []
+
+
+def get_master_tags(tags_list):
+    _list = []
+
+    if not isinstance(tags_list, list):
+        print("tags_list is not a list.")
+        return _list
+
+    for tag in tags_list:
+        if "master" in tag and tag != "master-latest":
+            _list.append(tag)
+    return _list
+
+
+def get_config_digest(url, token):
+    headers = {'Content-type': "application/json",
+               "charset": "UTF-8",
+               "Accept": "application/vnd.docker.distribution.manifest.v2+json",
+               "Authorization": "Bearer %s" % token}
+    try:
+        rep = requests.get(url, headers=headers)
+        data = json.loads(rep.text)
+
+        digest = ''
+        if 'config' in data and 'digest' in data["config"]:
+            digest = data["config"]["digest"]
+        else:
+            print("Can not get the digest")
+        return digest
+    except:
+        print("Can not get the digest")
+        return ""
+
+
+def get_latest_tag(limit=100):
+    service = "registry.docker.io"
+    repository = "milvusdb/milvus-dev"
+
+    auth_url = "https://auth.docker.io/token?service=%s&scope=repository:%s:pull" % (service, repository)
+    tags_url = "https://index.docker.io/v2/%s/tags/list" % repository
+    tag_url = "https://index.docker.io/v2/milvusdb/milvus-dev/manifests/"
+
+    master_latest_digest = get_config_digest(tag_url + "master-latest", get_token(auth_url))
+    tags = get_tags(tags_url, get_token(auth_url))
+    tag_list = get_master_tags(tags)
+
+    latest_tag = ""
+    for i in range(1, len(tag_list) + 1):
+        tag_name = str(tag_list[-i])
+        tag_digest = get_config_digest(tag_url + tag_name, get_token(auth_url))
+        if tag_digest == master_latest_digest:
+            latest_tag = tag_name
+            break
+        if i > limit:
+            break
+
+    if latest_tag == "":
+        latest_tag = "master-latest"
+        print("Can't find the latest image name")
+    print("The image name used is %s" % str(latest_tag))
+    return latest_tag
+
+
 class MyThread(threading.Thread):
     def __init__(self, target, args=()):
         threading.Thread.__init__(self, target=target, args=args)
@@ -1023,4 +1118,3 @@ class MyThread(threading.Thread):
         super(MyThread, self).join()
         if self.exc:
             raise self.exc
-