[skii e2e] Add scale test pipeline server logs archive (#15234)

Signed-off-by: ThreadDao <yufen.zong@zilliz.com>

build/ci/jenkins/Scale.groovy

@@ -31,6 +31,7 @@ pipeline {
         TEST_TYPE = "scale-test"
 //        SEMVER = "${BRANCH_NAME.contains('/') ? BRANCH_NAME.substring(BRANCH_NAME.lastIndexOf('/') + 1) : BRANCH_NAME}"
         ARTIFACTS = "${env.WORKSPACE}/_artifacts"
+        MILVUS_LOGS = "/tmp/milvus_logs/*"
     }
 
     stages {
@@ -77,7 +78,16 @@ pipeline {
                     script {
                         dir("${env.ARTIFACTS}") {
                             sh "tar -zcvf artifacts-${PROJECT_NAME}-${TEST_TYPE}-pytest-logs.tar.gz /tmp/ci_logs --remove-files || true"
-                            archiveArtifacts artifacts: "artifacts-${PROJECT_NAME}-${TEST_TYPE}-pytest-logs.tar.gz ", allowEmptyArchive: true                        }
+                            archiveArtifacts artifacts: "artifacts-${PROJECT_NAME}-${TEST_TYPE}-pytest-logs.tar.gz ", allowEmptyArchive: true
+                            DIR_LIST = sh(returnStdout: true, script: 'ls -d1 ${MILVUS_LOGS}').trim()
+                            for (d in DIR_LIST.tokenize("\n")) {
+                                sh "echo $d"
+                                def release_name = d.split('/')[-1]
+                                sh "tar -zcvf artifacts-${PROJECT_NAME}-${TEST_TYPE}-${release_name}-logs.tar.gz ${d} --remove-files || true"
+                                archiveArtifacts artifacts: "artifacts-${PROJECT_NAME}-${TEST_TYPE}-${release_name}-logs.tar.gz ", allowEmptyArchive: true
+
+                            }
+                        }
                     }
                 }
             }

tests/python_client/scale/test_data_node_scale.py

@@ -10,7 +10,7 @@ from customize.milvus_operator import MilvusOperator
 from scale import constants
 from pymilvus import connections
 from utils.util_log import test_log as log
-from utils.util_k8s import wait_pods_ready
+from utils.util_k8s import wait_pods_ready, export_pod_logs
 from utils.util_pymilvus import get_latest_tag
 
 prefix = "data_scale"
@@ -55,6 +55,7 @@ class TestDataNodeScale:
         host = mic.endpoint(release_name, constants.NAMESPACE).split(':')[0]
         # host = '10.98.0.4'
 
+        try:
             # connect
             connections.add_connection(default={"host": host, "port": 19530})
             connections.connect(alias='default')
@@ -103,4 +104,12 @@ class TestDataNodeScale:
             time.sleep(300)
             log.debug("Shrink dataNode test finished")
 
+        except Exception as e:
+            raise Exception(str(e))
+
+        finally:
+            label = f"app.kubernetes.io/instance={release_name}"
+            log.info('Start to export milvus pod logs')
+            export_pod_logs(namespace=constants.NAMESPACE, label_selector=label, release_name=release_name)
+
             mic.uninstall(release_name, namespace=constants.NAMESPACE)

tests/python_client/scale/test_index_node_scale.py

@@ -10,6 +10,7 @@ from customize.milvus_operator import MilvusOperator
 from scale import constants
 from common import common_func as cf
 from common import common_type as ct
+from utils.util_k8s import export_pod_logs
 from utils.util_log import test_log as log
 from utils.util_pymilvus import get_latest_tag
 
@@ -97,6 +98,9 @@ class TestIndexNodeScale:
             raise Exception(str(e))
 
         finally:
+            label = f"app.kubernetes.io/instance={release_name}"
+            log.info('Start to export milvus pod logs')
+            export_pod_logs(namespace=constants.NAMESPACE, label_selector=label, release_name=release_name)
             mic.uninstall(release_name, namespace=constants.NAMESPACE)
 
     @pytest.mark.tags(CaseLabel.L3)
@@ -175,4 +179,7 @@ class TestIndexNodeScale:
             raise Exception(str(e))
 
         finally:
+            label = f"app.kubernetes.io/instance={release_name}"
+            log.info('Start to export milvus pod logs')
+            export_pod_logs(namespace=constants.NAMESPACE, label_selector=label, release_name=release_name)
             mic.uninstall(release_name, namespace=constants.NAMESPACE)

tests/python_client/scale/test_proxy_scale.py

@@ -6,7 +6,7 @@ from common import common_func as cf
 from common.common_type import CaseLabel
 from scale import scale_common as sc, constants
 from utils.util_log import test_log as log
-from utils.util_k8s import wait_pods_ready
+from utils.util_k8s import wait_pods_ready, export_pod_logs
 from utils.util_pymilvus import get_latest_tag
 
 prefix = "proxy_scale"
@@ -56,6 +56,7 @@ class TestProxyScale:
         host = mic.endpoint(release_name, constants.NAMESPACE).split(':')[0]
         # host = "10.98.0.7"
 
+        try:
             c_name = cf.gen_unique_str(prefix)
             self.e2e_milvus_parallel(5, host, c_name)
             log.info('Milvus test before expand')
@@ -74,4 +75,11 @@ class TestProxyScale:
             self.e2e_milvus_parallel(2, host, c_name)
             log.info('Milvus test after shrink')
 
+        except Exception as e:
+            raise Exception(str(e))
+
+        finally:
+            label = f"app.kubernetes.io/instance={release_name}"
+            log.info('Start to export milvus pod logs')
+            export_pod_logs(namespace=constants.NAMESPACE, label_selector=label, release_name=release_name)
             mic.uninstall(release_name, namespace=constants.NAMESPACE)

tests/python_client/scale/test_query_node_scale.py

@@ -11,7 +11,7 @@ from common import common_type as ct
 from scale import constants
 from pymilvus import Index, connections
 from utils.util_log import test_log as log
-from utils.util_k8s import wait_pods_ready
+from utils.util_k8s import wait_pods_ready, export_pod_logs
 from utils.util_pymilvus import get_latest_tag
 
 prefix = "search_scale"
@@ -55,6 +55,7 @@ class TestQueryNodeScale:
         host = mic.endpoint(release_name, constants.NAMESPACE).split(':')[0]
         # host = "10.98.0.8"
 
+        try:
             # connect
             connections.add_connection(default={"host": host, "port": 19530})
             connections.connect(alias='default')
@@ -121,4 +122,11 @@ class TestQueryNodeScale:
             time.sleep(60)
             log.debug("Shrink querynode test finished")
 
+        except Exception as e:
+            raise Exception(str(e))
+
+        finally:
+            label = f"app.kubernetes.io/instance={release_name}"
+            log.info('Start to export milvus pod logs')
+            export_pod_logs(namespace=constants.NAMESPACE, label_selector=label, release_name=release_name)
             mic.uninstall(release_name, namespace=constants.NAMESPACE)

tests/python_client/utils/util_k8s.py

@@ -82,7 +82,7 @@ def get_pod_list(namespace, label_selector):
         raise Exception(str(e))
 
 
-def export_pod_logs(namespace, label_selector):
+def export_pod_logs(namespace, label_selector, release_name=None):
     """
     export pod logs with label selector to '/tmp/milvus'
 
@@ -92,10 +92,19 @@ def export_pod_logs(namespace, label_selector):
     :param label_selector: labels to restrict which pods logs to export
     :type label_selector: str
 
+    :param release_name: use the release name as server logs director name
+    :type label_selector: str
+
     :example:
             >>> export_pod_logs("chaos-testing", "app.kubernetes.io/instance=mic-milvus")
     """
-    pod_log_path = '/tmp/milvus_logs'
+    if isinstance(release_name, str):
+        if len(release_name.strip()) == 0:
+            raise ValueError("Got an unexpected space release_name")
+    else:
+        raise TypeError("Got an unexpected non-string release_name")
+    pod_log_path = '/tmp/milvus_logs' if release_name is None else f'/tmp/milvus_logs/{release_name}'
+
     if not os.path.isdir(pod_log_path):
         os.makedirs(pod_log_path)