admin/milvus
1
0
Fork 0
mirror of https://gitee.com/milvus-io/milvus.git synced 2025-12-06 17:18:35 +08:00
Code Issues Packages Projects Releases Wiki Activity

[skii e2e] Add scale test pipeline server logs archive (#15234)

Browse Source 
Signed-off-by: ThreadDao <yufen.zong@zilliz.com>
This commit is contained in:
ThreadDao 2022-01-18 09:21:36 +08:00 committed by GitHub
parent 1b1d4e502b
commit 2abc3b3260
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 169 additions and 118 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
build/ci/jenkins/Scale.groovy
View File

@ -31,6 +31,7 @@ pipeline {
TEST_TYPE = "scale-test"
// SEMVER = "${BRANCH_NAME.contains('/') ? BRANCH_NAME.substring(BRANCH_NAME.lastIndexOf('/') + 1) : BRANCH_NAME}"
ARTIFACTS = "${env.WORKSPACE}/_artifacts"
MILVUS_LOGS = "/tmp/milvus_logs/*"
}
stages {
@ -77,7 +78,16 @@ pipeline {
script {
dir("${env.ARTIFACTS}") {
sh "tar -zcvf artifacts-${PROJECT_NAME}-${TEST_TYPE}-pytest-logs.tar.gz /tmp/ci_logs --remove-files || true"
archiveArtifacts artifacts: "artifacts-${PROJECT_NAME}-${TEST_TYPE}-pytest-logs.tar.gz ", allowEmptyArchive: true }
archiveArtifacts artifacts: "artifacts-${PROJECT_NAME}-${TEST_TYPE}-pytest-logs.tar.gz ", allowEmptyArchive: true
DIR_LIST = sh(returnStdout: true, script: 'ls -d1 ${MILVUS_LOGS}').trim()
for (d in DIR_LIST.tokenize("\n")) {
sh "echo $d"
def release_name = d.split('/')[-1]
sh "tar -zcvf artifacts-${PROJECT_NAME}-${TEST_TYPE}-${release_name}-logs.tar.gz ${d} --remove-files || true"
archiveArtifacts artifacts: "artifacts-${PROJECT_NAME}-${TEST_TYPE}-${release_name}-logs.tar.gz ", allowEmptyArchive: true
}
}
}
}
}

11
tests/python_client/scale/test_data_node_scale.py
View File

@ -10,7 +10,7 @@ from customize.milvus_operator import MilvusOperator
from scale import constants
from pymilvus import connections
from utils.util_log import test_log as log
from utils.util_k8s import wait_pods_ready
from utils.util_k8s import wait_pods_ready, export_pod_logs
from utils.util_pymilvus import get_latest_tag
prefix = "data_scale"
@ -55,6 +55,7 @@ class TestDataNodeScale:
host = mic.endpoint(release_name, constants.NAMESPACE).split(':')[0]
# host = '10.98.0.4'
try:
# connect
connections.add_connection(default={"host": host, "port": 19530})
connections.connect(alias='default')
@ -103,4 +104,12 @@ class TestDataNodeScale:
time.sleep(300)
log.debug("Shrink dataNode test finished")
except Exception as e:
raise Exception(str(e))
finally:
label = f"app.kubernetes.io/instance={release_name}"
log.info('Start to export milvus pod logs')
export_pod_logs(namespace=constants.NAMESPACE, label_selector=label, release_name=release_name)
mic.uninstall(release_name, namespace=constants.NAMESPACE)

7
tests/python_client/scale/test_index_node_scale.py
View File

@ -10,6 +10,7 @@ from customize.milvus_operator import MilvusOperator
from scale import constants
from common import common_func as cf
from common import common_type as ct
from utils.util_k8s import export_pod_logs
from utils.util_log import test_log as log
from utils.util_pymilvus import get_latest_tag
@ -97,6 +98,9 @@ class TestIndexNodeScale:
raise Exception(str(e))
finally:
label = f"app.kubernetes.io/instance={release_name}"
log.info('Start to export milvus pod logs')
export_pod_logs(namespace=constants.NAMESPACE, label_selector=label, release_name=release_name)
mic.uninstall(release_name, namespace=constants.NAMESPACE)
@pytest.mark.tags(CaseLabel.L3)
@ -175,4 +179,7 @@ class TestIndexNodeScale:
raise Exception(str(e))
finally:
label = f"app.kubernetes.io/instance={release_name}"
log.info('Start to export milvus pod logs')
export_pod_logs(namespace=constants.NAMESPACE, label_selector=label, release_name=release_name)
mic.uninstall(release_name, namespace=constants.NAMESPACE)

10
tests/python_client/scale/test_proxy_scale.py
View File

@ -6,7 +6,7 @@ from common import common_func as cf
from common.common_type import CaseLabel
from scale import scale_common as sc, constants
from utils.util_log import test_log as log
from utils.util_k8s import wait_pods_ready
from utils.util_k8s import wait_pods_ready, export_pod_logs
from utils.util_pymilvus import get_latest_tag
prefix = "proxy_scale"
@ -56,6 +56,7 @@ class TestProxyScale:
host = mic.endpoint(release_name, constants.NAMESPACE).split(':')[0]
# host = "10.98.0.7"
try:
c_name = cf.gen_unique_str(prefix)
self.e2e_milvus_parallel(5, host, c_name)
log.info('Milvus test before expand')
@ -74,4 +75,11 @@ class TestProxyScale:
self.e2e_milvus_parallel(2, host, c_name)
log.info('Milvus test after shrink')
except Exception as e:
raise Exception(str(e))
finally:
label = f"app.kubernetes.io/instance={release_name}"
log.info('Start to export milvus pod logs')
export_pod_logs(namespace=constants.NAMESPACE, label_selector=label, release_name=release_name)
mic.uninstall(release_name, namespace=constants.NAMESPACE)

10
tests/python_client/scale/test_query_node_scale.py
View File

@ -11,7 +11,7 @@ from common import common_type as ct
from scale import constants
from pymilvus import Index, connections
from utils.util_log import test_log as log
from utils.util_k8s import wait_pods_ready
from utils.util_k8s import wait_pods_ready, export_pod_logs
from utils.util_pymilvus import get_latest_tag
prefix = "search_scale"
@ -55,6 +55,7 @@ class TestQueryNodeScale:
host = mic.endpoint(release_name, constants.NAMESPACE).split(':')[0]
# host = "10.98.0.8"
try:
# connect
connections.add_connection(default={"host": host, "port": 19530})
connections.connect(alias='default')
@ -121,4 +122,11 @@ class TestQueryNodeScale:
time.sleep(60)
log.debug("Shrink querynode test finished")
except Exception as e:
raise Exception(str(e))
finally:
label = f"app.kubernetes.io/instance={release_name}"
log.info('Start to export milvus pod logs')
export_pod_logs(namespace=constants.NAMESPACE, label_selector=label, release_name=release_name)
mic.uninstall(release_name, namespace=constants.NAMESPACE)

13
tests/python_client/utils/util_k8s.py
View File

@ -82,7 +82,7 @@ def get_pod_list(namespace, label_selector):
raise Exception(str(e))
def export_pod_logs(namespace, label_selector):
def export_pod_logs(namespace, label_selector, release_name=None):
"""
export pod logs with label selector to '/tmp/milvus'
@ -92,10 +92,19 @@ def export_pod_logs(namespace, label_selector):
:param label_selector: labels to restrict which pods logs to export
:type label_selector: str
:param release_name: use the release name as server logs director name
:type label_selector: str
:example:
>>> export_pod_logs("chaos-testing", "app.kubernetes.io/instance=mic-milvus")
"""
pod_log_path = '/tmp/milvus_logs'
if isinstance(release_name, str):
if len(release_name.strip()) == 0:
raise ValueError("Got an unexpected space release_name")
else:
raise TypeError("Got an unexpected non-string release_name")
pod_log_path = '/tmp/milvus_logs' if release_name is None else f'/tmp/milvus_logs/{release_name}'
if not os.path.isdir(pod_log_path):
os.makedirs(pod_log_path)