diff --git a/tests/python_client/scale/test_data_node_scale.py b/tests/python_client/scale/test_data_node_scale.py index a1f15450ae..59aa7ca9bd 100644 --- a/tests/python_client/scale/test_data_node_scale.py +++ b/tests/python_client/scale/test_data_node_scale.py @@ -10,7 +10,7 @@ from customize.milvus_operator import MilvusOperator from scale import constants from pymilvus import connections from utils.util_log import test_log as log -from utils.util_k8s import wait_pods_ready, export_pod_logs +from utils.util_k8s import wait_pods_ready, read_pod_log from utils.util_pymilvus import get_latest_tag prefix = "data_scale" @@ -110,6 +110,6 @@ class TestDataNodeScale: finally: label = f"app.kubernetes.io/instance={release_name}" log.info('Start to export milvus pod logs') - export_pod_logs(namespace=constants.NAMESPACE, label_selector=label, release_name=release_name) + read_pod_log(namespace=constants.NAMESPACE, label_selector=label, release_name=release_name) mic.uninstall(release_name, namespace=constants.NAMESPACE) diff --git a/tests/python_client/scale/test_index_node_scale.py b/tests/python_client/scale/test_index_node_scale.py index 84e31893d5..2621752e5b 100644 --- a/tests/python_client/scale/test_index_node_scale.py +++ b/tests/python_client/scale/test_index_node_scale.py @@ -10,7 +10,7 @@ from customize.milvus_operator import MilvusOperator from scale import constants from common import common_func as cf from common import common_type as ct -from utils.util_k8s import export_pod_logs +from utils.util_k8s import read_pod_log from utils.util_log import test_log as log from utils.util_pymilvus import get_latest_tag @@ -100,7 +100,7 @@ class TestIndexNodeScale: finally: label = f"app.kubernetes.io/instance={release_name}" log.info('Start to export milvus pod logs') - export_pod_logs(namespace=constants.NAMESPACE, label_selector=label, release_name=release_name) + read_pod_log(namespace=constants.NAMESPACE, label_selector=label, release_name=release_name) mic.uninstall(release_name, namespace=constants.NAMESPACE) @pytest.mark.tags(CaseLabel.L3) @@ -181,5 +181,5 @@ class TestIndexNodeScale: finally: label = f"app.kubernetes.io/instance={release_name}" log.info('Start to export milvus pod logs') - export_pod_logs(namespace=constants.NAMESPACE, label_selector=label, release_name=release_name) + read_pod_log(namespace=constants.NAMESPACE, label_selector=label, release_name=release_name) mic.uninstall(release_name, namespace=constants.NAMESPACE) diff --git a/tests/python_client/scale/test_proxy_scale.py b/tests/python_client/scale/test_proxy_scale.py index b176b646c8..c42e0a585e 100644 --- a/tests/python_client/scale/test_proxy_scale.py +++ b/tests/python_client/scale/test_proxy_scale.py @@ -6,7 +6,7 @@ from common import common_func as cf from common.common_type import CaseLabel from scale import scale_common as sc, constants from utils.util_log import test_log as log -from utils.util_k8s import wait_pods_ready, export_pod_logs +from utils.util_k8s import wait_pods_ready, read_pod_log from utils.util_pymilvus import get_latest_tag prefix = "proxy_scale" @@ -81,5 +81,5 @@ class TestProxyScale: finally: label = f"app.kubernetes.io/instance={release_name}" log.info('Start to export milvus pod logs') - export_pod_logs(namespace=constants.NAMESPACE, label_selector=label, release_name=release_name) + read_pod_log(namespace=constants.NAMESPACE, label_selector=label, release_name=release_name) mic.uninstall(release_name, namespace=constants.NAMESPACE) diff --git a/tests/python_client/scale/test_query_node_scale.py b/tests/python_client/scale/test_query_node_scale.py index 720c2eb57d..4d587c9a6d 100644 --- a/tests/python_client/scale/test_query_node_scale.py +++ b/tests/python_client/scale/test_query_node_scale.py @@ -11,7 +11,7 @@ from common import common_type as ct from scale import constants from pymilvus import Index, connections from utils.util_log import test_log as log -from utils.util_k8s import wait_pods_ready, export_pod_logs +from utils.util_k8s import wait_pods_ready, read_pod_log from utils.util_pymilvus import get_latest_tag prefix = "search_scale" @@ -128,5 +128,5 @@ class TestQueryNodeScale: finally: label = f"app.kubernetes.io/instance={release_name}" log.info('Start to export milvus pod logs') - export_pod_logs(namespace=constants.NAMESPACE, label_selector=label, release_name=release_name) + read_pod_log(namespace=constants.NAMESPACE, label_selector=label, release_name=release_name) mic.uninstall(release_name, namespace=constants.NAMESPACE) \ No newline at end of file diff --git a/tests/python_client/utils/util_k8s.py b/tests/python_client/utils/util_k8s.py index 2cb394f78b..daf575310c 100644 --- a/tests/python_client/utils/util_k8s.py +++ b/tests/python_client/utils/util_k8s.py @@ -38,7 +38,7 @@ def wait_pods_ready(namespace, label_selector, expected_num=None, timeout=360): for item in api_response.items: if item.status.phase != 'Running': all_pos_ready_flag = False - break + break for c in item.status.container_statuses: log.info(f"{c.name} status is {c.ready}") if c.ready is False: @@ -119,6 +119,30 @@ def export_pod_logs(namespace, label_selector, release_name=None): raise Exception(str(e)) +def read_pod_log(namespace, label_selector, release_name): + config.load_kube_config() + items = get_pod_list(namespace, label_selector=label_selector) + + try: + # export log to /tmp/release_name path + pod_log_path = f'/tmp/milvus_logs/{release_name}' + if not os.path.isdir(pod_log_path): + os.makedirs(pod_log_path) + + api_instance = client.CoreV1Api() + + for item in items: + pod = item.metadata.name + log.debug(f'Start to read {pod} log') + logs = api_instance.read_namespaced_pod_log(name=pod, namespace=namespace, async_req=True) + with open(f'{pod_log_path}/{pod}.log', "w") as f: + f.write(logs.get()) + + except ApiException as e: + log.error(f"Exception when read pod {pod} logs: %s\n" % e) + raise Exception(str(e)) + + if __name__ == '__main__': label = "app.kubernetes.io/instance=test-proxy-pod-failure, component=proxy" res = get_pod_list("chaos-testing", label_selector=label)