From 703d90f7864a24d0356659756aaba729a9779a9f Mon Sep 17 00:00:00 2001 From: eleanorjboyd Date: Wed, 20 Nov 2024 14:56:29 -0800 Subject: [PATCH 1/7] flaky --- python_files/vscode_pytest/__init__.py | 70 +++++++++++++++++++++++++- 1 file changed, 68 insertions(+), 2 deletions(-) diff --git a/python_files/vscode_pytest/__init__.py b/python_files/vscode_pytest/__init__.py index 59a1b75e9688..553be4019919 100644 --- a/python_files/vscode_pytest/__init__.py +++ b/python_files/vscode_pytest/__init__.py @@ -33,6 +33,17 @@ USES_PYTEST_DESCRIBE = True +sys.path.append("/Users/eleanorboyd/vscode-python/.nox/install_python_libs/lib/python3.10") +sys.path.append("/Users/eleanorboyd/vscode-python-debugger") +sys.path.append("/Users/eleanorboyd/vscode-python-debugger/bundled") +sys.path.append("/Users/eleanorboyd/vscode-python-debugger/bundled/libs") + +import debugpy # noqa: E402 + +debugpy.connect(5678) +debugpy.breakpoint() # noqa: E702 + + class TestData(TypedDict): """A general class that all test objects inherit from.""" @@ -68,6 +79,7 @@ def __init__(self, message): collected_tests_so_far = [] TEST_RUN_PIPE = os.getenv("TEST_RUN_PIPE") SYMLINK_PATH = None +FLAKY_MAX_RUNS = None def pytest_load_initial_conftests(early_config, parser, args): # noqa: ARG001 @@ -92,6 +104,10 @@ def pytest_load_initial_conftests(early_config, parser, args): # noqa: ARG001 global IS_DISCOVERY IS_DISCOVERY = True + # set the reruns value, -1 if not set + global FLAKY_MAX_RUNS + FLAKY_MAX_RUNS = get_reruns_value(args) + # check if --rootdir is in the args for arg in args: if "--rootdir=" in arg: @@ -120,6 +136,29 @@ def pytest_load_initial_conftests(early_config, parser, args): # noqa: ARG001 SYMLINK_PATH = rootdir +def get_reruns_value(args): + """ + Extracts the value of the --reruns argument from a list of command-line arguments. + + Args: + args (list of str): A list of command-line arguments. + + Returns: + int: The integer value of the --reruns argument if found, otherwise -1. + """ + for arg in args: + if arg.startswith("--reruns"): + if "=" in arg: + # Extract the value from --reruns= + return int(arg.split("=")[1]) + else: + # Get the value from the next argument + index = args.index(arg) + if index + 1 < len(args): + return int(args[index + 1]) + return -1 + + def pytest_internalerror(excrepr, excinfo): # noqa: ARG001 """A pytest hook that is called when an internal error occurs. @@ -150,11 +189,37 @@ def pytest_exception_interact(node, call, report): else: ERRORS.append(report.longreprtext + "\n Check Python Test Logs for more details.") else: + node_id = get_absolute_test_id(node.nodeid, get_node_path(node)) + # Check if Pytest-rerunfailures is enabled + # check # run this is + # check # of reruns allowed + # if # of reruns is reached, then send the error message, otherwise do not send the error message + + try: + exec_count = node.execution_count + # global is set if arg is present during pytest_load_initial_conftests + # if not present, then -1 + flaky_max_runs = FLAKY_MAX_RUNS + + # check for rerunfailures marker + for m in node.own_markers: + if m.name == "flaky": + flaky_max_runs = m.kwargs.get("reruns", 0) + break + + # flaky_max_runs != -1 means test is flaky + if flaky_max_runs != -1 and exec_count <= flaky_max_runs: + return + elif flaky_max_runs != -1 and exec_count > flaky_max_runs: + print("Plugin info[vscode-pytest]: max reruns reached.") + except AttributeError: + pass + # If during execution, send this data that the given node failed. report_value = "error" if call.excinfo.typename == "AssertionError": report_value = "failure" - node_id = get_absolute_test_id(node.nodeid, get_node_path(node)) + # Only add test to collected_tests_so_far if it is not a flaky test that will re-run if node_id not in collected_tests_so_far: collected_tests_so_far.append(node_id) item_result = create_test_outcome( @@ -280,7 +345,8 @@ def pytest_report_teststatus(report, config): # noqa: ARG001 node_path = cwd # Calculate the absolute test id and use this as the ID moving forward. absolute_node_id = get_absolute_test_id(report.nodeid, node_path) - if absolute_node_id not in collected_tests_so_far: + # If the test is not a rerun, add it to the collected_tests_so_far list. + if report.outcome != "rerun" and absolute_node_id not in collected_tests_so_far: collected_tests_so_far.append(absolute_node_id) item_result = create_test_outcome( absolute_node_id, From fd43b9039c093e1bc657fc6e82dbd0bc734305f6 Mon Sep 17 00:00:00 2001 From: eleanorjboyd Date: Thu, 21 Nov 2024 08:18:09 -0800 Subject: [PATCH 2/7] add test --- .../.data/test_rerunfailures_plugin.py | 13 +++++++++++++ .../expected_execution_test_output.py | 15 +++++++++++++++ .../tests/pytestadapter/test_execution.py | 7 +++++++ python_files/vscode_pytest/__init__.py | 10 ---------- 4 files changed, 35 insertions(+), 10 deletions(-) create mode 100644 python_files/tests/pytestadapter/.data/test_rerunfailures_plugin.py diff --git a/python_files/tests/pytestadapter/.data/test_rerunfailures_plugin.py b/python_files/tests/pytestadapter/.data/test_rerunfailures_plugin.py new file mode 100644 index 000000000000..93d58378ffa1 --- /dev/null +++ b/python_files/tests/pytestadapter/.data/test_rerunfailures_plugin.py @@ -0,0 +1,13 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +import pytest +import os + +@pytest.mark.flaky(reruns=2) +def test_flaky(): # test_marker--test_flaky + # count is not set for first run, but set to 2 for the second run + count = os.environ.get("COUNT") + os.environ["COUNT"] = "2" + # this will fail on the first run, but pass on the second (1 passed, 1 rerun) + assert count == "2" diff --git a/python_files/tests/pytestadapter/expected_execution_test_output.py b/python_files/tests/pytestadapter/expected_execution_test_output.py index 8f378074343d..8ea2fe1f1c0b 100644 --- a/python_files/tests/pytestadapter/expected_execution_test_output.py +++ b/python_files/tests/pytestadapter/expected_execution_test_output.py @@ -734,3 +734,18 @@ "subtest": None, }, } + +test_rerunfailures_plugin_path = TEST_DATA_PATH / "test_rerunfailures_plugin.py" +rerunfailures_plugin_expected_execution_output = { + get_absolute_test_id( + "test_rerunfailures_plugin.py::test_flaky", test_rerunfailures_plugin_path + ): { + "test": get_absolute_test_id( + "test_rerunfailures_plugin.py::test_flaky", test_rerunfailures_plugin_path + ), + "outcome": "success", + "message": None, + "traceback": None, + "subtest": None, + } +} diff --git a/python_files/tests/pytestadapter/test_execution.py b/python_files/tests/pytestadapter/test_execution.py index 245b13cf5d46..af34fdc07b01 100644 --- a/python_files/tests/pytestadapter/test_execution.py +++ b/python_files/tests/pytestadapter/test_execution.py @@ -194,6 +194,13 @@ def test_rootdir_specified(): expected_execution_test_output.nested_describe_expected_execution_output, id="nested_describe_plugin", ), + pytest.param( + [ + "test_rerunfailures_plugin.py::test_flaky", + ], + expected_execution_test_output.rerunfailures_plugin_expected_execution_output, + id="test_rerunfailures_plugin", + ), ], ) def test_pytest_execution(test_ids, expected_const): diff --git a/python_files/vscode_pytest/__init__.py b/python_files/vscode_pytest/__init__.py index 553be4019919..87812b5ae0f8 100644 --- a/python_files/vscode_pytest/__init__.py +++ b/python_files/vscode_pytest/__init__.py @@ -33,16 +33,6 @@ USES_PYTEST_DESCRIBE = True -sys.path.append("/Users/eleanorboyd/vscode-python/.nox/install_python_libs/lib/python3.10") -sys.path.append("/Users/eleanorboyd/vscode-python-debugger") -sys.path.append("/Users/eleanorboyd/vscode-python-debugger/bundled") -sys.path.append("/Users/eleanorboyd/vscode-python-debugger/bundled/libs") - -import debugpy # noqa: E402 - -debugpy.connect(5678) -debugpy.breakpoint() # noqa: E702 - class TestData(TypedDict): """A general class that all test objects inherit from.""" From 4e2d4583b46f61e0656d2505e8c8e73c37b5733a Mon Sep 17 00:00:00 2001 From: eleanorjboyd Date: Thu, 21 Nov 2024 09:22:15 -0800 Subject: [PATCH 3/7] reformat --- python_files/vscode_pytest/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/python_files/vscode_pytest/__init__.py b/python_files/vscode_pytest/__init__.py index 87812b5ae0f8..1dab66e6ceac 100644 --- a/python_files/vscode_pytest/__init__.py +++ b/python_files/vscode_pytest/__init__.py @@ -33,7 +33,6 @@ USES_PYTEST_DESCRIBE = True - class TestData(TypedDict): """A general class that all test objects inherit from.""" From a7f749b3cc4ac558f654e72648a5958415bbb00d Mon Sep 17 00:00:00 2001 From: eleanorjboyd Date: Thu, 21 Nov 2024 09:30:30 -0800 Subject: [PATCH 4/7] add test for rerunfailures arg --- .../.data/test_rerunfailures_plugin.py | 8 ++++++++ .../expected_execution_test_output.py | 15 +++++++++++++++ .../tests/pytestadapter/test_execution.py | 17 +++++++++++++++++ 3 files changed, 40 insertions(+) diff --git a/python_files/tests/pytestadapter/.data/test_rerunfailures_plugin.py b/python_files/tests/pytestadapter/.data/test_rerunfailures_plugin.py index 93d58378ffa1..7f152cbb68c4 100644 --- a/python_files/tests/pytestadapter/.data/test_rerunfailures_plugin.py +++ b/python_files/tests/pytestadapter/.data/test_rerunfailures_plugin.py @@ -11,3 +11,11 @@ def test_flaky(): # test_marker--test_flaky os.environ["COUNT"] = "2" # this will fail on the first run, but pass on the second (1 passed, 1 rerun) assert count == "2" + +def test_flaky_no_marker(): + # this test is flaky and will be run via the command line argument + # count is not set for first run, but set to 2 for the second run + count = os.environ.get("COUNT") + os.environ["COUNT"] = "2" + # this will fail on the first run, but pass on the second (1 passed, 1 rerun) + assert count == "2" diff --git a/python_files/tests/pytestadapter/expected_execution_test_output.py b/python_files/tests/pytestadapter/expected_execution_test_output.py index 8ea2fe1f1c0b..0d8e9af943f3 100644 --- a/python_files/tests/pytestadapter/expected_execution_test_output.py +++ b/python_files/tests/pytestadapter/expected_execution_test_output.py @@ -749,3 +749,18 @@ "subtest": None, } } + +test_rerunfailures_plugin_path = TEST_DATA_PATH / "test_rerunfailures_plugin.py" +rerunfailures_with_arg_expected_execution_output = { + get_absolute_test_id( + "test_rerunfailures_plugin.py::test_flaky_no_marker", test_rerunfailures_plugin_path + ): { + "test": get_absolute_test_id( + "test_rerunfailures_plugin.py::test_flaky_no_marker", test_rerunfailures_plugin_path + ), + "outcome": "success", + "message": None, + "traceback": None, + "subtest": None, + } +} diff --git a/python_files/tests/pytestadapter/test_execution.py b/python_files/tests/pytestadapter/test_execution.py index af34fdc07b01..746a86dca9bd 100644 --- a/python_files/tests/pytestadapter/test_execution.py +++ b/python_files/tests/pytestadapter/test_execution.py @@ -65,6 +65,23 @@ def test_rootdir_specified(): assert actual_result_dict == expected_const +def test_rerunfailure_with_arg(): + """Test pytest execution when a --rootdir is specified.""" + args = ["--reruns=2", "test_rerunfailures_plugin.py::test_flaky_no_marker"] + actual = runner(args) + expected_const = expected_execution_test_output.rerunfailures_with_arg_expected_execution_output + assert actual + actual_list: List[Dict[str, Dict[str, Any]]] = actual + assert len(actual_list) == len(expected_const) + actual_result_dict = {} + if actual_list is not None: + for actual_item in actual_list: + assert all(item in actual_item for item in ("status", "cwd", "result")) + assert actual_item.get("status") == "success" + actual_result_dict.update(actual_item["result"]) + assert actual_result_dict == expected_const + + @pytest.mark.parametrize( ("test_ids", "expected_const"), [ From 415bf7f9193533fe0ea8098a453157ff3adeee5a Mon Sep 17 00:00:00 2001 From: eleanorjboyd Date: Thu, 21 Nov 2024 13:52:54 -0800 Subject: [PATCH 5/7] logging to check on failing tests --- python_files/tests/pytestadapter/helpers.py | 1 + python_files/tests/pytestadapter/test_discovery.py | 1 + python_files/vscode_pytest/__init__.py | 4 ++++ 3 files changed, 6 insertions(+) diff --git a/python_files/tests/pytestadapter/helpers.py b/python_files/tests/pytestadapter/helpers.py index 7a75e6248844..d04646471e5c 100644 --- a/python_files/tests/pytestadapter/helpers.py +++ b/python_files/tests/pytestadapter/helpers.py @@ -331,6 +331,7 @@ def runner_with_cwd_env( os.mkfifo(pipe_name) ################# + print("beginning run request") completed = threading.Event() result = [] # result is a string array to store the data during threading diff --git a/python_files/tests/pytestadapter/test_discovery.py b/python_files/tests/pytestadapter/test_discovery.py index 276753149410..a38be7f92b4c 100644 --- a/python_files/tests/pytestadapter/test_discovery.py +++ b/python_files/tests/pytestadapter/test_discovery.py @@ -308,6 +308,7 @@ def test_config_sub_folder(): session node is correctly updated to the common path. """ folder_path = helpers.TEST_DATA_PATH / "config_sub_folder" + print("running test_config_sub_folder") actual = helpers.runner_with_cwd( [ "--collect-only", diff --git a/python_files/vscode_pytest/__init__.py b/python_files/vscode_pytest/__init__.py index 1dab66e6ceac..6d7cbe2236d0 100644 --- a/python_files/vscode_pytest/__init__.py +++ b/python_files/vscode_pytest/__init__.py @@ -96,6 +96,7 @@ def pytest_load_initial_conftests(early_config, parser, args): # noqa: ARG001 # set the reruns value, -1 if not set global FLAKY_MAX_RUNS FLAKY_MAX_RUNS = get_reruns_value(args) + print("Plugin info[vscode-pytest]: global FLAKY_MAX_RUNS set to: ", FLAKY_MAX_RUNS) # check if --rootdir is in the args for arg in args: @@ -198,6 +199,7 @@ def pytest_exception_interact(node, call, report): # flaky_max_runs != -1 means test is flaky if flaky_max_runs != -1 and exec_count <= flaky_max_runs: + print("flaky test rerun: ", exec_count) return elif flaky_max_runs != -1 and exec_count > flaky_max_runs: print("Plugin info[vscode-pytest]: max reruns reached.") @@ -709,6 +711,8 @@ def build_nested_folders( counter = 0 max_iter = 100 while iterator_path != session_node_path: + print("iterator_path: ", iterator_path) + print("session_node_path: ", session_node_path) curr_folder_name = iterator_path.name try: curr_folder_node: TestNode = created_files_folders_dict[os.fspath(iterator_path)] From 03170f6695c156bf20a96592100cc52b2ae0aef8 Mon Sep 17 00:00:00 2001 From: eleanorjboyd Date: Thu, 21 Nov 2024 14:39:09 -0800 Subject: [PATCH 6/7] add timeout & more logging --- python_files/tests/pytestadapter/test_discovery.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/python_files/tests/pytestadapter/test_discovery.py b/python_files/tests/pytestadapter/test_discovery.py index a38be7f92b4c..be0435df4702 100644 --- a/python_files/tests/pytestadapter/test_discovery.py +++ b/python_files/tests/pytestadapter/test_discovery.py @@ -247,10 +247,12 @@ def test_symlink_root_dir(): def test_pytest_root_dir(): + """Test to test pytest discovery with the command line arg --rootdir specified to be a subfolder of the workspace root. Discovery should succeed and testids should be relative to workspace root. """ + print("running test_pytest_root_dir") rd = f"--rootdir={helpers.TEST_DATA_PATH / 'root' / 'tests'}" actual = helpers.runner_with_cwd( [ @@ -279,6 +281,7 @@ def test_pytest_config_file(): Discovery should succeed and testids should be relative to workspace root. """ + print("running test_pytest_config_file") actual = helpers.runner_with_cwd( [ "--collect-only", @@ -301,6 +304,7 @@ def test_pytest_config_file(): ), f"Tests tree does not match expected value. \n Expected: {json.dumps(expected_discovery_test_output.root_with_config_expected_output, indent=4)}. \n Actual: {json.dumps(actual_item.get('tests'), indent=4)}" +@pytest.mark.timeout(60) def test_config_sub_folder(): """Here the session node will be a subfolder of the workspace root and the test are in another subfolder. From 366eb15010406c96874741ab55dc3466fbc1a937 Mon Sep 17 00:00:00 2001 From: eleanorjboyd Date: Thu, 21 Nov 2024 16:01:59 -0800 Subject: [PATCH 7/7] ruff fix error --- python_files/tests/pytestadapter/test_discovery.py | 1 - 1 file changed, 1 deletion(-) diff --git a/python_files/tests/pytestadapter/test_discovery.py b/python_files/tests/pytestadapter/test_discovery.py index be0435df4702..22581a98fa92 100644 --- a/python_files/tests/pytestadapter/test_discovery.py +++ b/python_files/tests/pytestadapter/test_discovery.py @@ -247,7 +247,6 @@ def test_symlink_root_dir(): def test_pytest_root_dir(): - """Test to test pytest discovery with the command line arg --rootdir specified to be a subfolder of the workspace root. Discovery should succeed and testids should be relative to workspace root.