Fail the parent pytest test when one of its subtests fails

python_files/tests/pytestadapter/.data/subtests_execution.py

@@ -0,0 +1,42 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+import unittest
+
+import pytest
+
+
+# Each subtest reports with its parent's node id, so the parent must fail if any subtest fails,
+# no matter which subtest reports first.
+def test_pass_then_fail(subtests):
+    with subtests.test("passes"):
+        assert True
+    with subtests.test("fails"):
+        assert 1 == 2
+
+
+def test_fail_then_pass(subtests):
+    with subtests.test("fails"):
+        assert 1 == 2
+    with subtests.test("passes"):
+        assert True
+
+
+def test_all_pass(subtests):
+    with subtests.test("first"):
+        assert True
+    with subtests.test("second"):
+        assert True
+
+
+def test_pass_then_body_fails(subtests):
+    with subtests.test("passes"):
+        assert True
+    pytest.fail("the test body fails after its subtests")
+
+
+class TestUnittestSubTest(unittest.TestCase):
+    def test_pass_then_fail(self):
+        with self.subTest(i=0):
+            self.assertEqual(1, 1)
+        with self.subTest(i=1):
+            self.assertEqual(1, 2)

python_files/tests/pytestadapter/expected_execution_test_output.py

@@ -747,3 +747,69 @@
         "subtest": None,
     }
 }
+
+# This is the expected output for the subtests_execution.py tests.
+# A failed subtest fails its parent test, whichever subtest reports first.
+# └── subtests_execution.py
+#    ├── test_pass_then_fail: failure
+#    ├── test_fail_then_pass: failure
+#    ├── test_all_pass: success
+#    ├── test_pass_then_body_fails: failure
+#    └── TestUnittestSubTest
+#       └── test_pass_then_fail: failure
+subtests_execution_path = TEST_DATA_PATH / "subtests_execution.py"
+subtests_execution_expected_output = {
+    get_absolute_test_id("subtests_execution.py::test_pass_then_fail", subtests_execution_path): {
+        "test": get_absolute_test_id(
+            "subtests_execution.py::test_pass_then_fail", subtests_execution_path
+        ),
+        "outcome": FAILURE,
+        "message": "ERROR MESSAGE",
+        "traceback": None,
+        "subtest": None,
+    },
+    get_absolute_test_id("subtests_execution.py::test_fail_then_pass", subtests_execution_path): {
+        "test": get_absolute_test_id(
+            "subtests_execution.py::test_fail_then_pass", subtests_execution_path
+        ),
+        "outcome": FAILURE,
+        "message": "ERROR MESSAGE",
+        "traceback": None,
+        "subtest": None,
+    },
+    get_absolute_test_id("subtests_execution.py::test_all_pass", subtests_execution_path): {
+        "test": get_absolute_test_id(
+            "subtests_execution.py::test_all_pass", subtests_execution_path
+        ),
+        "outcome": SUCCESS,
+        "message": None,
+        "traceback": None,
+        "subtest": None,
+    },
+    get_absolute_test_id(
+        "subtests_execution.py::test_pass_then_body_fails", subtests_execution_path
+    ): {
+        "test": get_absolute_test_id(
+            "subtests_execution.py::test_pass_then_body_fails", subtests_execution_path
+        ),
+        "outcome": FAILURE,
+        "message": "ERROR MESSAGE",
+        "traceback": None,
+        "subtest": None,
+    },
+}
+
+unittest_subtest_execution_expected_output = {
+    get_absolute_test_id(
+        "subtests_execution.py::TestUnittestSubTest::test_pass_then_fail", subtests_execution_path
+    ): {
+        "test": get_absolute_test_id(
+            "subtests_execution.py::TestUnittestSubTest::test_pass_then_fail",
+            subtests_execution_path,
+        ),
+        "outcome": FAILURE,
+        "message": "ERROR MESSAGE",
+        "traceback": None,
+        "subtest": None,
+    }
+}

python_files/tests/pytestadapter/test_execution.py

@@ -1,5 +1,6 @@
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # Licensed under the MIT License.
+import importlib.util
 import json
 import os
 import pathlib
@@ -199,6 +200,26 @@ def test_rootdir_specified():
             expected_execution_test_output.skip_test_fixture_execution_expected_output,
             id="skip_test_fixture",
         ),
+        pytest.param(
+            [
+                "subtests_execution.py::test_pass_then_fail",
+                "subtests_execution.py::test_fail_then_pass",
+                "subtests_execution.py::test_all_pass",
+                "subtests_execution.py::test_pass_then_body_fails",
+            ],
+            expected_execution_test_output.subtests_execution_expected_output,
+            id="pytest_subtests",
+            marks=pytest.mark.skipif(
+                pytest.version_tuple < (9, 0)
+                and importlib.util.find_spec("pytest_subtests") is None,
+                reason="needs pytest 9 or the pytest-subtests plugin",
+            ),
+        ),
+        pytest.param(
+            ["subtests_execution.py::TestUnittestSubTest::test_pass_then_fail"],
+            expected_execution_test_output.unittest_subtest_execution_expected_output,
+            id="unittest_subtest",
+        ),
     ],
 )
 def test_pytest_execution(test_ids, expected_const):

python_files/vscode_pytest/__init__.py

@@ -90,6 +90,10 @@ def __init__(self, message):
 IS_DISCOVERY = False
 map_id_to_path = {}
 collected_tests_so_far = set()
+# Failure messages of subtests, keyed by the absolute id of their parent test. Subtest reports
+# share the parent's node id, so they are collected here and folded into the parent's single
+# outcome when its own call report arrives.
+subtest_failures: dict[str, list[str]] = {}
 TEST_RUN_PIPE = os.getenv("TEST_RUN_PIPE")
 PROJECT_ROOT_PATH = os.getenv(
     "PROJECT_ROOT_PATH"
@@ -202,6 +206,9 @@ def pytest_exception_interact(node, call, report):
         else:
             ERRORS.append(report.longreprtext + "\n Check Python Logs for more details.")
     else:
+        if is_subtest_report(report):
+            # Folded into the parent's outcome by pytest_report_teststatus.
+            return
         # If during execution, send this data that the given node failed.
         report_value = "error"
         if call.excinfo.typename == "AssertionError":
@@ -303,6 +310,18 @@ class TestRunResultDict(Dict[str, Dict[str, TestOutcome]]):
     tests: dict[str, TestOutcome]
 
 
+def is_subtest_report(report) -> bool:
+    """Return True for a report produced by a subtest rather than by the test itself.
+
+    pytest 9 reports `subtests.test()` blocks and unittest `subTest()` blocks as
+    `SubtestReport`; the pytest-subtests plugin used on older pytest versions reports
+    them as `SubTestReport`. Both are call phase reports that reuse the parent's node id.
+    """
+    return type(report).__name__ in ("SubtestReport", "SubTestReport") and hasattr(
+        report, "context"
+    )
+
+
 @pytest.hookimpl(hookwrapper=True, trylast=True)
 def pytest_report_teststatus(report, config):  # noqa: ARG001
     """A pytest hook that is called when a test is called.
@@ -332,8 +351,25 @@ def pytest_report_teststatus(report, config):  # noqa: ARG001
             node_path = cwd
         # Calculate the absolute test id and use this as the ID moving forward.
         absolute_node_id = get_absolute_test_id(report.nodeid, node_path)
-        if absolute_node_id not in collected_tests_so_far:
+        if is_subtest_report(report):
+            # A subtest report carries its parent's node id, so it must not publish or claim that
+            # id; its failure is kept for the parent's own call report. The terminal summary can
+            # call this hook again after the parent is published, hence the id check.
+            if report.failed and absolute_node_id not in collected_tests_so_far:
+                subtest_failures.setdefault(absolute_node_id, []).append(
+                    f"{report.head_line}\n{report.longreprtext}"
+                )
+        elif absolute_node_id not in collected_tests_so_far:
             collected_tests_so_far.add(absolute_node_id)
+            failed_subtests = subtest_failures.pop(absolute_node_id, None)
+            if failed_subtests:
+                # The parent's report can still say passed here: pytest 9 marks it failed only
+                # in its own hook, which runs after this code, and never does for unittest
+                # subTest() or the pytest-subtests plugin. Any failed subtest fails the parent.
+                if message:
+                    failed_subtests.append(message)
+                report_value = "failure"
+                message = "\n\n".join(failed_subtests)
             item_result = create_test_outcome(
                 absolute_node_id,
                 report_value,