Skip to content

Commit d05b83f

Browse files
eleanorjboydkarthiknadig
eleanorjboyd
and
karthiknadig
authored
Python test execution simple (#21053)
closes #20897 closes #20084 closes #20081 --------- Co-authored-by: Karthik Nadig <[email protected]>
1 parent 61882f7 commit d05b83f

File tree

7 files changed

+688
-35
lines changed

7 files changed

+688
-35
lines changed

‎pythonFiles/tests/pytestadapter/.data/unittest_folder/test_subtract.py

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,4 +22,5 @@ def test_subtract_negative_numbers( # test_marker--test_subtract_negative_numbe
2222
self,
2323
):
2424
result = subtract(-2, -3)
25-
self.assertEqual(result, 1)
25+
# This is intentional to test assertion failures
26+
self.assertEqual(result, 100000)

‎pythonFiles/tests/pytestadapter/expected_discovery_test_output.py

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,8 @@
33

44
from .helpers import TEST_DATA_PATH, find_test_line_number
55

6+
# This file contains the expected output dictionaries for tests discovery and is used in test_discovery.py.
7+
68
# This is the expected output for the empty_discovery.py file.
79
# └──
810
TEST_DATA_PATH_STR = os.fspath(TEST_DATA_PATH)

‎pythonFiles/tests/pytestadapter/expected_execution_test_output.py

Lines changed: 328 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,328 @@
1+
# Copyright (c) Microsoft Corporation. All rights reserved.
2+
# Licensed under the MIT License.
3+
4+
TEST_SUBTRACT_FUNCTION = "unittest_folder/test_subtract.py::TestSubtractFunction::"
5+
TEST_ADD_FUNCTION = "unittest_folder/test_add.py::TestAddFunction::"
6+
SUCCESS = "success"
7+
FAILURE = "failure"
8+
TEST_SUBTRACT_FUNCTION_NEGATIVE_NUMBERS_ERROR = "self = <test_subtract.TestSubtractFunction testMethod=test_subtract_negative_numbers>\n\n def test_subtract_negative_numbers( # test_marker--test_subtract_negative_numbers\n self,\n ):\n result = subtract(-2, -3)\n> self.assertEqual(result, 100000)\nE AssertionError: 1 != 100000\n\nunittest_folder/test_subtract.py:25: AssertionError"
9+
10+
# This is the expected output for the unittest_folder execute tests
11+
# └── unittest_folder
12+
# ├── test_add.py
13+
# │ └── TestAddFunction
14+
# │ ├── test_add_negative_numbers: success
15+
# │ └── test_add_positive_numbers: success
16+
# └── test_subtract.py
17+
# └── TestSubtractFunction
18+
# ├── test_subtract_negative_numbers: failure
19+
# └── test_subtract_positive_numbers: success
20+
uf_execution_expected_output = {
21+
f"{TEST_ADD_FUNCTION}test_add_negative_numbers": {
22+
"test": f"{TEST_ADD_FUNCTION}test_add_negative_numbers",
23+
"outcome": SUCCESS,
24+
"message": None,
25+
"traceback": None,
26+
"subtest": None,
27+
},
28+
f"{TEST_ADD_FUNCTION}test_add_positive_numbers": {
29+
"test": f"{TEST_ADD_FUNCTION}test_add_positive_numbers",
30+
"outcome": SUCCESS,
31+
"message": None,
32+
"traceback": None,
33+
"subtest": None,
34+
},
35+
f"{TEST_SUBTRACT_FUNCTION}test_subtract_negative_numbers": {
36+
"test": f"{TEST_SUBTRACT_FUNCTION}test_subtract_negative_numbers",
37+
"outcome": FAILURE,
38+
"message": "ERROR MESSAGE",
39+
"traceback": None,
40+
"subtest": None,
41+
},
42+
f"{TEST_SUBTRACT_FUNCTION}test_subtract_positive_numbers": {
43+
"test": f"{TEST_SUBTRACT_FUNCTION}test_subtract_positive_numbers",
44+
"outcome": SUCCESS,
45+
"message": None,
46+
"traceback": None,
47+
"subtest": None,
48+
},
49+
}
50+
51+
52+
# This is the expected output for the unittest_folder only execute add.py tests
53+
# └── unittest_folder
54+
# ├── test_add.py
55+
# │ └── TestAddFunction
56+
# │ ├── test_add_negative_numbers: success
57+
# │ └── test_add_positive_numbers: success
58+
uf_single_file_expected_output = {
59+
f"{TEST_ADD_FUNCTION}test_add_negative_numbers": {
60+
"test": f"{TEST_ADD_FUNCTION}test_add_negative_numbers",
61+
"outcome": SUCCESS,
62+
"message": None,
63+
"traceback": None,
64+
"subtest": None,
65+
},
66+
f"{TEST_ADD_FUNCTION}test_add_positive_numbers": {
67+
"test": f"{TEST_ADD_FUNCTION}test_add_positive_numbers",
68+
"outcome": SUCCESS,
69+
"message": None,
70+
"traceback": None,
71+
"subtest": None,
72+
},
73+
}
74+
75+
# This is the expected output for the unittest_folder execute only signle method
76+
# └── unittest_folder
77+
# ├── test_add.py
78+
# │ └── TestAddFunction
79+
# │ └── test_add_positive_numbers: success
80+
uf_single_method_execution_expected_output = {
81+
f"{TEST_ADD_FUNCTION}test_add_positive_numbers": {
82+
"test": f"{TEST_ADD_FUNCTION}test_add_positive_numbers",
83+
"outcome": SUCCESS,
84+
"message": None,
85+
"traceback": None,
86+
"subtest": None,
87+
}
88+
}
89+
90+
# This is the expected output for the unittest_folder tests run where two tests
91+
# run are in different files.
92+
# └── unittest_folder
93+
# ├── test_add.py
94+
# │ └── TestAddFunction
95+
# │ └── test_add_positive_numbers: success
96+
# └── test_subtract.py
97+
# └── TestSubtractFunction
98+
# └── test_subtract_positive_numbers: success
99+
uf_non_adjacent_tests_execution_expected_output = {
100+
TEST_SUBTRACT_FUNCTION
101+
+ "test_subtract_positive_numbers": {
102+
"test": TEST_SUBTRACT_FUNCTION + "test_subtract_positive_numbers",
103+
"outcome": SUCCESS,
104+
"message": None,
105+
"traceback": None,
106+
"subtest": None,
107+
},
108+
TEST_ADD_FUNCTION
109+
+ "test_add_positive_numbers": {
110+
"test": TEST_ADD_FUNCTION + "test_add_positive_numbers",
111+
"outcome": SUCCESS,
112+
"message": None,
113+
"traceback": None,
114+
"subtest": None,
115+
},
116+
}
117+
118+
# This is the expected output for the simple_pytest.py file.
119+
# └── simple_pytest.py
120+
# └── test_function: success
121+
simple_execution_pytest_expected_output = {
122+
"simple_pytest.py::test_function": {
123+
"test": "simple_pytest.py::test_function",
124+
"outcome": "success",
125+
"message": None,
126+
"traceback": None,
127+
"subtest": None,
128+
}
129+
}
130+
131+
# This is the expected output for the unittest_pytest_same_file.py file.
132+
# ├── unittest_pytest_same_file.py
133+
# ├── TestExample
134+
# │ └── test_true_unittest: success
135+
# └── test_true_pytest: success
136+
unit_pytest_same_file_execution_expected_output = {
137+
"unittest_pytest_same_file.py::TestExample::test_true_unittest": {
138+
"test": "unittest_pytest_same_file.py::TestExample::test_true_unittest",
139+
"outcome": "success",
140+
"message": None,
141+
"traceback": None,
142+
"subtest": None,
143+
},
144+
"unittest_pytest_same_file.py::test_true_pytest": {
145+
"test": "unittest_pytest_same_file.py::test_true_pytest",
146+
"outcome": "success",
147+
"message": None,
148+
"traceback": None,
149+
"subtest": None,
150+
},
151+
}
152+
153+
# This is the expected output for the dual_level_nested_folder.py tests
154+
# └── dual_level_nested_folder
155+
# └── test_top_folder.py
156+
# └── test_top_function_t: success
157+
# └── test_top_function_f: failure
158+
# └── nested_folder_one
159+
# └── test_bottom_folder.py
160+
# └── test_bottom_function_t: success
161+
# └── test_bottom_function_f: failure
162+
dual_level_nested_folder_execution_expected_output = {
163+
"dual_level_nested_folder/test_top_folder.py::test_top_function_t": {
164+
"test": "dual_level_nested_folder/test_top_folder.py::test_top_function_t",
165+
"outcome": "success",
166+
"message": None,
167+
"traceback": None,
168+
"subtest": None,
169+
},
170+
"dual_level_nested_folder/test_top_folder.py::test_top_function_f": {
171+
"test": "dual_level_nested_folder/test_top_folder.py::test_top_function_f",
172+
"outcome": "failure",
173+
"message": "ERROR MESSAGE",
174+
"traceback": None,
175+
"subtest": None,
176+
},
177+
"dual_level_nested_folder/nested_folder_one/test_bottom_folder.py::test_bottom_function_t": {
178+
"test": "dual_level_nested_folder/nested_folder_one/test_bottom_folder.py::test_bottom_function_t",
179+
"outcome": "success",
180+
"message": None,
181+
"traceback": None,
182+
"subtest": None,
183+
},
184+
"dual_level_nested_folder/nested_folder_one/test_bottom_folder.py::test_bottom_function_f": {
185+
"test": "dual_level_nested_folder/nested_folder_one/test_bottom_folder.py::test_bottom_function_f",
186+
"outcome": "failure",
187+
"message": "ERROR MESSAGE",
188+
"traceback": None,
189+
"subtest": None,
190+
},
191+
}
192+
193+
# This is the expected output for the nested_folder tests.
194+
# └── nested_folder_one
195+
# └── nested_folder_two
196+
# └── test_nest.py
197+
# └── test_function: success
198+
double_nested_folder_expected_execution_output = {
199+
"double_nested_folder/nested_folder_one/nested_folder_two/test_nest.py::test_function": {
200+
"test": "double_nested_folder/nested_folder_one/nested_folder_two/test_nest.py::test_function",
201+
"outcome": "success",
202+
"message": None,
203+
"traceback": None,
204+
"subtest": None,
205+
}
206+
}
207+
208+
# This is the expected output for the nested_folder tests.
209+
# └── parametrize_tests.py
210+
# └── test_adding[3+5-8]: success
211+
# └── test_adding[2+4-6]: success
212+
# └── test_adding[6+9-16]: failure
213+
parametrize_tests_expected_execution_output = {
214+
"parametrize_tests.py::test_adding[3+5-8]": {
215+
"test": "parametrize_tests.py::test_adding[3+5-8]",
216+
"outcome": "success",
217+
"message": None,
218+
"traceback": None,
219+
"subtest": None,
220+
},
221+
"parametrize_tests.py::test_adding[2+4-6]": {
222+
"test": "parametrize_tests.py::test_adding[2+4-6]",
223+
"outcome": "success",
224+
"message": None,
225+
"traceback": None,
226+
"subtest": None,
227+
},
228+
"parametrize_tests.py::test_adding[6+9-16]": {
229+
"test": "parametrize_tests.py::test_adding[6+9-16]",
230+
"outcome": "failure",
231+
"message": "ERROR MESSAGE",
232+
"traceback": None,
233+
"subtest": None,
234+
},
235+
}
236+
237+
# This is the expected output for the single parameterized tests.
238+
# └── parametrize_tests.py
239+
# └── test_adding[3+5-8]: success
240+
single_parametrize_tests_expected_execution_output = {
241+
"parametrize_tests.py::test_adding[3+5-8]": {
242+
"test": "parametrize_tests.py::test_adding[3+5-8]",
243+
"outcome": "success",
244+
"message": None,
245+
"traceback": None,
246+
"subtest": None,
247+
},
248+
}
249+
250+
# This is the expected output for the single parameterized tests.
251+
# └── text_docstring.txt
252+
# └── text_docstring: success
253+
doctest_pytest_expected_execution_output = {
254+
"text_docstring.txt::text_docstring.txt": {
255+
"test": "text_docstring.txt::text_docstring.txt",
256+
"outcome": "success",
257+
"message": None,
258+
"traceback": None,
259+
"subtest": None,
260+
}
261+
}
262+
263+
# Will run all tests in the cwd that fit the test file naming pattern.
264+
no_test_ids_pytest_execution_expected_output = {
265+
"double_nested_folder/nested_folder_one/nested_folder_two/test_nest.py::test_function": {
266+
"test": "double_nested_folder/nested_folder_one/nested_folder_two/test_nest.py::test_function",
267+
"outcome": "success",
268+
"message": None,
269+
"traceback": None,
270+
"subtest": None,
271+
},
272+
"dual_level_nested_folder/test_top_folder.py::test_top_function_t": {
273+
"test": "dual_level_nested_folder/test_top_folder.py::test_top_function_t",
274+
"outcome": "success",
275+
"message": None,
276+
"traceback": None,
277+
"subtest": None,
278+
},
279+
"dual_level_nested_folder/test_top_folder.py::test_top_function_f": {
280+
"test": "dual_level_nested_folder/test_top_folder.py::test_top_function_f",
281+
"outcome": "failure",
282+
"message": "ERROR MESSAGE",
283+
"traceback": None,
284+
"subtest": None,
285+
},
286+
"dual_level_nested_folder/nested_folder_one/test_bottom_folder.py::test_bottom_function_t": {
287+
"test": "dual_level_nested_folder/nested_folder_one/test_bottom_folder.py::test_bottom_function_t",
288+
"outcome": "success",
289+
"message": None,
290+
"traceback": None,
291+
"subtest": None,
292+
},
293+
"dual_level_nested_folder/nested_folder_one/test_bottom_folder.py::test_bottom_function_f": {
294+
"test": "dual_level_nested_folder/nested_folder_one/test_bottom_folder.py::test_bottom_function_f",
295+
"outcome": "failure",
296+
"message": "ERROR MESSAGE",
297+
"traceback": None,
298+
"subtest": None,
299+
},
300+
"unittest_folder/test_add.py::TestAddFunction::test_add_negative_numbers": {
301+
"test": "unittest_folder/test_add.py::TestAddFunction::test_add_negative_numbers",
302+
"outcome": "success",
303+
"message": None,
304+
"traceback": None,
305+
"subtest": None,
306+
},
307+
"unittest_folder/test_add.py::TestAddFunction::test_add_positive_numbers": {
308+
"test": "unittest_folder/test_add.py::TestAddFunction::test_add_positive_numbers",
309+
"outcome": "success",
310+
"message": None,
311+
"traceback": None,
312+
"subtest": None,
313+
},
314+
"unittest_folder/test_subtract.py::TestSubtractFunction::test_subtract_negative_numbers": {
315+
"test": "unittest_folder/test_subtract.py::TestSubtractFunction::test_subtract_negative_numbers",
316+
"outcome": "failure",
317+
"message": "ERROR MESSAGE",
318+
"traceback": None,
319+
"subtest": None,
320+
},
321+
"unittest_folder/test_subtract.py::TestSubtractFunction::test_subtract_positive_numbers": {
322+
"test": "unittest_folder/test_subtract.py::TestSubtractFunction::test_subtract_positive_numbers",
323+
"outcome": "success",
324+
"message": None,
325+
"traceback": None,
326+
"subtest": None,
327+
},
328+
}

‎pythonFiles/tests/pytestadapter/helpers.py

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@
1111
import subprocess
1212
import sys
1313
import uuid
14-
from typing import Dict, List, Union
14+
from typing import Any, Dict, List, Optional, Union
1515

1616
TEST_DATA_PATH = pathlib.Path(__file__).parent / ".data"
1717
from typing_extensions import TypedDict
@@ -83,7 +83,7 @@ def _new_sock() -> socket.socket:
8383
)
8484

8585

86-
def process_rpc_json(data: str) -> Dict[str, str]:
86+
def process_rpc_json(data: str) -> Dict[str, Any]:
8787
"""Process the JSON data which comes from the server which runs the pytest discovery."""
8888
str_stream: io.StringIO = io.StringIO(data)
8989

@@ -107,7 +107,7 @@ def process_rpc_json(data: str) -> Dict[str, str]:
107107
return json.loads(raw_json)
108108

109109

110-
def runner(args: List[str]) -> Union[Dict[str, str], None]:
110+
def runner(args: List[str]) -> Optional[Dict[str, Any]]:
111111
"""Run the pytest discovery and return the JSON data from the server."""
112112
process_args: List[str] = [
113113
sys.executable,

0 commit comments

Comments
 (0)