Skip to content

Commit 6b3dec4

Browse files
authored
Support EOT for testing (#21876)
Adds support for the end of transmission (EOT) operator to all pytest and unittest responses. this PR includes: - addition of an EOT that is added to run and discovery returns and processed by the extension to initiate cleanup after run/discovery finishes - updates to all tests to support the use of EOT - redesign of how cleanup works following run/discover to make it more streamlined - full functional tests that check multiple different types of payload splitting from the buffer - tests for the cancellation token during run and debug modes
1 parent 5838ea6 commit 6b3dec4

25 files changed

+1882
-892
lines changed

pythonFiles/testing_tools/socket_manager.py

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,12 @@ def __init__(self, addr):
2323
self.socket = None
2424

2525
def __enter__(self):
26+
return self.connect()
27+
28+
def __exit__(self, *_):
29+
self.close()
30+
31+
def connect(self):
2632
self.socket = socket.socket(
2733
socket.AF_INET, socket.SOCK_STREAM, socket.IPPROTO_TCP
2834
)
@@ -35,7 +41,7 @@ def __enter__(self):
3541

3642
return self
3743

38-
def __exit__(self, *_):
44+
def close(self):
3945
if self.socket:
4046
try:
4147
self.socket.shutdown(socket.SHUT_RDWR)

pythonFiles/tests/pytestadapter/test_discovery.py

Lines changed: 86 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -29,15 +29,26 @@ def test_import_error(tmp_path):
2929
temp_dir.mkdir()
3030
p = temp_dir / "error_pytest_import.py"
3131
shutil.copyfile(file_path, p)
32-
actual_list: Optional[List[Dict[str, Any]]] = runner(
33-
["--collect-only", os.fspath(p)]
34-
)
35-
assert actual_list
36-
for actual in actual_list:
37-
assert all(item in actual for item in ("status", "cwd", "error"))
38-
assert actual["status"] == "error"
39-
assert actual["cwd"] == os.fspath(TEST_DATA_PATH)
40-
assert len(actual["error"]) == 2
32+
actual: Optional[List[Dict[str, Any]]] = runner(["--collect-only", os.fspath(p)])
33+
assert actual
34+
actual_list: List[Dict[str, Any]] = actual
35+
if actual_list is not None:
36+
assert actual_list.pop(-1).get("eot")
37+
for actual_item in actual_list:
38+
assert all(
39+
item in actual_item.keys() for item in ("status", "cwd", "error")
40+
)
41+
assert actual_item.get("status") == "error"
42+
assert actual_item.get("cwd") == os.fspath(TEST_DATA_PATH)
43+
44+
# Ensure that 'error' is a list and then check its length
45+
error_content = actual_item.get("error")
46+
if error_content is not None and isinstance(
47+
error_content, (list, tuple, str)
48+
): # You can add other types if needed
49+
assert len(error_content) == 2
50+
else:
51+
assert False
4152

4253

4354
def test_syntax_error(tmp_path):
@@ -60,13 +71,25 @@ def test_syntax_error(tmp_path):
6071
p = temp_dir / "error_syntax_discovery.py"
6172
shutil.copyfile(file_path, p)
6273
actual = runner(["--collect-only", os.fspath(p)])
63-
if actual:
64-
actual = actual[0]
65-
assert actual
66-
assert all(item in actual for item in ("status", "cwd", "error"))
67-
assert actual["status"] == "error"
68-
assert actual["cwd"] == os.fspath(TEST_DATA_PATH)
69-
assert len(actual["error"]) == 2
74+
assert actual
75+
actual_list: List[Dict[str, Any]] = actual
76+
if actual_list is not None:
77+
assert actual_list.pop(-1).get("eot")
78+
for actual_item in actual_list:
79+
assert all(
80+
item in actual_item.keys() for item in ("status", "cwd", "error")
81+
)
82+
assert actual_item.get("status") == "error"
83+
assert actual_item.get("cwd") == os.fspath(TEST_DATA_PATH)
84+
85+
# Ensure that 'error' is a list and then check its length
86+
error_content = actual_item.get("error")
87+
if error_content is not None and isinstance(
88+
error_content, (list, tuple, str)
89+
): # You can add other types if needed
90+
assert len(error_content) == 2
91+
else:
92+
assert False
7093

7194

7295
def test_parameterized_error_collect():
@@ -76,12 +99,25 @@ def test_parameterized_error_collect():
7699
"""
77100
file_path_str = "error_parametrize_discovery.py"
78101
actual = runner(["--collect-only", file_path_str])
79-
if actual:
80-
actual = actual[0]
81-
assert all(item in actual for item in ("status", "cwd", "error"))
82-
assert actual["status"] == "error"
83-
assert actual["cwd"] == os.fspath(TEST_DATA_PATH)
84-
assert len(actual["error"]) == 2
102+
assert actual
103+
actual_list: List[Dict[str, Any]] = actual
104+
if actual_list is not None:
105+
assert actual_list.pop(-1).get("eot")
106+
for actual_item in actual_list:
107+
assert all(
108+
item in actual_item.keys() for item in ("status", "cwd", "error")
109+
)
110+
assert actual_item.get("status") == "error"
111+
assert actual_item.get("cwd") == os.fspath(TEST_DATA_PATH)
112+
113+
# Ensure that 'error' is a list and then check its length
114+
error_content = actual_item.get("error")
115+
if error_content is not None and isinstance(
116+
error_content, (list, tuple, str)
117+
): # You can add other types if needed
118+
assert len(error_content) == 2
119+
else:
120+
assert False
85121

86122

87123
@pytest.mark.parametrize(
@@ -146,13 +182,16 @@ def test_pytest_collect(file, expected_const):
146182
os.fspath(TEST_DATA_PATH / file),
147183
]
148184
)
149-
if actual:
150-
actual = actual[0]
151-
assert actual
152-
assert all(item in actual for item in ("status", "cwd", "tests"))
153-
assert actual["status"] == "success"
154-
assert actual["cwd"] == os.fspath(TEST_DATA_PATH)
155-
assert actual["tests"] == expected_const
185+
186+
assert actual
187+
actual_list: List[Dict[str, Any]] = actual
188+
if actual_list is not None:
189+
assert actual_list.pop(-1).get("eot")
190+
actual_item = actual_list.pop(0)
191+
assert all(item in actual_item.keys() for item in ("status", "cwd", "error"))
192+
assert actual_item.get("status") == "success"
193+
assert actual_item.get("cwd") == os.fspath(TEST_DATA_PATH)
194+
assert actual_item.get("tests") == expected_const
156195

157196

158197
def test_pytest_root_dir():
@@ -168,14 +207,16 @@ def test_pytest_root_dir():
168207
],
169208
TEST_DATA_PATH / "root",
170209
)
171-
if actual:
172-
actual = actual[0]
173-
assert actual
174-
assert all(item in actual for item in ("status", "cwd", "tests"))
175-
assert actual["status"] == "success"
176-
assert actual["cwd"] == os.fspath(TEST_DATA_PATH / "root")
210+
assert actual
211+
actual_list: List[Dict[str, Any]] = actual
212+
if actual_list is not None:
213+
assert actual_list.pop(-1).get("eot")
214+
actual_item = actual_list.pop(0)
215+
assert all(item in actual_item.keys() for item in ("status", "cwd", "error"))
216+
assert actual_item.get("status") == "success"
217+
assert actual_item.get("cwd") == os.fspath(TEST_DATA_PATH / "root")
177218
assert (
178-
actual["tests"]
219+
actual_item.get("tests")
179220
== expected_discovery_test_output.root_with_config_expected_output
180221
)
181222

@@ -193,13 +234,15 @@ def test_pytest_config_file():
193234
],
194235
TEST_DATA_PATH / "root",
195236
)
196-
if actual:
197-
actual = actual[0]
198-
assert actual
199-
assert all(item in actual for item in ("status", "cwd", "tests"))
200-
assert actual["status"] == "success"
201-
assert actual["cwd"] == os.fspath(TEST_DATA_PATH / "root")
237+
assert actual
238+
actual_list: List[Dict[str, Any]] = actual
239+
if actual_list is not None:
240+
assert actual_list.pop(-1).get("eot")
241+
actual_item = actual_list.pop(0)
242+
assert all(item in actual_item.keys() for item in ("status", "cwd", "error"))
243+
assert actual_item.get("status") == "success"
244+
assert actual_item.get("cwd") == os.fspath(TEST_DATA_PATH / "root")
202245
assert (
203-
actual["tests"]
246+
actual_item.get("tests")
204247
== expected_discovery_test_output.root_with_config_expected_output
205248
)

pythonFiles/tests/pytestadapter/test_execution.py

Lines changed: 73 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
# Licensed under the MIT License.
33
import os
44
import shutil
5+
from typing import Any, Dict, List
56

67
import pytest
78

@@ -23,14 +24,19 @@ def test_config_file():
2324
expected_execution_test_output.config_file_pytest_expected_execution_output
2425
)
2526
assert actual
26-
assert len(actual) == len(expected_const)
27+
actual_list: List[Dict[str, Any]] = actual
28+
assert actual_list.pop(-1).get("eot")
29+
assert len(actual_list) == len(expected_const)
2730
actual_result_dict = dict()
28-
for a in actual:
29-
assert all(item in a for item in ("status", "cwd", "result"))
30-
assert a["status"] == "success"
31-
assert a["cwd"] == os.fspath(new_cwd)
32-
actual_result_dict.update(a["result"])
33-
assert actual_result_dict == expected_const
31+
if actual_list is not None:
32+
for actual_item in actual_list:
33+
assert all(
34+
item in actual_item.keys() for item in ("status", "cwd", "result")
35+
)
36+
assert actual_item.get("status") == "success"
37+
assert actual_item.get("cwd") == os.fspath(new_cwd)
38+
actual_result_dict.update(actual_item["result"])
39+
assert actual_result_dict == expected_const
3440

3541

3642
def test_rootdir_specified():
@@ -43,14 +49,19 @@ def test_rootdir_specified():
4349
expected_execution_test_output.config_file_pytest_expected_execution_output
4450
)
4551
assert actual
46-
assert len(actual) == len(expected_const)
52+
actual_list: List[Dict[str, Any]] = actual
53+
assert actual_list.pop(-1).get("eot")
54+
assert len(actual_list) == len(expected_const)
4755
actual_result_dict = dict()
48-
for a in actual:
49-
assert all(item in a for item in ("status", "cwd", "result"))
50-
assert a["status"] == "success"
51-
assert a["cwd"] == os.fspath(new_cwd)
52-
actual_result_dict.update(a["result"])
53-
assert actual_result_dict == expected_const
56+
if actual_list is not None:
57+
for actual_item in actual_list:
58+
assert all(
59+
item in actual_item.keys() for item in ("status", "cwd", "result")
60+
)
61+
assert actual_item.get("status") == "success"
62+
assert actual_item.get("cwd") == os.fspath(new_cwd)
63+
actual_result_dict.update(actual_item["result"])
64+
assert actual_result_dict == expected_const
5465

5566

5667
def test_syntax_error_execution(tmp_path):
@@ -73,13 +84,23 @@ def test_syntax_error_execution(tmp_path):
7384
p = temp_dir / "error_syntax_discovery.py"
7485
shutil.copyfile(file_path, p)
7586
actual = runner(["error_syntax_discover.py::test_function"])
76-
if actual:
77-
actual = actual[0]
78-
assert actual
79-
assert all(item in actual for item in ("status", "cwd", "error"))
80-
assert actual["status"] == "error"
81-
assert actual["cwd"] == os.fspath(TEST_DATA_PATH)
82-
assert len(actual["error"]) == 1
87+
assert actual
88+
actual_list: List[Dict[str, Any]] = actual
89+
assert actual_list.pop(-1).get("eot")
90+
if actual_list is not None:
91+
for actual_item in actual_list:
92+
assert all(
93+
item in actual_item.keys() for item in ("status", "cwd", "error")
94+
)
95+
assert actual_item.get("status") == "error"
96+
assert actual_item.get("cwd") == os.fspath(TEST_DATA_PATH)
97+
error_content = actual_item.get("error")
98+
if error_content is not None and isinstance(
99+
error_content, (list, tuple, str)
100+
): # You can add other types if needed
101+
assert len(error_content) == 1
102+
else:
103+
assert False
83104

84105

85106
def test_bad_id_error_execution():
@@ -88,13 +109,23 @@ def test_bad_id_error_execution():
88109
The json should still be returned but the errors list should be present.
89110
"""
90111
actual = runner(["not/a/real::test_id"])
91-
if actual:
92-
actual = actual[0]
93-
assert actual
94-
assert all(item in actual for item in ("status", "cwd", "error"))
95-
assert actual["status"] == "error"
96-
assert actual["cwd"] == os.fspath(TEST_DATA_PATH)
97-
assert len(actual["error"]) == 1
112+
assert actual
113+
actual_list: List[Dict[str, Any]] = actual
114+
assert actual_list.pop(-1).get("eot")
115+
if actual_list is not None:
116+
for actual_item in actual_list:
117+
assert all(
118+
item in actual_item.keys() for item in ("status", "cwd", "error")
119+
)
120+
assert actual_item.get("status") == "error"
121+
assert actual_item.get("cwd") == os.fspath(TEST_DATA_PATH)
122+
error_content = actual_item.get("error")
123+
if error_content is not None and isinstance(
124+
error_content, (list, tuple, str)
125+
): # You can add other types if needed
126+
assert len(error_content) == 1
127+
else:
128+
assert False
98129

99130

100131
@pytest.mark.parametrize(
@@ -195,7 +226,8 @@ def test_pytest_execution(test_ids, expected_const):
195226
3. uf_single_method_execution_expected_output: test run on a single method in a file.
196227
4. uf_non_adjacent_tests_execution_expected_output: test run on unittests in two files with single selection in test explorer.
197228
5. unit_pytest_same_file_execution_expected_output: test run on a file with both unittest and pytest tests.
198-
6. dual_level_nested_folder_execution_expected_output: test run on a file with one test file at the top level and one test file in a nested folder.
229+
6. dual_level_nested_folder_execution_expected_output: test run on a file with one test file
230+
at the top level and one test file in a nested folder.
199231
7. double_nested_folder_expected_execution_output: test run on a double nested folder.
200232
8. parametrize_tests_expected_execution_output: test run on a parametrize test with 3 inputs.
201233
9. single_parametrize_tests_expected_execution_output: test run on single parametrize test.
@@ -205,18 +237,22 @@ def test_pytest_execution(test_ids, expected_const):
205237
Keyword arguments:
206238
test_ids -- an array of test_ids to run.
207239
expected_const -- a dictionary of the expected output from running pytest discovery on the files.
208-
""" # noqa: E501
240+
"""
209241
args = test_ids
210242
actual = runner(args)
211243
assert actual
212-
print(actual)
213-
assert len(actual) == len(expected_const)
244+
actual_list: List[Dict[str, Any]] = actual
245+
assert actual_list.pop(-1).get("eot")
246+
assert len(actual_list) == len(expected_const)
214247
actual_result_dict = dict()
215-
for a in actual:
216-
assert all(item in a for item in ("status", "cwd", "result"))
217-
assert a["status"] == "success"
218-
assert a["cwd"] == os.fspath(TEST_DATA_PATH)
219-
actual_result_dict.update(a["result"])
248+
if actual_list is not None:
249+
for actual_item in actual_list:
250+
assert all(
251+
item in actual_item.keys() for item in ("status", "cwd", "result")
252+
)
253+
assert actual_item.get("status") == "success"
254+
assert actual_item.get("cwd") == os.fspath(TEST_DATA_PATH)
255+
actual_result_dict.update(actual_item["result"])
220256
for key in actual_result_dict:
221257
if (
222258
actual_result_dict[key]["outcome"] == "failure"

0 commit comments

Comments
 (0)