Implementation of Test Coverage (#24118)

fixes #22671

Author: eleanorjboyd

.gitignore

@@ -49,3 +49,8 @@ dist/**
 package.nls.*.json
 l10n/
 python-env-tools/**
+# coverage files produced as test output
+python_files/tests/*/.data/.coverage*
+python_files/tests/*/.data/*/.coverage*
+src/testTestingRootWkspc/coverageWorkspace/.coverage
+

build/functional-test-requirements.txt

@@ -1,3 +1,5 @@
 # List of requirements for functional tests
 versioneer
 numpy
+pytest
+pytest-cov

build/test-requirements.txt

@@ -27,3 +27,7 @@ namedpipe; platform_system == "Windows"
 
 # typing for Django files
 django-stubs
+
+# for coverage
+coverage
+pytest-cov

python_files/tests/pytestadapter/.data/coverage_gen/__init__.py

@@ -0,0 +1,2 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.

python_files/tests/pytestadapter/.data/coverage_gen/reverse.py

@@ -0,0 +1,19 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+
+def reverse_string(s):
+    if s is None or s == "":
+        return "Error: Input is None"
+    return s[::-1]
+
+def reverse_sentence(sentence):
+    if sentence is None or sentence == "":
+        return "Error: Input is None"
+    words = sentence.split()
+    reversed_words = [reverse_string(word) for word in words]
+    return " ".join(reversed_words)
+
+# Example usage
+if __name__ == "__main__":
+    sample_string = "hello"
+    print(reverse_string(sample_string))  # Output: "olleh"

python_files/tests/pytestadapter/.data/coverage_gen/test_reverse.py

@@ -0,0 +1,28 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+
+from .reverse import reverse_sentence, reverse_string
+
+
+def test_reverse_sentence():
+    """
+    Tests the reverse_sentence function to ensure it correctly reverses each word in a sentence.
+
+    Test cases:
+    - "hello world" should be reversed to "olleh dlrow"
+    - "Python is fun" should be reversed to "nohtyP si nuf"
+    - "a b c" should remain "a b c" as each character is a single word
+    """
+    assert reverse_sentence("hello world") == "olleh dlrow"
+    assert reverse_sentence("Python is fun") == "nohtyP si nuf"
+    assert reverse_sentence("a b c") == "a b c"
+
+def test_reverse_sentence_error():
+    assert reverse_sentence("") == "Error: Input is None"
+    assert reverse_sentence(None) == "Error: Input is None"
+
+
+def test_reverse_string():
+    assert reverse_string("hello") == "olleh"
+    assert reverse_string("Python") == "nohtyP"
+    # this test specifically does not cover the error cases

python_files/tests/pytestadapter/helpers.py

@@ -203,6 +203,26 @@ def runner_with_cwd(args: List[str], path: pathlib.Path) -> Optional[List[Dict[s
     return runner_with_cwd_env(args, path, {})
 
 
+def split_array_at_item(arr: List[str], item: str) -> Tuple[List[str], List[str]]:
+    """
+    Splits an array into two subarrays at the specified item.
+
+    Args:
+        arr (List[str]): The array to be split.
+        item (str): The item at which to split the array.
+
+    Returns:
+        Tuple[List[str], List[str]]: A tuple containing two subarrays. The first subarray includes the item and all elements before it. The second subarray includes all elements after the item. If the item is not found, the first subarray is the original array and the second subarray is empty.
+    """
+    if item in arr:
+        index = arr.index(item)
+        before = arr[: index + 1]
+        after = arr[index + 1 :]
+        return before, after
+    else:
+        return arr, []
+
+
 def runner_with_cwd_env(
     args: List[str], path: pathlib.Path, env_add: Dict[str, str]
 ) -> Optional[List[Dict[str, Any]]]:
@@ -217,10 +237,34 @@ def runner_with_cwd_env(
         # If we are running Django, generate a unittest-specific pipe name.
         process_args = [sys.executable, *args]
         pipe_name = generate_random_pipe_name("unittest-discovery-test")
+    elif "_TEST_VAR_UNITTEST" in env_add:
+        before_args, after_ids = split_array_at_item(args, "*test*.py")
+        process_args = [sys.executable, *before_args]
+        pipe_name = generate_random_pipe_name("unittest-execution-test")
+        test_ids_pipe = os.fspath(
+            script_dir / "tests" / "unittestadapter" / ".data" / "coverage_ex" / "10943021.txt"
+        )
+        env_add.update({"RUN_TEST_IDS_PIPE": test_ids_pipe})
+        test_ids_arr = after_ids
+        with open(test_ids_pipe, "w") as f:  # noqa: PTH123
+            f.write("\n".join(test_ids_arr))
     else:
         process_args = [sys.executable, "-m", "pytest", "-p", "vscode_pytest", "-s", *args]
         pipe_name = generate_random_pipe_name("pytest-discovery-test")
 
+    if "COVERAGE_ENABLED" in env_add and "_TEST_VAR_UNITTEST" not in env_add:
+        process_args = [
+            sys.executable,
+            "-m",
+            "pytest",
+            "-p",
+            "vscode_pytest",
+            "--cov=.",
+            "--cov-branch",
+            "-s",
+            *args,
+        ]
+
     # Generate pipe name, pipe name specific per OS type.
 
     # Windows design

python_files/tests/pytestadapter/test_coverage.py

@@ -0,0 +1,50 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+import os
+import pathlib
+import sys
+
+script_dir = pathlib.Path(__file__).parent.parent
+sys.path.append(os.fspath(script_dir))
+
+from .helpers import (  # noqa: E402
+    TEST_DATA_PATH,
+    runner_with_cwd_env,
+)
+
+
+def test_simple_pytest_coverage():
+    """
+    Test coverage payload is correct for simple pytest example. Output of coverage run is below.
+
+    Name              Stmts   Miss Branch BrPart  Cover
+    ---------------------------------------------------
+    __init__.py           0      0      0      0   100%
+    reverse.py           13      3      8      2    76%
+    test_reverse.py      11      0      0      0   100%
+    ---------------------------------------------------
+    TOTAL                24      3      8      2    84%
+
+    """
+    args = []
+    env_add = {"COVERAGE_ENABLED": "True"}
+    cov_folder_path = TEST_DATA_PATH / "coverage_gen"
+    actual = runner_with_cwd_env(args, cov_folder_path, env_add)
+    assert actual
+    coverage = actual[-1]
+    assert coverage
+    results = coverage["result"]
+    assert results
+    assert len(results) == 3
+    focal_function_coverage = results.get(os.fspath(TEST_DATA_PATH / "coverage_gen" / "reverse.py"))
+    assert focal_function_coverage
+    assert focal_function_coverage.get("lines_covered") is not None
+    assert focal_function_coverage.get("lines_missed") is not None
+    assert set(focal_function_coverage.get("lines_covered")) == {4, 5, 7, 9, 10, 11, 12, 13, 14, 17}
+    assert set(focal_function_coverage.get("lines_missed")) == {18, 19, 6}
+    assert (
+        focal_function_coverage.get("executed_branches") > 0
+    ), "executed_branches are a number greater than 0."
+    assert (
+        focal_function_coverage.get("total_branches") > 0
+    ), "total_branches are a number greater than 0."

python_files/tests/unittestadapter/.data/coverage_ex/__init__.py

@@ -0,0 +1,2 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.

python_files/tests/unittestadapter/.data/coverage_ex/reverse.py

@@ -0,0 +1,14 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+
+def reverse_string(s):
+    if s is None or s == "":
+        return "Error: Input is None"
+    return s[::-1]
+
+def reverse_sentence(sentence):
+    if sentence is None or sentence == "":
+        return "Error: Input is None"
+    words = sentence.split()
+    reversed_words = [reverse_string(word) for word in words]
+    return " ".join(reversed_words)

python_files/tests/unittestadapter/.data/coverage_ex/test_reverse.py

@@ -0,0 +1,32 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+
+import unittest
+from reverse import reverse_sentence, reverse_string
+
+class TestReverseFunctions(unittest.TestCase):
+
+    def test_reverse_sentence(self):
+        """
+        Tests the reverse_sentence function to ensure it correctly reverses each word in a sentence.
+
+        Test cases:
+        - "hello world" should be reversed to "olleh dlrow"
+        - "Python is fun" should be reversed to "nohtyP si nuf"
+        - "a b c" should remain "a b c" as each character is a single word
+        """
+        self.assertEqual(reverse_sentence("hello world"), "olleh dlrow")
+        self.assertEqual(reverse_sentence("Python is fun"), "nohtyP si nuf")
+        self.assertEqual(reverse_sentence("a b c"), "a b c")
+
+    def test_reverse_sentence_error(self):
+        self.assertEqual(reverse_sentence(""), "Error: Input is None")
+        self.assertEqual(reverse_sentence(None), "Error: Input is None")
+
+    def test_reverse_string(self):
+        self.assertEqual(reverse_string("hello"), "olleh")
+        self.assertEqual(reverse_string("Python"), "nohtyP")
+        # this test specifically does not cover the error cases
+
+if __name__ == '__main__':
+    unittest.main()

python_files/tests/unittestadapter/test_coverage.py

@@ -0,0 +1,57 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+
+import os
+import pathlib
+import sys
+
+sys.path.append(os.fspath(pathlib.Path(__file__).parent))
+
+python_files_path = pathlib.Path(__file__).parent.parent.parent
+sys.path.insert(0, os.fspath(python_files_path))
+sys.path.insert(0, os.fspath(python_files_path / "lib" / "python"))
+
+from tests.pytestadapter import helpers  # noqa: E402
+
+TEST_DATA_PATH = pathlib.Path(__file__).parent / ".data"
+
+
+def test_basic_coverage():
+    """This test runs on a simple django project with three tests, two of which pass and one that fails."""
+    coverage_ex_folder: pathlib.Path = TEST_DATA_PATH / "coverage_ex"
+    execution_script: pathlib.Path = python_files_path / "unittestadapter" / "execution.py"
+    test_ids = [
+        "test_reverse.TestReverseFunctions.test_reverse_sentence",
+        "test_reverse.TestReverseFunctions.test_reverse_sentence_error",
+        "test_reverse.TestReverseFunctions.test_reverse_string",
+    ]
+    argv = [os.fsdecode(execution_script), "--udiscovery", "-vv", "-s", ".", "-p", "*test*.py"]
+    argv = argv + test_ids
+
+    actual = helpers.runner_with_cwd_env(
+        argv,
+        coverage_ex_folder,
+        {"COVERAGE_ENABLED": os.fspath(coverage_ex_folder), "_TEST_VAR_UNITTEST": "True"},
+    )
+
+    assert actual
+    coverage = actual[-1]
+    assert coverage
+    results = coverage["result"]
+    assert results
+    assert len(results) == 3
+    focal_function_coverage = results.get(os.fspath(TEST_DATA_PATH / "coverage_ex" / "reverse.py"))
+    assert focal_function_coverage
+    assert focal_function_coverage.get("lines_covered") is not None
+    assert focal_function_coverage.get("lines_missed") is not None
+    assert set(focal_function_coverage.get("lines_covered")) == {4, 5, 7, 9, 10, 11, 12, 13, 14}
+    assert set(focal_function_coverage.get("lines_missed")) == {6}
+    assert (
+        focal_function_coverage.get("executed_branches") > 0
+    ), "executed_branches are a number greater than 0."
+    assert (
+        focal_function_coverage.get("total_branches") > 0
+    ), "total_branches are a number greater than 0."