feat: warnings are now real warnings

This makes coverage warnings visible when running test suites under
pytest.  But it also means some uninteresting warnings would show up in
our own test suite, so we had to catch or suppress those.

Author: nedbat

CHANGES.rst

@@ -33,6 +33,8 @@ Unreleased
   imported a file that will be measured" warnings about Django itself.  These
   have been fixed, closing `issue 1150`_.
 
+- Warnings generated by coverage.py are now real Python warnings.
+
 - The ``COVERAGE_DEBUG_FILE`` environment variable now accepts ``stdout`` and
   ``stderr`` to write to those destinations.
 

coverage/control.py

@@ -11,6 +11,7 @@
 import platform
 import sys
 import time
+import warnings
 
 from coverage import env
 from coverage.annotate import AnnotateReporter
@@ -20,7 +21,7 @@
 from coverage.data import CoverageData, combine_parallel_data
 from coverage.debug import DebugControl, short_stack, write_formatted_info
 from coverage.disposition import disposition_debug_msg
-from coverage.exceptions import CoverageException
+from coverage.exceptions import CoverageException, CoverageWarning
 from coverage.files import PathAliases, abs_file, relative_filename, set_relative_directory
 from coverage.html import HtmlReporter
 from coverage.inorout import InOrOut
@@ -362,7 +363,7 @@ def _warn(self, msg, slug=None, once=False):
             msg = f"{msg} ({slug})"
         if self._debug.should('pid'):
             msg = f"[{os.getpid()}] {msg}"
-        sys.stderr.write(f"Coverage.py warning: {msg}\n")
+        warnings.warn(msg, category=CoverageWarning, stacklevel=2)
 
         if once:
             self._no_warn_slugs.append(slug)

coverage/exceptions.py

@@ -46,3 +46,8 @@ class StopEverything(BaseCoverageException):
 
     """
     pass
+
+
+class CoverageWarning(Warning):
+    """A warning from Coverage.py."""
+    pass

coverage/inorout.py

@@ -356,10 +356,9 @@ def nope(disp, reason):
                         )
                     break
             except Exception:
-                self.warn(
-                    "Disabling plug-in %r due to an exception:" % (plugin._coverage_plugin_name)
-                )
-                traceback.print_exc()
+                plugin_name = plugin._coverage_plugin_name
+                tb = traceback.format_exc()
+                self.warn(f"Disabling plug-in {plugin_name!r} due to an exception:\n{tb}")
                 plugin._coverage_enabled = False
                 continue
         else:
@@ -503,10 +502,8 @@ def _warn_about_unmeasured_code(self, pkg):
         # The module was in sys.modules, and seems like a module with code, but
         # we never measured it. I guess that means it was imported before
         # coverage even started.
-        self.warn(
-            "Module %s was previously imported, but not measured" % pkg,
-            slug="module-not-measured",
-        )
+        msg = f"Module {pkg} was previously imported, but not measured"
+        self.warn(msg, slug="module-not-measured")
 
     def find_possibly_unexecuted_files(self):
         """Find files in the areas of interest that might be untraced.

coverage/pytracer.py

@@ -254,10 +254,8 @@ def stop(self):
             # has changed to None.
             dont_warn = (env.PYPY and env.PYPYVERSION >= (5, 4) and self.in_atexit and tf is None)
             if (not dont_warn) and tf != self._trace:   # pylint: disable=comparison-with-callable
-                self.warn(
-                    f"Trace function changed, measurement is likely wrong: {tf!r}",
-                    slug="trace-changed",
-                )
+                msg = f"Trace function changed, measurement is likely wrong: {tf!r}"
+                self.warn(msg, slug="trace-changed")
 
     def activity(self):
         """Has there been any activity?"""

tests/helpers.py

@@ -14,6 +14,7 @@
 
 from unittest import mock
 
+from coverage.exceptions import CoverageWarning
 from coverage.misc import output_encoding
 
 
@@ -262,3 +263,17 @@ def assert_count_equal(a, b):
     This only works for hashable elements.
     """
     assert collections.Counter(list(a)) == collections.Counter(list(b))
+
+
+def assert_coverage_warnings(warns, *msgs):
+    """
+    Assert that `warns` are all CoverageWarning's, and have `msgs` as messages.
+    """
+    assert msgs     # don't call this without some messages.
+    assert len(warns) == len(msgs)
+    assert all(w.category == CoverageWarning for w in warns)
+    for actual, expected in zip((w.message.args[0] for w in warns), msgs):
+        if hasattr(expected, "search"):
+            assert expected.search(actual), f"{actual!r} didn't match {expected!r}"
+        else:
+            assert expected == actual

tests/test_api.py

@@ -23,7 +23,7 @@
 from coverage.misc import import_local_file
 
 from tests.coveragetest import CoverageTest, TESTS_DIR, UsingModulesMixin
-from tests.helpers import assert_count_equal, change_dir, nice_file
+from tests.helpers import assert_count_equal, assert_coverage_warnings, change_dir, nice_file
 
 
 class ApiTest(CoverageTest):
@@ -300,9 +300,11 @@ def test_completely_zero_reporting(self):
         # If nothing was measured, the file-touching didn't happen properly.
         self.make_file("foo/bar.py", "print('Never run')")
         self.make_file("test.py", "assert True")
-        cov = coverage.Coverage(source=["foo"])
-        self.start_import_stop(cov, "test")
-        cov.report()
+        with pytest.warns(Warning) as warns:
+            cov = coverage.Coverage(source=["foo"])
+            self.start_import_stop(cov, "test")
+            cov.report()
+        assert_coverage_warnings(warns, "No data was collected. (no-data-collected)")
         # Name         Stmts   Miss  Cover
         # --------------------------------
         # foo/bar.py       1      1     0%
@@ -517,18 +519,19 @@ def test_warnings(self):
             import sys, os
             print("Hello")
             """)
-        cov = coverage.Coverage(source=["sys", "xyzzy", "quux"])
-        self.start_import_stop(cov, "hello")
-        cov.get_data()
-
-        out, err = self.stdouterr()
-        assert "Hello\n" in out
-        assert textwrap.dedent("""\
-            Coverage.py warning: Module sys has no Python source. (module-not-python)
-            Coverage.py warning: Module xyzzy was never imported. (module-not-imported)
-            Coverage.py warning: Module quux was never imported. (module-not-imported)
-            Coverage.py warning: No data was collected. (no-data-collected)
-            """) in err
+        with pytest.warns(Warning) as warns:
+            cov = coverage.Coverage(source=["sys", "xyzzy", "quux"])
+            self.start_import_stop(cov, "hello")
+            cov.get_data()
+
+        assert "Hello\n" == self.stdout()
+        assert_coverage_warnings(
+            warns,
+            "Module sys has no Python source. (module-not-python)",
+            "Module xyzzy was never imported. (module-not-imported)",
+            "Module quux was never imported. (module-not-imported)",
+            "No data was collected. (no-data-collected)",
+            )
 
     def test_warnings_suppressed(self):
         self.make_file("hello.py", """\
@@ -539,24 +542,25 @@ def test_warnings_suppressed(self):
             [run]
             disable_warnings = no-data-collected, module-not-imported
             """)
-        cov = coverage.Coverage(source=["sys", "xyzzy", "quux"])
-        self.start_import_stop(cov, "hello")
-        cov.get_data()
+        with pytest.warns(Warning) as warns:
+            cov = coverage.Coverage(source=["sys", "xyzzy", "quux"])
+            self.start_import_stop(cov, "hello")
+            cov.get_data()
 
-        out, err = self.stdouterr()
-        assert "Hello\n" in out
-        assert "Coverage.py warning: Module sys has no Python source. (module-not-python)" in err
-        assert "module-not-imported" not in err
-        assert "no-data-collected" not in err
+        assert "Hello\n" == self.stdout()
+        assert_coverage_warnings(warns, "Module sys has no Python source. (module-not-python)")
+        # No "module-not-imported" in warns
+        # No "no-data-collected" in warns
 
     def test_warn_once(self):
-        cov = coverage.Coverage()
-        cov.load()
-        cov._warn("Warning, warning 1!", slug="bot", once=True)
-        cov._warn("Warning, warning 2!", slug="bot", once=True)
-        err = self.stderr()
-        assert "Warning, warning 1!" in err
-        assert "Warning, warning 2!" not in err
+        with pytest.warns(Warning) as warns:
+            cov = coverage.Coverage()
+            cov.load()
+            cov._warn("Warning, warning 1!", slug="bot", once=True)
+            cov._warn("Warning, warning 2!", slug="bot", once=True)
+
+        assert_coverage_warnings(warns, "Warning, warning 1! (bot)")
+        # No "Warning, warning 2!" in warns
 
     def test_source_and_include_dont_conflict(self):
         # A bad fix made this case fail: https://github.com/nedbat/coveragepy/issues/541
@@ -683,12 +687,12 @@ def test_dynamic_context_conflict(self):
         cov = coverage.Coverage(source=["."])
         cov.set_option("run:dynamic_context", "test_function")
         cov.start()
-        # Switch twice, but only get one warning.
-        cov.switch_context("test1")                                     # pragma: nested
-        cov.switch_context("test2")                                     # pragma: nested
-        expected = "Coverage.py warning: Conflicting dynamic contexts (dynamic-conflict)\n"
-        assert expected == self.stderr()
+        with pytest.warns(Warning) as warns:
+            # Switch twice, but only get one warning.
+            cov.switch_context("test1")                                 # pragma: nested
+            cov.switch_context("test2")                                 # pragma: nested
         cov.stop()                                                      # pragma: nested
+        assert_coverage_warnings(warns, "Conflicting dynamic contexts (dynamic-conflict)")
 
     def test_switch_context_unstarted(self):
         # Coverage must be started to switch context

tests/test_html.py

@@ -24,7 +24,7 @@
 from tests.coveragetest import CoverageTest, TESTS_DIR
 from tests.goldtest import gold_path
 from tests.goldtest import compare, contains, doesnt_contain, contains_any
-from tests.helpers import change_dir
+from tests.helpers import assert_coverage_warnings, change_dir
 
 
 class HtmlTestHelpers(CoverageTest):
@@ -341,13 +341,14 @@ def test_dotpy_not_python_ignored(self):
         self.make_file("innocuous.py", "a = 2")
         cov = coverage.Coverage()
         self.start_import_stop(cov, "main")
+
         self.make_file("innocuous.py", "<h1>This isn't python!</h1>")
-        cov.html_report(ignore_errors=True)
-        msg = "Expected a warning to be thrown when an invalid python file is parsed"
-        assert 1 == len(cov._warnings), msg
-        msg = "Warning message should be in 'invalid file' warning"
-        assert "Couldn't parse Python file" in cov._warnings[0], msg
-        assert "innocuous.py" in cov._warnings[0], "Filename should be in 'invalid file' warning"
+        with pytest.warns(Warning) as warns:
+            cov.html_report(ignore_errors=True)
+        assert_coverage_warnings(
+            warns,
+            re.compile(r"Couldn't parse Python file '.*innocuous.py' \(couldnt-parse\)"),
+            )
         self.assert_exists("htmlcov/index.html")
         # This would be better as a glob, if the HTML layout changes:
         self.assert_doesnt_exist("htmlcov/innocuous.html")

tests/test_oddball.py

@@ -81,17 +81,18 @@ def recur(n):
     def test_long_recursion(self):
         # We can't finish a very deep recursion, but we don't crash.
         with pytest.raises(RuntimeError):
-            self.check_coverage("""\
-                def recur(n):
-                    if n == 0:
-                        return 0
-                    else:
-                        return recur(n-1)+1
-
-                recur(100000)  # This is definitely too many frames.
-                """,
-                [1, 2, 3, 5, 7], ""
-                )
+            with pytest.warns(None):
+                self.check_coverage("""\
+                    def recur(n):
+                        if n == 0:
+                            return 0
+                        else:
+                            return recur(n-1)+1
+
+                    recur(100000)  # This is definitely too many frames.
+                    """,
+                    [1, 2, 3, 5, 7], ""
+                    )
 
     def test_long_recursion_recovery(self):
         # Test the core of bug 93: https://github.com/nedbat/coveragepy/issues/93
@@ -117,7 +118,8 @@ def recur(n):
             """)
 
         cov = coverage.Coverage()
-        self.start_import_stop(cov, "recur")
+        with pytest.warns(None):
+            self.start_import_stop(cov, "recur")
 
         pytrace = (cov._collector.tracer_name() == "PyTracer")
         expected_missing = [3]

tests/test_plugins.py

@@ -14,7 +14,7 @@
 from coverage import env
 from coverage.control import Plugins
 from coverage.data import line_counts
-from coverage.exceptions import CoverageException
+from coverage.exceptions import CoverageException, CoverageWarning
 from coverage.misc import import_local_file
 
 import coverage.plugin
@@ -193,7 +193,9 @@ def coverage_init(reg, options):
         cov = coverage.Coverage(debug=["sys"])
         cov._debug_file = debug_out
         cov.set_option("run:plugins", ["plugin_sys_info"])
-        cov.start()
+        with pytest.warns(None):
+            # Catch warnings so we don't see "plugins aren't supported on PyTracer"
+            cov.start()
         cov.stop()      # pragma: nested
 
         out_lines = [line.strip() for line in debug_out.getvalue().splitlines()]
@@ -631,28 +633,29 @@ def run_bad_plugin(self, module_name, plugin_name, our_error=True, excmsg=None,
         explaining why.
 
         """
-        self.run_plugin(module_name)
+        with pytest.warns(Warning) as warns:
+            self.run_plugin(module_name)
 
         stderr = self.stderr()
-
+        stderr += "".join(w.message.args[0] for w in warns)
         if our_error:
-            errors = stderr.count("# Oh noes!")
             # The exception we're causing should only appear once.
-            assert errors == 1
+            assert stderr.count("# Oh noes!") == 1
 
         # There should be a warning explaining what's happening, but only one.
         # The message can be in two forms:
         #   Disabling plug-in '...' due to previous exception
         # or:
         #   Disabling plug-in '...' due to an exception:
-        msg = f"Disabling plug-in '{module_name}.{plugin_name}' due to "
-        warnings = stderr.count(msg)
-        assert warnings == 1
+        assert len(warns) == 1
+        assert issubclass(warns[0].category, CoverageWarning)
+        warnmsg = warns[0].message.args[0]
+        assert f"Disabling plug-in '{module_name}.{plugin_name}' due to " in warnmsg
 
         if excmsg:
             assert excmsg in stderr
         if excmsgs:
-            assert any(em in stderr for em in excmsgs), "expected one of %r" % excmsgs
+            assert any(em in stderr for em in excmsgs), f"expected one of {excmsgs} in stderr"
 
     def test_file_tracer_has_no_file_tracer_method(self):
         self.make_file("bad_plugin.py", """\