Turn warnings into errors WIP, waiting #2598

.travis.yml

@@ -11,12 +11,9 @@ env:
     - TOXENV=coveralls
     # note: please use "tox --listenvs" to populate the build matrix below
     - TOXENV=linting
-    - TOXENV=py26
     - TOXENV=py27
-    - TOXENV=py33
     - TOXENV=py34
     - TOXENV=py35
-    - TOXENV=pypy
     - TOXENV=py27-pexpect
     - TOXENV=py27-xdist
     - TOXENV=py27-trial
@@ -32,6 +29,12 @@ env:
 
 matrix:
   include:
+    - env: TOXENV=py26
+      python: '2.6'
+    - env: TOXENV=py33
+      python: '3.3'
+    - env: TOXENV=pypy
+      python: 'pypy-5.4'
     - env: TOXENV=py36
       python: '3.6'
     - env: TOXENV=py37

changelog/2588.trivial

@@ -0,0 +1 @@
+Turn warnings into errors in pytest's own test suite in order to catch regressions due to deprecations more promptly.

testing/python/collect.py

@@ -12,6 +12,9 @@
 )
 
 
+ignore_parametrized_marks = pytest.mark.filterwarnings('ignore:Applying marks directly to parameters')
+
+
 class TestModule(object):
     def test_failing_import(self, testdir):
         modcol = testdir.getmodulecol("import alksdjalskdjalkjals")
@@ -567,7 +570,8 @@ def test_overridden_via_param(value):
         rec = testdir.inline_run()
         rec.assertoutcome(passed=1)
 
-    def test_parametrize_with_mark(selfself, testdir):
+    @ignore_parametrized_marks
+    def test_parametrize_with_mark(self, testdir):
         items = testdir.getitems("""
             import pytest
             @pytest.mark.foo
@@ -640,6 +644,7 @@ def test2(self, x, y):
         assert colitems[2].name == 'test2[a-c]'
         assert colitems[3].name == 'test2[b-c]'
 
+    @ignore_parametrized_marks
     def test_parametrize_skipif(self, testdir):
         testdir.makepyfile("""
             import pytest
@@ -653,6 +658,7 @@ def test_skip_if(x):
         result = testdir.runpytest()
         result.stdout.fnmatch_lines('* 2 passed, 1 skipped in *')
 
+    @ignore_parametrized_marks
     def test_parametrize_skip(self, testdir):
         testdir.makepyfile("""
             import pytest
@@ -666,6 +672,7 @@ def test_skip(x):
         result = testdir.runpytest()
         result.stdout.fnmatch_lines('* 2 passed, 1 skipped in *')
 
+    @ignore_parametrized_marks
     def test_parametrize_skipif_no_skip(self, testdir):
         testdir.makepyfile("""
             import pytest
@@ -679,6 +686,7 @@ def test_skipif_no_skip(x):
         result = testdir.runpytest()
         result.stdout.fnmatch_lines('* 1 failed, 2 passed in *')
 
+    @ignore_parametrized_marks
     def test_parametrize_xfail(self, testdir):
         testdir.makepyfile("""
             import pytest
@@ -692,6 +700,7 @@ def test_xfail(x):
         result = testdir.runpytest()
         result.stdout.fnmatch_lines('* 2 passed, 1 xfailed in *')
 
+    @ignore_parametrized_marks
     def test_parametrize_passed(self, testdir):
         testdir.makepyfile("""
             import pytest
@@ -705,6 +714,7 @@ def test_xfail(x):
         result = testdir.runpytest()
         result.stdout.fnmatch_lines('* 2 passed, 1 xpassed in *')
 
+    @ignore_parametrized_marks
     def test_parametrize_xfail_passed(self, testdir):
         testdir.makepyfile("""
             import pytest

testing/python/metafunc.py

@@ -1289,8 +1289,9 @@ def pytest_generate_tests(metafunc):
         assert output.count('preparing foo-3') == 1
 
 
+@pytest.mark.filterwarnings('ignore:Applying marks directly to parameters')
+@pytest.mark.issue308
 class TestMarkersWithParametrization(object):
-    pytestmark = pytest.mark.issue308
 
     def test_simple_mark(self, testdir):
         s = """

testing/python/setup_only.py

@@ -187,7 +187,7 @@ def dynamically_requested_fixture():
             pass
         @pytest.fixture()
         def dependent_fixture(request):
-            request.getfuncargvalue('dynamically_requested_fixture')
+            request.getfixturevalue('dynamically_requested_fixture')
         def test_dyn(dependent_fixture):
             pass
     ''')

testing/test_mark.py

@@ -787,6 +787,7 @@ def assert_test_is_not_selected(keyword):
                   marks=[pytest.mark.xfail, pytest.mark.skip], id=None)),
 
 ])
+@pytest.mark.filterwarnings('ignore')
 def test_parameterset_extractfrom(argval, expected):
     extracted = ParameterSet.extract_from(argval)
     assert extracted == expected

testing/test_recwarn.py

@@ -2,7 +2,6 @@
 import warnings
 import re
 import py
-import sys
 
 import pytest
 from _pytest.recwarn import WarningsRecorder
@@ -125,6 +124,7 @@ def f():
     @pytest.mark.parametrize('warning_type', [PendingDeprecationWarning, DeprecationWarning])
     @pytest.mark.parametrize('mode', ['context_manager', 'call'])
     @pytest.mark.parametrize('call_f_first', [True, False])
+    @pytest.mark.filterwarnings('ignore')
     def test_deprecated_call_modes(self, warning_type, mode, call_f_first):
         """Ensure deprecated_call() captures a deprecation warning as expected inside its
         block/function.
@@ -170,32 +170,6 @@ def f():
                 with pytest.deprecated_call():
                     f()
 
-    def test_deprecated_function_already_called(self, testdir):
-        """deprecated_call should be able to catch a call to a deprecated
-        function even if that function has already been called in the same
-        module. See #1190.
-        """
-        testdir.makepyfile("""
-            import warnings
-            import pytest
-
-            def deprecated_function():
-                warnings.warn("deprecated", DeprecationWarning)
-
-            def test_one():
-                deprecated_function()
-
-            def test_two():
-                pytest.deprecated_call(deprecated_function)
-        """)
-        result = testdir.runpytest()
-        # for some reason in py26 catch_warnings manages to catch the deprecation warning
-        # from deprecated_function(), even with default filters active (which ignore deprecation
-        # warnings)
-        py26 = sys.version_info[:2] == (2, 6)
-        expected = '*=== 2 passed in *===' if not py26 else '*=== 2 passed, 1 warnings in *==='
-        result.stdout.fnmatch_lines(expected)
-
 
 class TestWarns(object):
     def test_strings(self):

testing/test_unittest.py

@@ -9,9 +9,9 @@ def test_simple_unittest(testdir):
         import unittest
         class MyTestCase(unittest.TestCase):
             def testpassing(self):
-                self.assertEquals('foo', 'foo')
+                self.assertEqual('foo', 'foo')
             def test_failing(self):
-                self.assertEquals('foo', 'bar')
+                self.assertEqual('foo', 'bar')
     """)
     reprec = testdir.inline_run(testpath)
     assert reprec.matchreport("testpassing").passed
@@ -23,10 +23,10 @@ def test_runTest_method(testdir):
         import unittest
         class MyTestCaseWithRunTest(unittest.TestCase):
             def runTest(self):
-                self.assertEquals('foo', 'foo')
+                self.assertEqual('foo', 'foo')
         class MyTestCaseWithoutRunTest(unittest.TestCase):
             def runTest(self):
-                self.assertEquals('foo', 'foo')
+                self.assertEqual('foo', 'foo')
             def test_something(self):
                 pass
         """)
@@ -59,7 +59,7 @@ def setUp(self):
             def setup_method(self, method):
                 self.foo2 = 1
             def test_both(self):
-                self.assertEquals(1, self.foo)
+                self.assertEqual(1, self.foo)
                 assert self.foo2 == 1
             def teardown_method(self, method):
                 assert 0, "42"
@@ -136,7 +136,7 @@ def tearDown(self):
                 self.l.append(None)
         class Second(unittest.TestCase):
             def test_check(self):
-                self.assertEquals(MyTestCase.l, [None])
+                self.assertEqual(MyTestCase.l, [None])
     """)
     reprec = testdir.inline_run(testpath)
     passed, skipped, failed = reprec.countoutcomes()
@@ -351,61 +351,12 @@ def test_func1(self):
     reprec.assertoutcome(skipped=1)
 
 
-def test_trial_testcase_skip_property(testdir):
-    pytest.importorskip('twisted.trial.unittest')
-    testpath = testdir.makepyfile("""
-        from twisted.trial import unittest
-        class MyTestCase(unittest.TestCase):
-            skip = 'dont run'
-            def test_func(self):
-                pass
-        """)
-    reprec = testdir.inline_run(testpath, "-s")
-    reprec.assertoutcome(skipped=1)
-
-
-def test_trial_testfunction_skip_property(testdir):
-    pytest.importorskip('twisted.trial.unittest')
-    testpath = testdir.makepyfile("""
-        from twisted.trial import unittest
-        class MyTestCase(unittest.TestCase):
-            def test_func(self):
-                pass
-            test_func.skip = 'dont run'
-        """)
-    reprec = testdir.inline_run(testpath, "-s")
-    reprec.assertoutcome(skipped=1)
-
-
-def test_trial_testcase_todo_property(testdir):
-    pytest.importorskip('twisted.trial.unittest')
-    testpath = testdir.makepyfile("""
-        from twisted.trial import unittest
-        class MyTestCase(unittest.TestCase):
-            todo = 'dont run'
-            def test_func(self):
-                assert 0
-        """)
-    reprec = testdir.inline_run(testpath, "-s")
-    reprec.assertoutcome(skipped=1)
-
-
-def test_trial_testfunction_todo_property(testdir):
-    pytest.importorskip('twisted.trial.unittest')
-    testpath = testdir.makepyfile("""
-        from twisted.trial import unittest
-        class MyTestCase(unittest.TestCase):
-            def test_func(self):
-                assert 0
-            test_func.todo = 'dont run'
-        """)
-    reprec = testdir.inline_run(testpath, "-s")
-    reprec.assertoutcome(skipped=1)
-
-
 class TestTrialUnittest(object):
     def setup_class(cls):
         cls.ut = pytest.importorskip("twisted.trial.unittest")
+        # on windows trial uses a socket for a reactor and apparently doesn't close it properly
+        # https://twistedmatrix.com/trac/ticket/9227
+        cls.ignore_unclosed_socket_warning = ('-W', 'always')
 
     def test_trial_testcase_runtest_not_collected(self, testdir):
         testdir.makepyfile("""
@@ -415,7 +366,7 @@ class TC(TestCase):
                 def test_hello(self):
                     pass
         """)
-        reprec = testdir.inline_run()
+        reprec = testdir.inline_run(*self.ignore_unclosed_socket_warning)
         reprec.assertoutcome(passed=1)
         testdir.makepyfile("""
             from twisted.trial.unittest import TestCase
@@ -424,7 +375,7 @@ class TC(TestCase):
                 def runTest(self):
                     pass
         """)
-        reprec = testdir.inline_run()
+        reprec = testdir.inline_run(*self.ignore_unclosed_socket_warning)
         reprec.assertoutcome(passed=1)
 
     def test_trial_exceptions_with_skips(self, testdir):
@@ -462,7 +413,7 @@ def test_method(self):
         """)
         from _pytest.compat import _is_unittest_unexpected_success_a_failure
         should_fail = _is_unittest_unexpected_success_a_failure()
-        result = testdir.runpytest("-rxs")
+        result = testdir.runpytest("-rxs", *self.ignore_unclosed_socket_warning)
         result.stdout.fnmatch_lines_random([
             "*XFAIL*test_trial_todo*",
             "*trialselfskip*",
@@ -537,6 +488,50 @@ def test_hello(self):
         child.expect("hellopdb")
         child.sendeof()
 
+    def test_trial_testcase_skip_property(self, testdir):
+        testpath = testdir.makepyfile("""
+            from twisted.trial import unittest
+            class MyTestCase(unittest.TestCase):
+                skip = 'dont run'
+                def test_func(self):
+                    pass
+            """)
+        reprec = testdir.inline_run(testpath, "-s")
+        reprec.assertoutcome(skipped=1)
+
+    def test_trial_testfunction_skip_property(self, testdir):
+        testpath = testdir.makepyfile("""
+            from twisted.trial import unittest
+            class MyTestCase(unittest.TestCase):
+                def test_func(self):
+                    pass
+                test_func.skip = 'dont run'
+            """)
+        reprec = testdir.inline_run(testpath, "-s")
+        reprec.assertoutcome(skipped=1)
+
+    def test_trial_testcase_todo_property(self, testdir):
+        testpath = testdir.makepyfile("""
+            from twisted.trial import unittest
+            class MyTestCase(unittest.TestCase):
+                todo = 'dont run'
+                def test_func(self):
+                    assert 0
+            """)
+        reprec = testdir.inline_run(testpath, "-s")
+        reprec.assertoutcome(skipped=1)
+
+    def test_trial_testfunction_todo_property(self, testdir):
+        testpath = testdir.makepyfile("""
+            from twisted.trial import unittest
+            class MyTestCase(unittest.TestCase):
+                def test_func(self):
+                    assert 0
+                test_func.todo = 'dont run'
+            """)
+        reprec = testdir.inline_run(testpath, "-s", *self.ignore_unclosed_socket_warning)
+        reprec.assertoutcome(skipped=1)
+
 
 def test_djangolike_testcase(testdir):
     # contributed from Morten Breekevold
@@ -598,7 +593,7 @@ def test_unittest_not_shown_in_traceback(testdir):
         class t(unittest.TestCase):
             def test_hello(self):
                 x = 3
-                self.assertEquals(x, 4)
+                self.assertEqual(x, 4)
     """)
     res = testdir.runpytest()
     assert "failUnlessEqual" not in res.stdout.str()

testing/test_warnings.py

@@ -33,6 +33,7 @@ def test_func():
     })
 
 
+@pytest.mark.filterwarnings('always')
 def test_normal_flow(testdir, pyfile_with_warnings):
     """
     Check that the warnings section is displayed, containing test node ids followed by
@@ -54,6 +55,7 @@ def test_normal_flow(testdir, pyfile_with_warnings):
     assert result.stdout.str().count('test_normal_flow.py::test_func') == 1
 
 
+@pytest.mark.filterwarnings('always')
 def test_setup_teardown_warnings(testdir, pyfile_with_warnings):
     testdir.makepyfile('''
         import warnings
@@ -115,6 +117,7 @@ def test_ignore(testdir, pyfile_with_warnings, method):
 
 @pytest.mark.skipif(sys.version_info < (3, 0),
                     reason='warnings message is unicode is ok in python3')
+@pytest.mark.filterwarnings('always')
 def test_unicode(testdir, pyfile_with_warnings):
     testdir.makepyfile('''
         # -*- coding: utf8 -*-
@@ -152,6 +155,7 @@ def fix():
             warnings.warn(u"测试")
             yield
 
+        @pytest.mark.filterwarnings('always')
         def test_func(fix):
             pass
     ''')