diff --git a/.travis.yml b/.travis.yml
index fd5905316981..b743514ef1c0 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -9,12 +9,16 @@ matrix:
       env: TEST_CMD="./tests/mypy_test.py"
     - python: "2.7"
       env: TEST_CMD="./tests/pytype_test.py --num-parallel=4"
+    - python: "3.5"
+      env: TEST_CMD="./tests/runtime_test.py"
+    - python: "2.7"
+      env: TEST_CMD="./tests/runtime_test.py"
 
 install:
   # pytype needs py-2.7, mypy needs py-3.2+. Additional logic in runtests.py
   - if [[ $TRAVIS_PYTHON_VERSION == '3.6-dev' ]]; then pip install -U flake8==3.2.1 flake8-bugbear>=16.12.2 flake8-pyi>=16.12.2; fi
-  - if [[ $TRAVIS_PYTHON_VERSION == '3.5' ]]; then pip install -U git+git://github.com/python/mypy git+git://github.com/dropbox/typed_ast; fi
-  - if [[ $TRAVIS_PYTHON_VERSION == '2.7' ]]; then pip install -U git+git://github.com/google/pytype; fi
+  - if [[ $TRAVIS_PYTHON_VERSION == '3.5' ]]; then pip install -U pytest git+git://github.com/python/mypy git+git://github.com/dropbox/typed_ast; fi
+  - if [[ $TRAVIS_PYTHON_VERSION == '2.7' ]]; then pip install -U pytest git+git://github.com/google/pytype; fi
 
 script:
   - $TEST_CMD
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index bf1eaa689eb7..4105c0ca105c 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -14,6 +14,7 @@ are important to the project's success.
     * [Contact us](#discussion) before starting significant work.
     * IMPORTANT: For new libraries, [get permission from the library owner first](#adding-a-new-library).
     * Create your stubs [conforming to the coding style](#stub-file-coding-style).
+    * Optionally, [add some unit tests for your stubs](#adding-tests-for-stubs).
     * Make sure `runtests.sh` passes cleanly on Mypy, pytype, and flake8.
 4. [Submit your changes](#submitting-changes):
     * Open a pull request
@@ -213,6 +214,69 @@ documentation.  Whenever you find them disagreeing, model the type
 information after the actual implementation and file an issue on the
 project's tracker to fix their documentation.
 
+### Adding tests for stubs
+
+If you're fixing a stub file due to some false errors reported by your type
+checker, consider adding a unit test that would show that your changes actually
+fix these false errors.
+
+For example, Mypy is showing false errors about a hypothetical `example`
+module:
+
+```python
+import example
+
+example.foo()  # error: "module" has no attribute "foo"
+example.bar(10)  # error: Argument 1 to "bar" has incompatible type "int"
+```
+
+You might start with a pytest test that passes at run-time, but fails
+(incorrectly) during type checking. For a third-party library `example` that
+runs on both Python 2 and 3 put it into
+`test_data/third_party/2and3/example_test.py`:
+
+```python
+def test_foo_bar():
+    import example
+    
+    example.foo()
+    assert example.bar(10) - 10 == 0
+```
+
+Since `example` is a third-party library, create `example_requirements.txt`
+with the requirements specs for your test next to your test file:
+
+```python
+example==1.2.0
+```
+
+pytest will install the requirements automatically via a test fixture.
+
+You should put your third-party imports inside test functions in your test file,
+so the imports are not executed until pytype installs your requirements.
+
+Then, run both static and run-time tests for your test data. Alternatively, run
+them for all the test data:
+
+```python
+./tests/mypy_test.py
+./tests/runtime_test.py
+```
+
+The first one should fail with a false error (since you haven't fixed it yet)
+while the second one should pass, showing that you're using the API of `example`
+correctly.
+
+Next, add the stub file `third_party/2and3/example.pyi` that actually fixes
+these false errors:
+
+```python
+def foo() -> None: ...
+def bar(x: int) -> int: ...
+```
+
+Finally, re-run the tests to make sure the problem has gone.
+
 ## Issue-tracker conventions
 
 We aim to reply to all new issues promptly.  We'll assign a milestone
diff --git a/test_data/conftest.py b/test_data/conftest.py
new file mode 100644
index 000000000000..7bb182a18885
--- /dev/null
+++ b/test_data/conftest.py
@@ -0,0 +1,12 @@
+import re
+import pip
+import pytest
+
+
+@pytest.fixture(scope='module')
+def requirements(request):
+    requirements_path = re.sub(r'(.*)_test\.py', r'\1_requirements.txt',
+                               request.module.__file__)
+    pip.main(['install', '-r', requirements_path])
+    yield
+    # We could uninstall everything here after the module tests finish
diff --git a/test_data/pytest.ini b/test_data/pytest.ini
new file mode 100644
index 000000000000..33740040f9df
--- /dev/null
+++ b/test_data/pytest.ini
@@ -0,0 +1,2 @@
+[pytest]
+usefixtures = requirements
diff --git a/test_data/stdlib/2and3/collections_test.py b/test_data/stdlib/2and3/collections_test.py
new file mode 100644
index 000000000000..6ef1fd533607
--- /dev/null
+++ b/test_data/stdlib/2and3/collections_test.py
@@ -0,0 +1,14 @@
+def test_namedtuple():
+    from collections import namedtuple
+
+    Point = namedtuple('Point', 'x y')
+    p = Point(1, 2)
+
+    assert p == Point(1, 2)
+    assert p == (1, 2)
+    assert p._replace(y=3.14).y == 3.14
+    assert p._asdict()['x'] == 1
+    assert (p.x, p.y) == (1, 2)
+    assert p[0] + p[1] == 3
+    assert p.index(1) == 0
+    assert Point._make([1, 3.14]).y == 3.14
diff --git a/test_data/third_party/2and3/six_requirements.txt b/test_data/third_party/2and3/six_requirements.txt
new file mode 100644
index 000000000000..b6e34eb294e6
--- /dev/null
+++ b/test_data/third_party/2and3/six_requirements.txt
@@ -0,0 +1 @@
+six==1.10.0
diff --git a/test_data/third_party/2and3/six_test.py b/test_data/third_party/2and3/six_test.py
new file mode 100644
index 000000000000..ab9898eb84b7
--- /dev/null
+++ b/test_data/third_party/2and3/six_test.py
@@ -0,0 +1,13 @@
+def test_python_checks():
+    from six import PY2, PY3
+
+    assert PY2 ^ PY3
+
+
+def test_xrange():
+    from six.moves import xrange
+
+    xs = xrange(5)
+    assert xs.__iter__
+    assert xs[0] == 0
+    assert sum(xs) == 10
diff --git a/tests/mypy_test.py b/tests/mypy_test.py
index d474009e4d01..af8ca2c57853 100755
--- a/tests/mypy_test.py
+++ b/tests/mypy_test.py
@@ -63,7 +63,7 @@ def libpath(major, minor):
     versions.append('2and3')
     paths = []
     for v in versions:
-        for top in ['stdlib', 'third_party']:
+        for top in ['stdlib', 'third_party', 'test_data/stdlib', 'test_data/third_party']:
             p = os.path.join(top, v)
             if os.path.isdir(p):
                 paths.append(p)
@@ -124,6 +124,7 @@ def main():
             runs += 1
             flags = ['--python-version', '%d.%d' % (major, minor)]
             flags.append('--strict-optional')
+            flags.append('--check-untyped-defs')
             if (major, minor) >= (3, 6):
                 flags.append('--fast-parser')
             # flags.append('--warn-unused-ignores')  # Fast parser and regular parser disagree.
diff --git a/tests/runtime_test.py b/tests/runtime_test.py
new file mode 100755
index 000000000000..5d5755908c18
--- /dev/null
+++ b/tests/runtime_test.py
@@ -0,0 +1,29 @@
+#!/usr/bin/env python
+
+import pytest
+import sys
+import os
+
+
+def main():
+    if sys.version_info < (3, 0):
+        version_dirs = [
+            '2and3',
+            '2',
+        ]
+    else:
+        version_dirs = [
+            '2and3',
+            '3',
+            '%d.%d' % (sys.version_info[0], sys.version_info[1]),
+        ]
+    top_dirs = ['stdlib', 'third_party']
+    possible_paths = [os.path.join('test_data', t, v)
+                      for t in top_dirs
+                      for v in version_dirs]
+    paths = [path for path in possible_paths if os.path.exists(path)]
+    pytest.main(paths)
+
+
+if __name__ == '__main__':
+    main()