incremental testsuite: don't write subsequent step files until they are needed

This matters with namespace packages - since an empty package with
some .2 files in it will be considered a namespace package. This is
truer - by only writing the files at the point of the step. This helps
unblock python#9636

Author: nipunn1313

mypy/test/data.py

@@ -18,7 +18,7 @@
 
 # File modify/create operation: copy module contents from source_path.
 UpdateFile = NamedTuple('UpdateFile', [('module', str),
-                                       ('source_path', str),
+                                       ('content', str),
                                        ('target_path', str)])
 
 # File delete operation: delete module file.
@@ -270,23 +270,28 @@ def setup(self) -> None:
         self.tmpdir = tempfile.TemporaryDirectory(prefix='mypy-test-')
         os.chdir(self.tmpdir.name)
         os.mkdir(test_temp_dir)
-        for path, content in self.files:
-            dir = os.path.dirname(path)
-            os.makedirs(dir, exist_ok=True)
-            with open(path, 'w', encoding='utf8') as f:
-                f.write(content)
 
         # Precalculate steps for find_steps()
         steps: Dict[int, List[FileOperation]] = {}
-        for path, _ in self.files:
+
+        for path, content in self.files:
             m = re.match(r'.*\.([0-9]+)$', path)
             if m:
+                # Skip writing subsequent incremental steps - rather
+                # store them as operations.
                 num = int(m.group(1))
                 assert num >= 2
                 target_path = re.sub(r'\.[0-9]+$', '', path)
                 module = module_from_path(target_path)
-                operation = UpdateFile(module, path, target_path)
+                operation = UpdateFile(module, content, target_path)
                 steps.setdefault(num, []).append(operation)
+            else:
+                # Write the first incremental steps
+                dir = os.path.dirname(path)
+                os.makedirs(dir, exist_ok=True)
+                with open(path, 'w', encoding='utf8') as f:
+                    f.write(content)
+
         for num, paths in self.deleted_paths.items():
             assert num >= 2
             for path in paths:

mypy/test/helpers.py

@@ -406,7 +406,7 @@ def split_lines(*streams: bytes) -> List[str]:
     ]
 
 
-def copy_and_fudge_mtime(source_path: str, target_path: str) -> None:
+def write_and_fudge_mtime(content: str, target_path: str) -> None:
     # In some systems, mtime has a resolution of 1 second which can
     # cause annoying-to-debug issues when a file has the same size
     # after a change. We manually set the mtime to circumvent this.
@@ -419,7 +419,10 @@ def copy_and_fudge_mtime(source_path: str, target_path: str) -> None:
         new_time = os.stat(target_path).st_mtime + 1
 
     # Use retries to work around potential flakiness on Windows (AppVeyor).
-    retry_on_error(lambda: shutil.copy(source_path, target_path))
+    dir = os.path.dirname(target_path)
+    os.makedirs(dir, exist_ok=True)
+    with open(target_path, "w") as target:
+        target.write(content)
 
     if new_time:
         os.utime(target_path, times=(new_time, new_time))
@@ -430,7 +433,7 @@ def perform_file_operations(
     for op in operations:
         if isinstance(op, UpdateFile):
             # Modify/create file
-            copy_and_fudge_mtime(op.source_path, op.target_path)
+            write_and_fudge_mtime(op.content, op.target_path)
         else:
             # Delete file/directory
             if os.path.isdir(op.path):

test-data/unit/fine-grained-follow-imports.test

@@ -638,6 +638,7 @@ import p2.m2
 p1/m1.py:1: error: "int" not callable
 main.py:2: error: Cannot find implementation or library stub for module named "p2.m2"
 main.py:2: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports
+main.py:2: error: Cannot find implementation or library stub for module named "p2"
 ==
 p2/m2.py:1: error: "str" not callable
 p1/m1.py:1: error: "int" not callable