More straightforward fix

Author: Ivan Levkivskyi

mypy/semanal.py

@@ -253,6 +253,8 @@ def visit_file(self, file_node: MypyFile, fnam: str, options: Options) -> None:
 
         if self.cur_mod_id == 'builtins':
             remove_imported_names_from_symtable(self.globals, 'builtins')
+            for alias_name in ['List', 'Dict', 'Set']:
+                self.globals.pop(alias_name, None)
 
         if '__all__' in self.globals:
             for name, g in self.globals.items():
@@ -3453,9 +3455,6 @@ def remove_imported_names_from_symtable(names: SymbolTable,
             removed.append(name)
     for name in removed:
         del names[name]
-    if module == 'builtins':
-        for alias_name in ['List', 'Dict', 'Set']:
-            names.pop(alias_name, None)
 
 
 def infer_reachability_of_if_statement(s: IfStmt,