Implement maxNodeModuleJsDepth, noResolve (#1189)

Author: jakebailey

internal/ast/parseoptions.go

@@ -104,13 +104,19 @@ func getExternalModuleIndicator(file *SourceFile, opts ExternalModuleIndicatorOp
 
 func isFileProbablyExternalModule(sourceFile *SourceFile) *Node {
 	for _, statement := range sourceFile.Statements.Nodes {
-		if IsExternalModuleIndicator(statement) {
+		if isAnExternalModuleIndicatorNode(statement) {
 			return statement
 		}
 	}
 	return getImportMetaIfNecessary(sourceFile)
 }
 
+func isAnExternalModuleIndicatorNode(node *Node) bool {
+	return HasSyntacticModifier(node, ModifierFlagsExport) ||
+		IsImportEqualsDeclaration(node) && IsExternalModuleReference(node.AsImportEqualsDeclaration().ModuleReference) ||
+		IsImportDeclaration(node) || IsExportAssignment(node) || IsExportDeclaration(node)
+}
+
 func getImportMetaIfNecessary(sourceFile *SourceFile) *Node {
 	if sourceFile.AsNode().Flags&NodeFlagsPossiblyContainsImportMeta != 0 {
 		return findChildNode(sourceFile.AsNode(), IsImportMeta)

internal/ast/utilities.go

@@ -1615,9 +1615,8 @@ func isCommonJSContainingModuleKind(kind core.ModuleKind) bool {
 }
 
 func IsExternalModuleIndicator(node *Statement) bool {
-	return HasSyntacticModifier(node, ModifierFlagsExport) ||
-		IsImportEqualsDeclaration(node) && IsExternalModuleReference(node.AsImportEqualsDeclaration().ModuleReference) ||
-		IsImportDeclaration(node) || IsExportAssignment(node) || IsExportDeclaration(node)
+	// Exported top-level member indicates moduleness
+	return IsAnyImportOrReExport(node) || IsExportAssignment(node) || HasSyntacticModifier(node, ModifierFlagsExport)
 }
 
 func IsExportNamespaceAsDefaultDeclaration(node *Node) bool {

internal/compiler/fileloader.go

@@ -10,6 +10,7 @@ import (
 	"github.com/microsoft/typescript-go/internal/ast"
 	"github.com/microsoft/typescript-go/internal/collections"
 	"github.com/microsoft/typescript-go/internal/core"
+	"github.com/microsoft/typescript-go/internal/diagnostics"
 	"github.com/microsoft/typescript-go/internal/module"
 	"github.com/microsoft/typescript-go/internal/tsoptions"
 	"github.com/microsoft/typescript-go/internal/tspath"
@@ -64,6 +65,10 @@ func processAllProgramFiles(
 	compilerOptions := opts.Config.CompilerOptions()
 	rootFiles := opts.Config.FileNames()
 	supportedExtensions := tsoptions.GetSupportedExtensions(compilerOptions, nil /*extraFileExtensions*/)
+	var maxNodeModuleJsDepth int
+	if p := opts.Config.CompilerOptions().MaxNodeModuleJsDepth; p != nil {
+		maxNodeModuleJsDepth = *p
+	}
 	loader := fileLoader{
 		opts:               opts,
 		defaultLibraryPath: tspath.GetNormalizedAbsolutePath(opts.Host.DefaultLibraryPath(), opts.Host.GetCurrentDirectory()),
@@ -72,12 +77,11 @@ func processAllProgramFiles(
 			CurrentDirectory:          opts.Host.GetCurrentDirectory(),
 		},
 		parseTasks: &fileLoaderWorker[*parseTask]{
-			wg:          core.NewWorkGroup(singleThreaded),
-			getSubTasks: getSubTasksOfParseTask,
+			wg:       core.NewWorkGroup(singleThreaded),
+			maxDepth: maxNodeModuleJsDepth,
 		},
 		projectReferenceParseTasks: &fileLoaderWorker[*projectReferenceParseTask]{
-			wg:          core.NewWorkGroup(singleThreaded),
-			getSubTasks: getSubTasksOfProjectReferenceParseTask,
+			wg: core.NewWorkGroup(singleThreaded),
 		},
 		rootTasks:           make([]*parseTask, 0, len(rootFiles)+len(libs)),
 		supportedExtensions: core.Flatten(tsoptions.GetSupportedExtensionsWithJsonIfResolveJsonModule(compilerOptions, supportedExtensions)),
@@ -289,30 +293,36 @@ func (p *fileLoader) parseSourceFile(t *parseTask) *ast.SourceFile {
 	return sourceFile
 }
 
-func (p *fileLoader) resolveTripleslashPathReference(moduleName string, containingFile string) string {
+func (p *fileLoader) resolveTripleslashPathReference(moduleName string, containingFile string) resolvedRef {
 	basePath := tspath.GetDirectoryPath(containingFile)
 	referencedFileName := moduleName
 
 	if !tspath.IsRootedDiskPath(moduleName) {
 		referencedFileName = tspath.CombinePaths(basePath, moduleName)
 	}
-	return tspath.NormalizePath(referencedFileName)
+	return resolvedRef{
+		fileName: tspath.NormalizePath(referencedFileName),
+	}
 }
 
 func (p *fileLoader) resolveTypeReferenceDirectives(file *ast.SourceFile, meta ast.SourceFileMetaData) (
-	toParse []string,
+	toParse []resolvedRef,
 	typeResolutionsInFile module.ModeAwareCache[*module.ResolvedTypeReferenceDirective],
 ) {
 	if len(file.TypeReferenceDirectives) != 0 {
-		toParse = make([]string, 0, len(file.TypeReferenceDirectives))
+		toParse = make([]resolvedRef, 0, len(file.TypeReferenceDirectives))
 		typeResolutionsInFile = make(module.ModeAwareCache[*module.ResolvedTypeReferenceDirective], len(file.TypeReferenceDirectives))
 		for _, ref := range file.TypeReferenceDirectives {
 			redirect := p.projectReferenceFileMapper.getRedirectForResolution(file)
 			resolutionMode := getModeForTypeReferenceDirectiveInFile(ref, file, meta, module.GetCompilerOptionsWithRedirect(p.opts.Config.CompilerOptions(), redirect))
 			resolved := p.resolver.ResolveTypeReferenceDirective(ref.FileName, file.FileName(), resolutionMode, redirect)
 			typeResolutionsInFile[module.ModeAwareCacheKey{Name: ref.FileName, Mode: resolutionMode}] = resolved
 			if resolved.IsResolved() {
-				toParse = append(toParse, resolved.ResolvedFileName)
+				toParse = append(toParse, resolvedRef{
+					fileName:      resolved.ResolvedFileName,
+					increaseDepth: resolved.IsExternalLibraryImport,
+					elideOnDepth:  false,
+				})
 			}
 		}
 	}
@@ -322,19 +332,12 @@ func (p *fileLoader) resolveTypeReferenceDirectives(file *ast.SourceFile, meta a
 const externalHelpersModuleNameText = "tslib" // TODO(jakebailey): dedupe
 
 func (p *fileLoader) resolveImportsAndModuleAugmentations(file *ast.SourceFile, meta ast.SourceFileMetaData) (
-	toParse []string,
+	toParse []resolvedRef,
 	resolutionsInFile module.ModeAwareCache[*module.ResolvedModule],
 	importHelpersImportSpecifier *ast.Node,
 	jsxRuntimeImportSpecifier_ *jsxRuntimeImportSpecifier,
 ) {
 	moduleNames := make([]*ast.Node, 0, len(file.Imports())+len(file.ModuleAugmentations)+2)
-	moduleNames = append(moduleNames, file.Imports()...)
-	for _, imp := range file.ModuleAugmentations {
-		if imp.Kind == ast.KindStringLiteral {
-			moduleNames = append(moduleNames, imp)
-		}
-		// Do nothing if it's an Identifier; we don't need to do module resolution for `declare global`.
-	}
 
 	isJavaScriptFile := ast.IsSourceFileJS(file)
 	isExternalModuleFile := ast.IsExternalModule(file)
@@ -359,51 +362,118 @@ func (p *fileLoader) resolveImportsAndModuleAugmentations(file *ast.SourceFile,
 		}
 	}
 
+	importsStart := len(moduleNames)
+
+	moduleNames = append(moduleNames, file.Imports()...)
+	for _, imp := range file.ModuleAugmentations {
+		if imp.Kind == ast.KindStringLiteral {
+			moduleNames = append(moduleNames, imp)
+		}
+		// Do nothing if it's an Identifier; we don't need to do module resolution for `declare global`.
+	}
+
 	if len(moduleNames) != 0 {
-		toParse = make([]string, 0, len(moduleNames))
+		toParse = make([]resolvedRef, 0, len(moduleNames))
+		resolutionsInFile = make(module.ModeAwareCache[*module.ResolvedModule], len(moduleNames))
 
-		resolutions := p.resolveModuleNames(moduleNames, file, meta, redirect)
+		for index, entry := range moduleNames {
+			moduleName := entry.Text()
+			if moduleName == "" {
+				continue
+			}
 
-		resolutionsInFile = make(module.ModeAwareCache[*module.ResolvedModule], len(resolutions))
+			mode := getModeForUsageLocation(file.FileName(), meta, entry, module.GetCompilerOptionsWithRedirect(p.opts.Config.CompilerOptions(), redirect))
+			resolvedModule := p.resolver.ResolveModuleName(moduleName, file.FileName(), mode, redirect)
+			resolutionsInFile[module.ModeAwareCacheKey{Name: moduleName, Mode: mode}] = resolvedModule
 
-		for _, resolution := range resolutions {
-			resolvedFileName := resolution.resolvedModule.ResolvedFileName
-			// TODO(ercornel): !!!: check if from node modules
+			if !resolvedModule.IsResolved() {
+				continue
+			}
 
-			mode := getModeForUsageLocation(file.FileName(), meta, resolution.node, optionsForFile)
-			resolutionsInFile[module.ModeAwareCacheKey{Name: resolution.node.Text(), Mode: mode}] = resolution.resolvedModule
+			resolvedFileName := resolvedModule.ResolvedFileName
+			isFromNodeModulesSearch := resolvedModule.IsExternalLibraryImport
+			// Don't treat redirected files as JS files.
+			isJsFile := !tspath.FileExtensionIsOneOf(resolvedFileName, tspath.SupportedTSExtensionsWithJsonFlat) && p.projectReferenceFileMapper.getRedirectForResolution(ast.NewHasFileName(resolvedFileName, p.toPath(resolvedFileName))) == nil
+			isJsFileFromNodeModules := isFromNodeModulesSearch && isJsFile && strings.Contains(resolvedFileName, "/node_modules/")
 
 			// add file to program only if:
 			// - resolution was successful
 			// - noResolve is falsy
 			// - module name comes from the list of imports
 			// - it's not a top level JavaScript module that exceeded the search max
 
-			// const elideImport = isJSFileFromNodeModules && currentNodeModulesDepth > maxNodeModuleJsDepth;
+			importIndex := index - importsStart
 
-			// Don't add the file if it has a bad extension (e.g. 'tsx' if we don't have '--allowJs')
-			// This may still end up being an untyped module -- the file won't be included but imports will be allowed.
-			hasAllowedExtension := false
-			if optionsForFile.GetResolveJsonModule() {
-				hasAllowedExtension = tspath.FileExtensionIsOneOf(resolvedFileName, tspath.SupportedTSExtensionsWithJsonFlat)
-			} else if optionsForFile.AllowJs.IsTrue() {
-				hasAllowedExtension = tspath.FileExtensionIsOneOf(resolvedFileName, tspath.SupportedJSExtensionsFlat) || tspath.FileExtensionIsOneOf(resolvedFileName, tspath.SupportedTSExtensionsFlat)
-			} else {
-				hasAllowedExtension = tspath.FileExtensionIsOneOf(resolvedFileName, tspath.SupportedTSExtensionsFlat)
-			}
-			shouldAddFile := resolution.resolvedModule.IsResolved() && hasAllowedExtension
-			// TODO(ercornel): !!!: other checks on whether or not to add the file
+			shouldAddFile := moduleName != "" &&
+				getResolutionDiagnostic(optionsForFile, resolvedModule, file) == nil &&
+				!optionsForFile.NoResolve.IsTrue() &&
+				!(isJsFile && !optionsForFile.GetAllowJS()) &&
+				(importIndex < 0 || (importIndex < len(file.Imports()) && (ast.IsInJSFile(file.Imports()[importIndex]) || file.Imports()[importIndex].Flags&ast.NodeFlagsJSDoc == 0)))
 
 			if shouldAddFile {
-				// p.findSourceFile(resolvedFileName, FileIncludeReason{Import, 0})
-				toParse = append(toParse, resolvedFileName)
+				toParse = append(toParse, resolvedRef{
+					fileName:      resolvedFileName,
+					increaseDepth: resolvedModule.IsExternalLibraryImport,
+					elideOnDepth:  isJsFileFromNodeModules,
+				})
 			}
 		}
 	}
 
 	return toParse, resolutionsInFile, importHelpersImportSpecifier, jsxRuntimeImportSpecifier_
 }
 
+// Returns a DiagnosticMessage if we won't include a resolved module due to its extension.
+// The DiagnosticMessage's parameters are the imported module name, and the filename it resolved to.
+// This returns a diagnostic even if the module will be an untyped module.
+func getResolutionDiagnostic(options *core.CompilerOptions, resolvedModule *module.ResolvedModule, file *ast.SourceFile) *diagnostics.Message {
+	needJsx := func() *diagnostics.Message {
+		if options.Jsx != core.JsxEmitNone {
+			return nil
+		}
+		return diagnostics.Module_0_was_resolved_to_1_but_jsx_is_not_set
+	}
+
+	needAllowJs := func() *diagnostics.Message {
+		if options.GetAllowJS() || !options.NoImplicitAny.DefaultIfUnknown(options.Strict).IsTrue() {
+			return nil
+		}
+		return diagnostics.Module_0_was_resolved_to_1_but_resolveJsonModule_is_not_used
+	}
+
+	needResolveJsonModule := func() *diagnostics.Message {
+		if options.GetResolveJsonModule() {
+			return nil
+		}
+		return diagnostics.Module_0_was_resolved_to_1_but_resolveJsonModule_is_not_used
+	}
+
+	needAllowArbitraryExtensions := func() *diagnostics.Message {
+		if file.IsDeclarationFile || options.AllowArbitraryExtensions.IsTrue() {
+			return nil
+		}
+		return diagnostics.Module_0_was_resolved_to_1_but_allowArbitraryExtensions_is_not_set
+	}
+
+	switch resolvedModule.Extension {
+	case tspath.ExtensionTs, tspath.ExtensionDts,
+		tspath.ExtensionMts, tspath.ExtensionDmts,
+		tspath.ExtensionCts, tspath.ExtensionDcts:
+		// These are always allowed.
+		return nil
+	case tspath.ExtensionTsx:
+		return needJsx()
+	case tspath.ExtensionJsx:
+		return core.Coalesce(needJsx(), needAllowJs())
+	case tspath.ExtensionJs, tspath.ExtensionMjs, tspath.ExtensionCjs:
+		return needAllowJs()
+	case tspath.ExtensionJson:
+		return needResolveJsonModule()
+	default:
+		return needAllowArbitraryExtensions()
+	}
+}
+
 func (p *fileLoader) resolveModuleNames(entries []*ast.Node, file *ast.SourceFile, meta ast.SourceFileMetaData, redirect *tsoptions.ParsedCommandLine) []*resolution {
 	if len(entries) == 0 {
 		return nil

internal/compiler/fileloadertask.go

@@ -1,43 +1,75 @@
 package compiler
 
 import (
+	"math"
+	"sync"
+
 	"github.com/microsoft/typescript-go/internal/collections"
 	"github.com/microsoft/typescript-go/internal/core"
 	"github.com/microsoft/typescript-go/internal/tspath"
 )
 
-type fileLoaderWorkerTask interface {
+type fileLoaderWorkerTask[T any] interface {
 	comparable
 	FileName() string
-	start(loader *fileLoader)
+	isLoaded() bool
+	load(loader *fileLoader)
+	getSubTasks() []T
+	shouldIncreaseDepth() bool
+	shouldElideOnDepth() bool
 }
 
-type fileLoaderWorker[K fileLoaderWorkerTask] struct {
+type fileLoaderWorker[K fileLoaderWorkerTask[K]] struct {
 	wg              core.WorkGroup
-	tasksByFileName collections.SyncMap[string, K]
-	getSubTasks     func(t K) []K
+	tasksByFileName collections.SyncMap[string, *queuedTask[K]]
+	maxDepth        int
+}
+
+type queuedTask[K fileLoaderWorkerTask[K]] struct {
+	task        K
+	mu          sync.Mutex
+	lowestDepth int
 }
 
 func (w *fileLoaderWorker[K]) runAndWait(loader *fileLoader, tasks []K) {
-	w.start(loader, tasks)
+	w.start(loader, tasks, 0)
 	w.wg.RunAndWait()
 }
 
-func (w *fileLoaderWorker[K]) start(loader *fileLoader, tasks []K) {
-	if len(tasks) > 0 {
-		for i, task := range tasks {
-			loadedTask, loaded := w.tasksByFileName.LoadOrStore(task.FileName(), task)
-			if loaded {
-				// dedup tasks to ensure correct file order, regardless of which task would be started first
-				tasks[i] = loadedTask
-			} else {
-				w.wg.Queue(func() {
-					task.start(loader)
-					subTasks := w.getSubTasks(task)
-					w.start(loader, subTasks)
-				})
-			}
+func (w *fileLoaderWorker[K]) start(loader *fileLoader, tasks []K, depth int) {
+	for i, task := range tasks {
+		newTask := &queuedTask[K]{task: task, lowestDepth: math.MaxInt}
+		loadedTask, loaded := w.tasksByFileName.LoadOrStore(task.FileName(), newTask)
+		task = loadedTask.task
+		if loaded {
+			tasks[i] = task
+		}
+
+		currentDepth := depth
+		if task.shouldIncreaseDepth() {
+			currentDepth++
+		}
+
+		if task.shouldElideOnDepth() && currentDepth > w.maxDepth {
+			continue
 		}
+
+		w.wg.Queue(func() {
+			loadedTask.mu.Lock()
+			defer loadedTask.mu.Unlock()
+
+			if !task.isLoaded() {
+				task.load(loader)
+			}
+
+			if currentDepth < loadedTask.lowestDepth {
+				// If we're seeing this task at a lower depth than before,
+				// reprocess its subtasks to ensure they are loaded.
+				loadedTask.lowestDepth = currentDepth
+				subTasks := task.getSubTasks()
+				w.start(loader, subTasks, currentDepth)
+			}
+		})
 	}
 }
 
@@ -49,12 +81,12 @@ func (w *fileLoaderWorker[K]) collectWorker(loader *fileLoader, tasks []K, itera
 	var results []tspath.Path
 	for _, task := range tasks {
 		// ensure we only walk each task once
-		if seen.Has(task) {
+		if !task.isLoaded() || seen.Has(task) {
 			continue
 		}
 		seen.Add(task)
 		var subResults []tspath.Path
-		if subTasks := w.getSubTasks(task); len(subTasks) > 0 {
+		if subTasks := task.getSubTasks(); len(subTasks) > 0 {
 			subResults = w.collectWorker(loader, subTasks, iterate, seen)
 		}
 		iterate(task, subResults)