Add configurable concurrency option

.github/workflows/ci.yml

@@ -97,7 +97,7 @@ jobs:
     env:
       TSGO_HEREBY_RACE: ${{ (matrix.config.race && 'true') || 'false' }}
       TSGO_HEREBY_NOEMBED: ${{ (matrix.config.noembed && 'true') || 'false' }}
-      TSGO_HEREBY_CONCURRENT_TEST_PROGRAMS: ${{ (matrix.config.concurrent-test-programs && 'true') || 'false' }}
+      TSGO_HEREBY_TEST_PROGRAM_CONCURRENCY: ${{ (matrix.config.concurrent-test-programs && 'true') || 'false' }}
       TSGO_HEREBY_COVERAGE: ${{ (matrix.config.coverage && 'true') || 'false' }}
 
     steps:

Herebyfile.mjs

@@ -51,6 +51,20 @@ function parseEnvBoolean(name, defaultValue = false) {
     throw new Error(`Invalid value for ${name}: ${value}`);
 }
 
+/**
+ * @param {string} name
+ * @param {string} defaultValue
+ * @returns {string}
+ */
+function parseEnvString(name, defaultValue = "") {
+    name = "TSGO_HEREBY_" + name.toUpperCase();
+    const value = process.env[name];
+    if (!value) {
+        return defaultValue;
+    }
+    return value;
+}
+
 const { values: rawOptions } = parseArgs({
     args: process.argv.slice(2),
     options: {
@@ -65,7 +79,7 @@ const { values: rawOptions } = parseArgs({
 
         race: { type: "boolean", default: parseEnvBoolean("RACE") },
         noembed: { type: "boolean", default: parseEnvBoolean("NOEMBED") },
-        concurrentTestPrograms: { type: "boolean", default: parseEnvBoolean("CONCURRENT_TEST_PROGRAMS") },
+        concurrency: { type: "string", default: parseEnvString("TEST_PROGRAM_CONCURRENCY") },
         coverage: { type: "boolean", default: parseEnvBoolean("COVERAGE") },
     },
     strict: false,
@@ -291,7 +305,7 @@ function goTestFlags(taskName) {
 }
 
 const goTestEnv = {
-    ...(options.concurrentTestPrograms ? { TS_TEST_PROGRAM_SINGLE_THREADED: "false" } : {}),
+    ...(options.concurrency ? { TSGO_TEST_PROGRAM_CONCURRENCY: "false" } : {}),
     // Go test caching takes a long time on Windows.
     // https://github.com/golang/go/issues/72992
     ...(process.platform === "win32" ? { GOFLAGS: "-count=1" } : {}),

internal/compiler/program.go

@@ -26,7 +26,6 @@ type ProgramOptions struct {
 	Host                        CompilerHost
 	Config                      *tsoptions.ParsedCommandLine
 	UseSourceOfProjectReference bool
-	SingleThreaded              core.Tristate
 	CreateCheckerPool           func(*Program) CheckerPool
 	TypingsLocation             string
 	ProjectName                 string
@@ -39,7 +38,11 @@ func (p *ProgramOptions) canUseProjectReferenceSource() bool {
 type Program struct {
 	opts        ProgramOptions
 	nodeModules map[string]*ast.SourceFile
-	checkerPool CheckerPool
+
+	concurrency     core.Concurrency
+	concurrencyOnce sync.Once
+	checkerPool     CheckerPool
+	checkerPoolOnce sync.Once
 
 	comparePathsOptions tspath.ComparePathsOptions
 
@@ -191,9 +194,6 @@ func NewProgram(opts ProgramOptions) *Program {
 	if p.opts.Host == nil {
 		panic("host required")
 	}
-	p.initCheckerPool()
-
-	// p.maxNodeModuleJsDepth = p.options.MaxNodeModuleJsDepth
 
 	// TODO(ercornel): !!! tracing?
 	// tracing?.push(tracing.Phase.Program, "createProgram", { configFilePath: options.configFilePath, rootDir: options.rootDir }, /*separateBeginAndEnd*/ true);
@@ -239,7 +239,6 @@ func (p *Program) UpdateProgram(changedFilePath tspath.Path) (*Program, bool) {
 		currentNodeModulesDepth:     p.currentNodeModulesDepth,
 		usesUriStyleNodeCoreModules: p.usesUriStyleNodeCoreModules,
 	}
-	result.initCheckerPool()
 	index := core.FindIndex(result.files, func(file *ast.SourceFile) bool { return file.Path() == newFile.Path() })
 	result.files = slices.Clone(result.files)
 	result.files[index] = newFile
@@ -248,14 +247,6 @@ func (p *Program) UpdateProgram(changedFilePath tspath.Path) (*Program, bool) {
 	return result, true
 }
 
-func (p *Program) initCheckerPool() {
-	if p.opts.CreateCheckerPool != nil {
-		p.checkerPool = p.opts.CreateCheckerPool(p)
-	} else {
-		p.checkerPool = newCheckerPool(core.IfElse(p.singleThreaded(), 1, 4), p)
-	}
-}
-
 func canReplaceFileInProgram(file1 *ast.SourceFile, file2 *ast.SourceFile) bool {
 	// TODO(jakebailey): metadata??
 	return file2 != nil &&
@@ -299,8 +290,26 @@ func (p *Program) GetConfigFileParsingDiagnostics() []*ast.Diagnostic {
 	return slices.Clip(p.opts.Config.GetConfigFileParsingDiagnostics())
 }
 
+func (p *Program) getConcurrency() core.Concurrency {
+	p.concurrencyOnce.Do(func() {
+		p.concurrency = core.ParseConcurrency(p.Options())
+	})
+	return p.concurrency
+}
+
 func (p *Program) singleThreaded() bool {
-	return p.opts.SingleThreaded.DefaultIfUnknown(p.Options().SingleThreaded).IsTrue()
+	return p.getConcurrency().SingleThreaded()
+}
+
+func (p *Program) getCheckerPool() CheckerPool {
+	p.checkerPoolOnce.Do(func() {
+		if p.opts.CreateCheckerPool != nil {
+			p.checkerPool = p.opts.CreateCheckerPool(p)
+		} else {
+			p.checkerPool = newCheckerPool(p.getConcurrency().CheckerCount(len(p.files)), p)
+		}
+	})
+	return p.checkerPool
 }
 
 func (p *Program) BindSourceFiles() {
@@ -317,11 +326,11 @@ func (p *Program) BindSourceFiles() {
 
 func (p *Program) CheckSourceFiles(ctx context.Context) {
 	wg := core.NewWorkGroup(p.singleThreaded())
-	checkers, done := p.checkerPool.GetAllCheckers(ctx)
+	checkers, done := p.getCheckerPool().GetAllCheckers(ctx)
 	defer done()
 	for _, checker := range checkers {
 		wg.Queue(func() {
-			for file := range p.checkerPool.Files(checker) {
+			for file := range p.getCheckerPool().Files(checker) {
 				checker.CheckSourceFile(ctx, file)
 			}
 		})
@@ -331,19 +340,19 @@ func (p *Program) CheckSourceFiles(ctx context.Context) {
 
 // Return the type checker associated with the program.
 func (p *Program) GetTypeChecker(ctx context.Context) (*checker.Checker, func()) {
-	return p.checkerPool.GetChecker(ctx)
+	return p.getCheckerPool().GetChecker(ctx)
 }
 
 func (p *Program) GetTypeCheckers(ctx context.Context) ([]*checker.Checker, func()) {
-	return p.checkerPool.GetAllCheckers(ctx)
+	return p.getCheckerPool().GetAllCheckers(ctx)
 }
 
 // Return a checker for the given file. We may have multiple checkers in concurrent scenarios and this
 // method returns the checker that was tasked with checking the file. Note that it isn't possible to mix
 // types obtained from different checkers, so only non-type data (such as diagnostics or string
 // representations of types) should be obtained from checkers returned by this method.
 func (p *Program) GetTypeCheckerForFile(ctx context.Context, file *ast.SourceFile) (*checker.Checker, func()) {
-	return p.checkerPool.GetCheckerForFile(ctx, file)
+	return p.getCheckerPool().GetCheckerForFile(ctx, file)
 }
 
 func (p *Program) GetResolvedModule(file ast.HasFileName, moduleReference string, mode core.ResolutionMode) *module.ResolvedModule {
@@ -385,7 +394,7 @@ func (p *Program) GetSuggestionDiagnostics(ctx context.Context, sourceFile *ast.
 
 func (p *Program) GetGlobalDiagnostics(ctx context.Context) []*ast.Diagnostic {
 	var globalDiagnostics []*ast.Diagnostic
-	checkers, done := p.checkerPool.GetAllCheckers(ctx)
+	checkers, done := p.getCheckerPool().GetAllCheckers(ctx)
 	defer done()
 	for _, checker := range checkers {
 		globalDiagnostics = append(globalDiagnostics, checker.GetGlobalDiagnostics()...)
@@ -432,11 +441,11 @@ func (p *Program) getSemanticDiagnosticsForFile(ctx context.Context, sourceFile
 	var fileChecker *checker.Checker
 	var done func()
 	if sourceFile != nil {
-		fileChecker, done = p.checkerPool.GetCheckerForFile(ctx, sourceFile)
+		fileChecker, done = p.getCheckerPool().GetCheckerForFile(ctx, sourceFile)
 		defer done()
 	}
 	diags := slices.Clip(sourceFile.BindDiagnostics())
-	checkers, closeCheckers := p.checkerPool.GetAllCheckers(ctx)
+	checkers, closeCheckers := p.getCheckerPool().GetAllCheckers(ctx)
 	defer closeCheckers()
 
 	// Ask for diags from all checkers; checking one file may add diagnostics to other files.
@@ -525,13 +534,13 @@ func (p *Program) getSuggestionDiagnosticsForFile(ctx context.Context, sourceFil
 	var fileChecker *checker.Checker
 	var done func()
 	if sourceFile != nil {
-		fileChecker, done = p.checkerPool.GetCheckerForFile(ctx, sourceFile)
+		fileChecker, done = p.getCheckerPool().GetCheckerForFile(ctx, sourceFile)
 		defer done()
 	}
 
 	diags := slices.Clip(sourceFile.BindSuggestionDiagnostics)
 
-	checkers, closeCheckers := p.checkerPool.GetAllCheckers(ctx)
+	checkers, closeCheckers := p.getCheckerPool().GetAllCheckers(ctx)
 	defer closeCheckers()
 
 	// Ask for diags from all checkers; checking one file may add diagnostics to other files.
@@ -642,7 +651,7 @@ func (p *Program) SymbolCount() int {
 	for _, file := range p.files {
 		count += file.SymbolCount
 	}
-	checkers, done := p.checkerPool.GetAllCheckers(context.Background())
+	checkers, done := p.getCheckerPool().GetAllCheckers(context.Background())
 	defer done()
 	for _, checker := range checkers {
 		count += int(checker.SymbolCount)
@@ -652,7 +661,7 @@ func (p *Program) SymbolCount() int {
 
 func (p *Program) TypeCount() int {
 	var count int
-	checkers, done := p.checkerPool.GetAllCheckers(context.Background())
+	checkers, done := p.getCheckerPool().GetAllCheckers(context.Background())
 	defer done()
 	for _, checker := range checkers {
 		count += int(checker.TypeCount)
@@ -662,7 +671,7 @@ func (p *Program) TypeCount() int {
 
 func (p *Program) InstantiationCount() int {
 	var count int
-	checkers, done := p.checkerPool.GetAllCheckers(context.Background())
+	checkers, done := p.getCheckerPool().GetAllCheckers(context.Background())
 	defer done()
 	for _, checker := range checkers {
 		count += int(checker.TotalInstantiationCount)

internal/core/compileroptions.go

@@ -146,6 +146,7 @@ type CompilerOptions struct {
 
 	PprofDir       string   `json:"pprofDir,omitzero"`
 	SingleThreaded Tristate `json:"singleThreaded,omitzero"`
+	Concurrency    string   `json:"concurrency,omitzero"`
 	Quiet          Tristate `json:"quiet,omitzero"`
 
 	sourceFileAffectingCompilerOptionsOnce sync.Once

internal/core/concurrency.go

@@ -0,0 +1,76 @@
+package core
+
+import (
+	"os"
+	"runtime"
+	"strconv"
+	"strings"
+	"sync"
+
+	"github.com/microsoft/typescript-go/internal/testutil/race"
+)
+
+type Concurrency struct {
+	checkerCount int
+}
+
+func ParseConcurrency(options *CompilerOptions) Concurrency {
+	if options.SingleThreaded.IsTrue() {
+		return Concurrency{
+			checkerCount: 1,
+		}
+	}
+	return parseConcurrency(options.Concurrency)
+}
+
+func parseConcurrency(s string) Concurrency {
+	checkerCount := 4
+
+	switch strings.ToLower(s) {
+	case "default", "auto", "true", "yes", "on":
+		break
+	case "single", "none", "false", "no", "off":
+		checkerCount = 1
+	case "max":
+		checkerCount = runtime.GOMAXPROCS(0)
+	case "half":
+		checkerCount = max(1, runtime.GOMAXPROCS(0)/2)
+	case "checker-per-file":
+		checkerCount = -1
+	default:
+		if s != "" {
+			if v, err := strconv.Atoi(s); err == nil && v > 0 {
+				checkerCount = v
+			}
+		}
+	}
+
+	return Concurrency{
+		checkerCount: checkerCount,
+	}
+}
+
+func (c Concurrency) SingleThreaded() bool {
+	return c.checkerCount == 1
+}
+
+func (c Concurrency) CheckerCount(numFiles int) int {
+	checkerCount := c.checkerCount
+	if c.checkerCount == -1 {
+		return max(1, numFiles)
+	}
+	return min(max(1, checkerCount), numFiles)
+}
+
+var testProgramConcurrency = sync.OnceValues(func() (concurrency Concurrency, raw string) {
+	// Leave Program in SingleThreaded mode unless explicitly configured or in race mode.
+	v := os.Getenv("TSGO_TEST_PROGRAM_CONCURRENCY")
+	if v == "" && !race.Enabled {
+		v = "single"
+	}
+	return parseConcurrency(v), v
+})
+
+func TestProgramConcurrency() (concurrency Concurrency, raw string) {
+	return testProgramConcurrency()
+}

internal/core/concurrency_test.go

@@ -0,0 +1,59 @@
+package core
+
+import (
+	"runtime"
+	"testing"
+
+	"gotest.tools/v3/assert"
+)
+
+func TestConcurrency(t *testing.T) {
+	t.Parallel()
+
+	tests := []struct {
+		name           string
+		opts           *CompilerOptions
+		numFiles       int
+		singleThreaded bool
+		checkerCount   int
+	}{
+		{"defaults", &CompilerOptions{}, 100, false, 4},
+		{"default", &CompilerOptions{Concurrency: "default"}, 100, false, 4},
+		{"auto", &CompilerOptions{Concurrency: "true"}, 100, false, 4},
+		{"true", &CompilerOptions{Concurrency: "true"}, 100, false, 4},
+		{"yes", &CompilerOptions{Concurrency: "yes"}, 100, false, 4},
+		{"on", &CompilerOptions{Concurrency: "on"}, 100, false, 4},
+		{"singleThreaded", &CompilerOptions{SingleThreaded: TSTrue}, 100, true, 1},
+		{"single", &CompilerOptions{Concurrency: "single"}, 100, true, 1},
+		{"none", &CompilerOptions{Concurrency: "none"}, 100, true, 1},
+		{"false", &CompilerOptions{Concurrency: "false"}, 100, true, 1},
+		{"no", &CompilerOptions{Concurrency: "no"}, 100, true, 1},
+		{"off", &CompilerOptions{Concurrency: "off"}, 100, true, 1},
+		{"max", &CompilerOptions{Concurrency: "max"}, 1000, false, runtime.GOMAXPROCS(0)},
+		{"half", &CompilerOptions{Concurrency: "half"}, 1000, runtime.GOMAXPROCS(0)/2 == 1, runtime.GOMAXPROCS(0) / 2},
+		{"checker-per-file", &CompilerOptions{Concurrency: "checker-per-file"}, 100, false, 100},
+		{"more than files", &CompilerOptions{Concurrency: "1000"}, 100, false, 100},
+		{"10", &CompilerOptions{Concurrency: "10"}, 100, false, 10},
+		{"1", &CompilerOptions{Concurrency: "1"}, 100, true, 1},
+		{"invalid", &CompilerOptions{Concurrency: "i dunno"}, 100, false, 4},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+
+			c := ParseConcurrency(tt.opts)
+			singleThreaded := c.SingleThreaded()
+			checkerCount := c.CheckerCount(tt.numFiles)
+			assert.Equal(t, singleThreaded, tt.singleThreaded)
+			assert.Equal(t, checkerCount, tt.checkerCount)
+		})
+	}
+
+	t.Run("TestProgramConcurrency", func(t *testing.T) {
+		t.Parallel()
+
+		c, _ := TestProgramConcurrency()
+		assert.Assert(t, c.CheckerCount(10000) > 0)
+	})
+}

internal/core/tristate.go

@@ -13,6 +13,10 @@ const (
 	TSTrue
 )
 
+func (t Tristate) IsUnknown() bool {
+	return t == TSUnknown
+}
+
 func (t Tristate) IsTrue() bool {
 	return t == TSTrue
 }