go/types, types2: abort type unification if no progress is made

Fixes #59740.
For #59750.

Change-Id: I153d0a412bdfb15f81d6999e29691dc093fd0fcb
Reviewed-on: https://go-review.googlesource.com/c/go/+/487197
Reviewed-by: Robert Findley <[email protected]>
Run-TryBot: Robert Griesemer <[email protected]>
Auto-Submit: Robert Griesemer <[email protected]>
TryBot-Result: Gopher Robot <[email protected]>
Reviewed-by: Robert Griesemer <[email protected]>

Author: griesemer

src/cmd/compile/internal/types2/unify.go

@@ -46,7 +46,7 @@ const (
 	// Whether to panic when unificationDepthLimit is reached.
 	// If disabled, a recursion depth overflow results in a (quiet)
 	// unification failure.
-	panicAtUnificationDepthLimit = true
+	panicAtUnificationDepthLimit = false // go.dev/issue/59740
 
 	// If enableCoreTypeUnification is set, unification will consider
 	// the core types, if any, of non-local (unbound) type parameters.

src/go/types/unify.go

@@ -0,0 +1,25 @@
+// Copyright 2023 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package p
+
+type F[T any] func(func(F[T]))
+
+func f(F[int])      {}
+func g[T any](F[T]) {}
+
+func _() {
+	g(f /* ERROR "type func(F[int]) of f does not match F[T] (cannot infer T)" */) // type inference/unification must not panic
+}
+
+// original test case from issue
+
+type List[T any] func(T, func(T, List[T]) T) T
+
+func nil[T any](n T, _ List[T]) T        { return n }
+func cons[T any](h T, t List[T]) List[T] { return func(n T, f func(T, List[T]) T) T { return f(h, t) } }
+
+func nums[T any](t T) List[T] {
+	return cons(t, cons(t, nil /* ERROR "type func(n T, _ List[T]) T of nil[T] does not match inferred type List[T] for List[T]" */ [T]))
+}