Improve type relation for recursive mapped types

Recursive mapped types usually lead to the error "excess stack depth
comparing types" because of a new type relation rule added in #19564
which says that "A source type T is related to a target type { [P in
keyof T]: X } if T[P] is related to X".

Unfortunately, with self-recursive mapped types like

```ts
D<T> = { [P in keyof T]: D<T[P]> }
```
we get infinite recursion when trying to assign a type
parameter T to D<T>, as T[P] is compared to D<T[P]>, T[P][P] is compared
to D<T[P][P]>, and so on.

We can avoid many of these infinite recursions by replacing occurrences
of D in the template type with its type argument. This works because
mapped types will completely cover the tree, so checking assignability
of the top level implies that checking of lower level would succeed,
even if there are infinitely many levels.

For example:

```ts
D<T> = { [P in keyof T]: D<T[P]> | undefined }
<T>(t: T, dt: D<T>) => { dt = t }
```

would previously check that `T[P]` is assignable to `D<T[P]> |
undefined`. Now, after replacement, it checks that `T[P]` is assignable
to `T[P] | undefined`.

This implementation suffers from 3 limitations:
1. I use aliasSymbol to detect whether a type reference is a
self-recursive one. This only works when the mapped type is at the top
level of a type alias.
2. Not all instances of D<T> are replaced with T, just those in
intersections and unions. I think this covers almost all uses.
3. This doesn't fix #21048, which tries to assign an "off-by-one"
partial-deep type to itself.

Mostly fixes #21592. One repro there has a type alias to a union, and a
mapped type is a member of the union. But this can be split into two
aliases:

```ts
type SafeAnyMap<T> = { [K in keyof T]?: SafeAny<T[K] };
type SafeAny<T> = SafeAnyMap<T> | boolean | string | symbol | number | null | undefined;
```

Author: sandersn

src/compiler/checker.ts

@@ -10131,21 +10131,22 @@ namespace ts {
                     const template = getTemplateTypeFromMappedType(target);
                     const modifiers = getMappedTypeModifiers(target);
                     if (!(modifiers & MappedTypeModifiers.ExcludeOptional)) {
-                    if (template.flags & TypeFlags.IndexedAccess && (<IndexedAccessType>template).objectType === source &&
-                        (<IndexedAccessType>template).indexType === getTypeParameterFromMappedType(target)) {
-                        return Ternary.True;
-                    }
-                    // A source type T is related to a target type { [P in keyof T]: X } if T[P] is related to X.
-                    if (!isGenericMappedType(source) && getConstraintTypeFromMappedType(target) === getIndexType(source)) {
-                        const indexedAccessType = getIndexedAccessType(source, getTypeParameterFromMappedType(target));
-                        const templateType = getTemplateTypeFromMappedType(target);
-                        if (result = isRelatedTo(indexedAccessType, templateType, reportErrors)) {
-                            errorInfo = saveErrorInfo;
-                            return result;
+                        if (template.flags & TypeFlags.IndexedAccess && (<IndexedAccessType>template).objectType === source &&
+                            (<IndexedAccessType>template).indexType === getTypeParameterFromMappedType(target)) {
+                            return Ternary.True;
+                        }
+                        // A source type T is related to a target type { [P in keyof T]: X } if T[P] is related to X
+                        // *after* occurrences of D<T[P]> in X are replaced with T[P].
+                        if (!isGenericMappedType(source) && getConstraintTypeFromMappedType(target) === getIndexType(source)) {
+                            const indexedAccessType = getIndexedAccessType(source, getTypeParameterFromMappedType(target));
+                            const templateType = replaceRecursiveAliasReference(getTemplateTypeFromMappedType(target), target);
+                            if (result = isRelatedTo(indexedAccessType, templateType, reportErrors)) {
+                                errorInfo = saveErrorInfo;
+                                return result;
+                            }
                         }
                     }
                 }
-                }
 
                 if (source.flags & TypeFlags.TypeParameter) {
                     let constraint = getConstraintForRelation(<TypeParameter>source);
@@ -10292,6 +10293,18 @@ namespace ts {
                 return Ternary.False;
             }
 
+            function replaceRecursiveAliasReference(template: Type, target: Type): Type {
+                return mapType(template, t => {
+                    if (t.flags & TypeFlags.Intersection) {
+                        return getIntersectionType((t as IntersectionType).types.map(u => replaceRecursiveAliasReference(u, target)))
+                    }
+                    const hasSharedAliasSymbol = t.aliasSymbol !== undefined &&
+                        t.aliasTypeArguments && t.aliasTypeArguments.length === 1 &&
+                        t.aliasSymbol === target.aliasSymbol;
+                    return hasSharedAliasSymbol ? t.aliasTypeArguments[0] : t;
+                });
+            }
+
             // A type [P in S]: X is related to a type [Q in T]: Y if T is related to S and X' is
             // related to Y, where X' is an instantiation of X in which P is replaced with Q. Notice
             // that S and T are contra-variant whereas X and Y are co-variant.

tests/baselines/reference/recursiveMappedTypeAssignability.errors.txt

@@ -0,0 +1,43 @@
+tests/cases/compiler/recursiveMappedTypeAssignability.ts(17,49): error TS2321: Excessive stack depth comparing types 'T[K]' and 'SafeAny<T[K]>'.
+
+
+==== tests/cases/compiler/recursiveMappedTypeAssignability.ts (1 errors) ====
+    // type D<U> = { [P in keyof U]: D<U[P]> };
+    // <T>(t: T, dt: D<T>) => { dt = t };
+    // type DR<U> = { readonly [P in keyof U]: DR<U[P]> };
+    // <T>(t: T, dt: DR<T>) => { dt = t };
+    // type DP<U> = { [P in keyof U]?: DP<U[P]> };
+    // <T>(t: T, dt: DP<T>) => { dt = t };
+    // type DAP<U> = { [P in keyof U]?: DAP<U[P]> & U[P] };
+    // <T>(t: T, dt: DAP<T>) => { dt = t };
+    
+    // #21592
+    // doesn't work because aliasSymbol isn't set on the literal type
+    // since it's not top-level -- the union is.
+    type SafeAny<T> = {
+        [K in keyof T]?: SafeAny<T[K]>
+    } | boolean | number | string | symbol | null | undefined
+    type DataValidator<T> = {
+        [K in keyof T]?: (v: SafeAny<T[K]>) => v is T[K]
+                                                    ~~~~
+!!! error TS2321: Excessive stack depth comparing types 'T[K]' and 'SafeAny<T[K]>'.
+    }
+    
+    // modified repro with top-level mapped type, which works
+    // because the literal type has aliasSymbol set
+    type SafeAny2<T> = {
+        [K in keyof T]?: SafeAny2<T[K]>
+    }
+    <T>(t: T, sat: SafeAny2<T>) => { sat = t }
+    
+    
+    const fn = <T>(arg: T) => {
+        ((arg2: RecursivePartial<T>) => {
+            // ...
+        })(arg);
+    };
+    
+    type RecursivePartial<T> = {
+        [P in keyof T]?: RecursivePartial<T[P]>;
+    };
+    

tests/baselines/reference/recursiveMappedTypeAssignability.js

@@ -0,0 +1,55 @@
+//// [recursiveMappedTypeAssignability.ts]
+// type D<U> = { [P in keyof U]: D<U[P]> };
+// <T>(t: T, dt: D<T>) => { dt = t };
+// type DR<U> = { readonly [P in keyof U]: DR<U[P]> };
+// <T>(t: T, dt: DR<T>) => { dt = t };
+// type DP<U> = { [P in keyof U]?: DP<U[P]> };
+// <T>(t: T, dt: DP<T>) => { dt = t };
+// type DAP<U> = { [P in keyof U]?: DAP<U[P]> & U[P] };
+// <T>(t: T, dt: DAP<T>) => { dt = t };
+
+// #21592
+// doesn't work because aliasSymbol isn't set on the literal type
+// since it's not top-level -- the union is.
+type SafeAny<T> = {
+    [K in keyof T]?: SafeAny<T[K]>
+} | boolean | number | string | symbol | null | undefined
+type DataValidator<T> = {
+    [K in keyof T]?: (v: SafeAny<T[K]>) => v is T[K]
+}
+
+// modified repro with top-level mapped type, which works
+// because the literal type has aliasSymbol set
+type SafeAny2<T> = {
+    [K in keyof T]?: SafeAny2<T[K]>
+}
+<T>(t: T, sat: SafeAny2<T>) => { sat = t }
+
+
+const fn = <T>(arg: T) => {
+    ((arg2: RecursivePartial<T>) => {
+        // ...
+    })(arg);
+};
+
+type RecursivePartial<T> = {
+    [P in keyof T]?: RecursivePartial<T[P]>;
+};
+
+
+//// [recursiveMappedTypeAssignability.js]
+"use strict";
+// type D<U> = { [P in keyof U]: D<U[P]> };
+// <T>(t: T, dt: D<T>) => { dt = t };
+// type DR<U> = { readonly [P in keyof U]: DR<U[P]> };
+// <T>(t: T, dt: DR<T>) => { dt = t };
+// type DP<U> = { [P in keyof U]?: DP<U[P]> };
+// <T>(t: T, dt: DP<T>) => { dt = t };
+// type DAP<U> = { [P in keyof U]?: DAP<U[P]> & U[P] };
+// <T>(t: T, dt: DAP<T>) => { dt = t };
+(function (t, sat) { sat = t; });
+var fn = function (arg) {
+    (function (arg2) {
+        // ...
+    })(arg);
+};

tests/baselines/reference/recursiveMappedTypeAssignability.symbols

@@ -0,0 +1,95 @@
+=== tests/cases/compiler/recursiveMappedTypeAssignability.ts ===
+// type D<U> = { [P in keyof U]: D<U[P]> };
+// <T>(t: T, dt: D<T>) => { dt = t };
+// type DR<U> = { readonly [P in keyof U]: DR<U[P]> };
+// <T>(t: T, dt: DR<T>) => { dt = t };
+// type DP<U> = { [P in keyof U]?: DP<U[P]> };
+// <T>(t: T, dt: DP<T>) => { dt = t };
+// type DAP<U> = { [P in keyof U]?: DAP<U[P]> & U[P] };
+// <T>(t: T, dt: DAP<T>) => { dt = t };
+
+// #21592
+// doesn't work because aliasSymbol isn't set on the literal type
+// since it's not top-level -- the union is.
+type SafeAny<T> = {
+>SafeAny : Symbol(SafeAny, Decl(recursiveMappedTypeAssignability.ts, 0, 0))
+>T : Symbol(T, Decl(recursiveMappedTypeAssignability.ts, 12, 13))
+
+    [K in keyof T]?: SafeAny<T[K]>
+>K : Symbol(K, Decl(recursiveMappedTypeAssignability.ts, 13, 5))
+>T : Symbol(T, Decl(recursiveMappedTypeAssignability.ts, 12, 13))
+>SafeAny : Symbol(SafeAny, Decl(recursiveMappedTypeAssignability.ts, 0, 0))
+>T : Symbol(T, Decl(recursiveMappedTypeAssignability.ts, 12, 13))
+>K : Symbol(K, Decl(recursiveMappedTypeAssignability.ts, 13, 5))
+
+} | boolean | number | string | symbol | null | undefined
+type DataValidator<T> = {
+>DataValidator : Symbol(DataValidator, Decl(recursiveMappedTypeAssignability.ts, 14, 57))
+>T : Symbol(T, Decl(recursiveMappedTypeAssignability.ts, 15, 19))
+
+    [K in keyof T]?: (v: SafeAny<T[K]>) => v is T[K]
+>K : Symbol(K, Decl(recursiveMappedTypeAssignability.ts, 16, 5))
+>T : Symbol(T, Decl(recursiveMappedTypeAssignability.ts, 15, 19))
+>v : Symbol(v, Decl(recursiveMappedTypeAssignability.ts, 16, 22))
+>SafeAny : Symbol(SafeAny, Decl(recursiveMappedTypeAssignability.ts, 0, 0))
+>T : Symbol(T, Decl(recursiveMappedTypeAssignability.ts, 15, 19))
+>K : Symbol(K, Decl(recursiveMappedTypeAssignability.ts, 16, 5))
+>v : Symbol(v, Decl(recursiveMappedTypeAssignability.ts, 16, 22))
+>T : Symbol(T, Decl(recursiveMappedTypeAssignability.ts, 15, 19))
+>K : Symbol(K, Decl(recursiveMappedTypeAssignability.ts, 16, 5))
+}
+
+// modified repro with top-level mapped type, which works
+// because the literal type has aliasSymbol set
+type SafeAny2<T> = {
+>SafeAny2 : Symbol(SafeAny2, Decl(recursiveMappedTypeAssignability.ts, 17, 1))
+>T : Symbol(T, Decl(recursiveMappedTypeAssignability.ts, 21, 14))
+
+    [K in keyof T]?: SafeAny2<T[K]>
+>K : Symbol(K, Decl(recursiveMappedTypeAssignability.ts, 22, 5))
+>T : Symbol(T, Decl(recursiveMappedTypeAssignability.ts, 21, 14))
+>SafeAny2 : Symbol(SafeAny2, Decl(recursiveMappedTypeAssignability.ts, 17, 1))
+>T : Symbol(T, Decl(recursiveMappedTypeAssignability.ts, 21, 14))
+>K : Symbol(K, Decl(recursiveMappedTypeAssignability.ts, 22, 5))
+}
+<T>(t: T, sat: SafeAny2<T>) => { sat = t }
+>T : Symbol(T, Decl(recursiveMappedTypeAssignability.ts, 24, 1))
+>t : Symbol(t, Decl(recursiveMappedTypeAssignability.ts, 24, 4))
+>T : Symbol(T, Decl(recursiveMappedTypeAssignability.ts, 24, 1))
+>sat : Symbol(sat, Decl(recursiveMappedTypeAssignability.ts, 24, 9))
+>SafeAny2 : Symbol(SafeAny2, Decl(recursiveMappedTypeAssignability.ts, 17, 1))
+>T : Symbol(T, Decl(recursiveMappedTypeAssignability.ts, 24, 1))
+>sat : Symbol(sat, Decl(recursiveMappedTypeAssignability.ts, 24, 9))
+>t : Symbol(t, Decl(recursiveMappedTypeAssignability.ts, 24, 4))
+
+
+const fn = <T>(arg: T) => {
+>fn : Symbol(fn, Decl(recursiveMappedTypeAssignability.ts, 27, 5))
+>T : Symbol(T, Decl(recursiveMappedTypeAssignability.ts, 27, 12))
+>arg : Symbol(arg, Decl(recursiveMappedTypeAssignability.ts, 27, 15))
+>T : Symbol(T, Decl(recursiveMappedTypeAssignability.ts, 27, 12))
+
+    ((arg2: RecursivePartial<T>) => {
+>arg2 : Symbol(arg2, Decl(recursiveMappedTypeAssignability.ts, 28, 6))
+>RecursivePartial : Symbol(RecursivePartial, Decl(recursiveMappedTypeAssignability.ts, 31, 2))
+>T : Symbol(T, Decl(recursiveMappedTypeAssignability.ts, 27, 12))
+
+        // ...
+    })(arg);
+>arg : Symbol(arg, Decl(recursiveMappedTypeAssignability.ts, 27, 15))
+
+};
+
+type RecursivePartial<T> = {
+>RecursivePartial : Symbol(RecursivePartial, Decl(recursiveMappedTypeAssignability.ts, 31, 2))
+>T : Symbol(T, Decl(recursiveMappedTypeAssignability.ts, 33, 22))
+
+    [P in keyof T]?: RecursivePartial<T[P]>;
+>P : Symbol(P, Decl(recursiveMappedTypeAssignability.ts, 34, 5))
+>T : Symbol(T, Decl(recursiveMappedTypeAssignability.ts, 33, 22))
+>RecursivePartial : Symbol(RecursivePartial, Decl(recursiveMappedTypeAssignability.ts, 31, 2))
+>T : Symbol(T, Decl(recursiveMappedTypeAssignability.ts, 33, 22))
+>P : Symbol(P, Decl(recursiveMappedTypeAssignability.ts, 34, 5))
+
+};
+

tests/baselines/reference/recursiveMappedTypeAssignability.types

@@ -0,0 +1,103 @@
+=== tests/cases/compiler/recursiveMappedTypeAssignability.ts ===
+// type D<U> = { [P in keyof U]: D<U[P]> };
+// <T>(t: T, dt: D<T>) => { dt = t };
+// type DR<U> = { readonly [P in keyof U]: DR<U[P]> };
+// <T>(t: T, dt: DR<T>) => { dt = t };
+// type DP<U> = { [P in keyof U]?: DP<U[P]> };
+// <T>(t: T, dt: DP<T>) => { dt = t };
+// type DAP<U> = { [P in keyof U]?: DAP<U[P]> & U[P] };
+// <T>(t: T, dt: DAP<T>) => { dt = t };
+
+// #21592
+// doesn't work because aliasSymbol isn't set on the literal type
+// since it's not top-level -- the union is.
+type SafeAny<T> = {
+>SafeAny : SafeAny<T>
+>T : T
+
+    [K in keyof T]?: SafeAny<T[K]>
+>K : K
+>T : T
+>SafeAny : SafeAny<T>
+>T : T
+>K : K
+
+} | boolean | number | string | symbol | null | undefined
+>null : null
+
+type DataValidator<T> = {
+>DataValidator : DataValidator<T>
+>T : T
+
+    [K in keyof T]?: (v: SafeAny<T[K]>) => v is T[K]
+>K : K
+>T : T
+>v : SafeAny<T[K]>
+>SafeAny : SafeAny<T>
+>T : T
+>K : K
+>v : any
+>T : T
+>K : K
+}
+
+// modified repro with top-level mapped type, which works
+// because the literal type has aliasSymbol set
+type SafeAny2<T> = {
+>SafeAny2 : SafeAny2<T>
+>T : T
+
+    [K in keyof T]?: SafeAny2<T[K]>
+>K : K
+>T : T
+>SafeAny2 : SafeAny2<T>
+>T : T
+>K : K
+}
+<T>(t: T, sat: SafeAny2<T>) => { sat = t }
+><T>(t: T, sat: SafeAny2<T>) => { sat = t } : <T>(t: T, sat: SafeAny2<T>) => void
+>T : T
+>t : T
+>T : T
+>sat : SafeAny2<T>
+>SafeAny2 : SafeAny2<T>
+>T : T
+>sat = t : T
+>sat : SafeAny2<T>
+>t : T
+
+
+const fn = <T>(arg: T) => {
+>fn : <T>(arg: T) => void
+><T>(arg: T) => {    ((arg2: RecursivePartial<T>) => {        // ...    })(arg);} : <T>(arg: T) => void
+>T : T
+>arg : T
+>T : T
+
+    ((arg2: RecursivePartial<T>) => {
+>((arg2: RecursivePartial<T>) => {        // ...    })(arg) : void
+>((arg2: RecursivePartial<T>) => {        // ...    }) : (arg2: RecursivePartial<T>) => void
+>(arg2: RecursivePartial<T>) => {        // ...    } : (arg2: RecursivePartial<T>) => void
+>arg2 : RecursivePartial<T>
+>RecursivePartial : RecursivePartial<T>
+>T : T
+
+        // ...
+    })(arg);
+>arg : T
+
+};
+
+type RecursivePartial<T> = {
+>RecursivePartial : RecursivePartial<T>
+>T : T
+
+    [P in keyof T]?: RecursivePartial<T[P]>;
+>P : P
+>T : T
+>RecursivePartial : RecursivePartial<T>
+>T : T
+>P : P
+
+};
+

tests/cases/compiler/recursiveMappedTypeAssignability.ts

@@ -0,0 +1,37 @@
+// @strict: true
+// type D<U> = { [P in keyof U]: D<U[P]> };
+// <T>(t: T, dt: D<T>) => { dt = t };
+// type DR<U> = { readonly [P in keyof U]: DR<U[P]> };
+// <T>(t: T, dt: DR<T>) => { dt = t };
+// type DP<U> = { [P in keyof U]?: DP<U[P]> };
+// <T>(t: T, dt: DP<T>) => { dt = t };
+// type DAP<U> = { [P in keyof U]?: DAP<U[P]> & U[P] };
+// <T>(t: T, dt: DAP<T>) => { dt = t };
+
+// #21592
+// doesn't work because aliasSymbol isn't set on the literal type
+// since it's not top-level -- the union is.
+type SafeAny<T> = {
+    [K in keyof T]?: SafeAny<T[K]>
+} | boolean | number | string | symbol | null | undefined
+type DataValidator<T> = {
+    [K in keyof T]?: (v: SafeAny<T[K]>) => v is T[K]
+}
+
+// modified repro with top-level mapped type, which works
+// because the literal type has aliasSymbol set
+type SafeAny2<T> = {
+    [K in keyof T]?: SafeAny2<T[K]>
+}
+<T>(t: T, sat: SafeAny2<T>) => { sat = t }
+
+
+const fn = <T>(arg: T) => {
+    ((arg2: RecursivePartial<T>) => {
+        // ...
+    })(arg);
+};
+
+type RecursivePartial<T> = {
+    [P in keyof T]?: RecursivePartial<T[P]>;
+};