Handle recursive type references up to a certain level of expansion in inference (microsoft#38011)

* Handle recursive type references up to a certain level of expansion in inference

* Move object cases into the same conditional branch

Author: weswigham

src/compiler/checker.ts

@@ -18090,15 +18090,32 @@ namespace ts {
         // In addition, this will also detect when an indexed access has been chained off of 5 or more times (which is essentially
         // the dual of the structural comparison), and likewise mark the type as deeply nested, potentially adding false positives
         // for finite but deeply expanding indexed accesses (eg, for `Q[P1][P2][P3][P4][P5]`).
+        // It also detects when a recursive type reference has expanded 5 or more times, eg, if the true branch of
+        // `type A<T> = null extends T ? [A<NonNullable<T>>] : [T]`
+        // has expanded into `[A<NonNullable<NonNullable<NonNullable<NonNullable<NonNullable<T>>>>>>]`
+        // in such cases we need to terminate the expansion, and we do so here.
         function isDeeplyNestedType(type: Type, stack: Type[], depth: number): boolean {
             // We track all object types that have an associated symbol (representing the origin of the type)
-            if (depth >= 5 && type.flags & TypeFlags.Object && !isObjectOrArrayLiteralType(type)) {
-                const symbol = type.symbol;
-                if (symbol) {
+            if (depth >= 5 && type.flags & TypeFlags.Object) {
+                if (!isObjectOrArrayLiteralType(type)) {
+                    const symbol = type.symbol;
+                    if (symbol) {
+                        let count = 0;
+                        for (let i = 0; i < depth; i++) {
+                            const t = stack[i];
+                            if (t.flags & TypeFlags.Object && t.symbol === symbol) {
+                                count++;
+                                if (count >= 5) return true;
+                            }
+                        }
+                    }
+                }
+                if (getObjectFlags(type) && ObjectFlags.Reference && !!(type as TypeReference).node) {
+                    const root = (type as TypeReference).target;
                     let count = 0;
                     for (let i = 0; i < depth; i++) {
                         const t = stack[i];
-                        if (t.flags & TypeFlags.Object && t.symbol === symbol) {
+                        if (getObjectFlags(t) && ObjectFlags.Reference && !!(t as TypeReference).node && (t as TypeReference).target === root) {
                             count++;
                             if (count >= 5) return true;
                         }
@@ -19190,6 +19207,8 @@ namespace ts {
             let propagationType: Type;
             let inferencePriority = InferencePriority.MaxValue;
             let allowComplexConstraintInference = true;
+            let objectTypeComparisonDepth = 0;
+            const targetStack: Type[] = [];
             inferFromTypes(originalSource, originalTarget);
 
             function inferFromTypes(source: Type, target: Type): void {
@@ -19623,15 +19642,27 @@ namespace ts {
                 // its symbol with the instance side which would lead to false positives.
                 const isNonConstructorObject = target.flags & TypeFlags.Object &&
                     !(getObjectFlags(target) & ObjectFlags.Anonymous && target.symbol && target.symbol.flags & SymbolFlags.Class);
-                const symbolOrType = isNonConstructorObject ? isTupleType(target) ? target.target : target.symbol : undefined;
+                const symbolOrType = getObjectFlags(target) & ObjectFlags.Reference && (target as TypeReference).node ? getNormalizedType(target, /*writing*/ false) : isNonConstructorObject ? isTupleType(target) ? target.target : target.symbol : undefined;
                 if (symbolOrType) {
                     if (contains(symbolOrTypeStack, symbolOrType)) {
+                        if (getObjectFlags(target) & ObjectFlags.Reference && (target as TypeReference).node) {
+                            // Don't set the circularity flag for re-encountered recursive type references just because we're already exploring them
+                            return;
+                        }
+                        inferencePriority = InferencePriority.Circularity;
+                        return;
+                    }
+                    targetStack[objectTypeComparisonDepth] = target;
+                    objectTypeComparisonDepth++;
+                    if (isDeeplyNestedType(target, targetStack, objectTypeComparisonDepth)) {
                         inferencePriority = InferencePriority.Circularity;
+                        objectTypeComparisonDepth--;
                         return;
                     }
                     (symbolOrTypeStack || (symbolOrTypeStack = [])).push(symbolOrType);
                     inferFromObjectTypesWorker(source, target);
                     symbolOrTypeStack.pop();
+                    objectTypeComparisonDepth--;
                 }
                 else {
                     inferFromObjectTypesWorker(source, target);

tests/baselines/reference/selfReferencingTypeReferenceInference.js

@@ -0,0 +1,23 @@
+//// [selfReferencingTypeReferenceInference.ts]
+interface Box<T> {
+    __: T
+}
+
+type Recursive<T> =
+    | T
+    | Box<Recursive<T>>
+
+type InferRecursive<T> = T extends Recursive<infer R> ? R : "never!"
+
+// the type we are testing with
+type t1 = Box<string | Box<number | boolean>>
+
+type t2 = InferRecursive<t1>
+type t3 = InferRecursive<Box<string | Box<number | boolean>>> // write t1 explicitly
+
+  // Why is t2 and t3 different??
+  // They have same input type!
+
+//// [selfReferencingTypeReferenceInference.js]
+// Why is t2 and t3 different??
+// They have same input type!

tests/baselines/reference/selfReferencingTypeReferenceInference.symbols

@@ -0,0 +1,49 @@
+=== tests/cases/compiler/selfReferencingTypeReferenceInference.ts ===
+interface Box<T> {
+>Box : Symbol(Box, Decl(selfReferencingTypeReferenceInference.ts, 0, 0))
+>T : Symbol(T, Decl(selfReferencingTypeReferenceInference.ts, 0, 14))
+
+    __: T
+>__ : Symbol(Box.__, Decl(selfReferencingTypeReferenceInference.ts, 0, 18))
+>T : Symbol(T, Decl(selfReferencingTypeReferenceInference.ts, 0, 14))
+}
+
+type Recursive<T> =
+>Recursive : Symbol(Recursive, Decl(selfReferencingTypeReferenceInference.ts, 2, 1))
+>T : Symbol(T, Decl(selfReferencingTypeReferenceInference.ts, 4, 15))
+
+    | T
+>T : Symbol(T, Decl(selfReferencingTypeReferenceInference.ts, 4, 15))
+
+    | Box<Recursive<T>>
+>Box : Symbol(Box, Decl(selfReferencingTypeReferenceInference.ts, 0, 0))
+>Recursive : Symbol(Recursive, Decl(selfReferencingTypeReferenceInference.ts, 2, 1))
+>T : Symbol(T, Decl(selfReferencingTypeReferenceInference.ts, 4, 15))
+
+type InferRecursive<T> = T extends Recursive<infer R> ? R : "never!"
+>InferRecursive : Symbol(InferRecursive, Decl(selfReferencingTypeReferenceInference.ts, 6, 23))
+>T : Symbol(T, Decl(selfReferencingTypeReferenceInference.ts, 8, 20))
+>T : Symbol(T, Decl(selfReferencingTypeReferenceInference.ts, 8, 20))
+>Recursive : Symbol(Recursive, Decl(selfReferencingTypeReferenceInference.ts, 2, 1))
+>R : Symbol(R, Decl(selfReferencingTypeReferenceInference.ts, 8, 50))
+>R : Symbol(R, Decl(selfReferencingTypeReferenceInference.ts, 8, 50))
+
+// the type we are testing with
+type t1 = Box<string | Box<number | boolean>>
+>t1 : Symbol(t1, Decl(selfReferencingTypeReferenceInference.ts, 8, 68))
+>Box : Symbol(Box, Decl(selfReferencingTypeReferenceInference.ts, 0, 0))
+>Box : Symbol(Box, Decl(selfReferencingTypeReferenceInference.ts, 0, 0))
+
+type t2 = InferRecursive<t1>
+>t2 : Symbol(t2, Decl(selfReferencingTypeReferenceInference.ts, 11, 45))
+>InferRecursive : Symbol(InferRecursive, Decl(selfReferencingTypeReferenceInference.ts, 6, 23))
+>t1 : Symbol(t1, Decl(selfReferencingTypeReferenceInference.ts, 8, 68))
+
+type t3 = InferRecursive<Box<string | Box<number | boolean>>> // write t1 explicitly
+>t3 : Symbol(t3, Decl(selfReferencingTypeReferenceInference.ts, 13, 28))
+>InferRecursive : Symbol(InferRecursive, Decl(selfReferencingTypeReferenceInference.ts, 6, 23))
+>Box : Symbol(Box, Decl(selfReferencingTypeReferenceInference.ts, 0, 0))
+>Box : Symbol(Box, Decl(selfReferencingTypeReferenceInference.ts, 0, 0))
+
+  // Why is t2 and t3 different??
+  // They have same input type!

tests/baselines/reference/selfReferencingTypeReferenceInference.types

@@ -0,0 +1,27 @@
+=== tests/cases/compiler/selfReferencingTypeReferenceInference.ts ===
+interface Box<T> {
+    __: T
+>__ : T
+}
+
+type Recursive<T> =
+>Recursive : Recursive<T>
+
+    | T
+    | Box<Recursive<T>>
+
+type InferRecursive<T> = T extends Recursive<infer R> ? R : "never!"
+>InferRecursive : InferRecursive<T>
+
+// the type we are testing with
+type t1 = Box<string | Box<number | boolean>>
+>t1 : t1
+
+type t2 = InferRecursive<t1>
+>t2 : string | number | boolean
+
+type t3 = InferRecursive<Box<string | Box<number | boolean>>> // write t1 explicitly
+>t3 : string | number | boolean
+
+  // Why is t2 and t3 different??
+  // They have same input type!

tests/cases/compiler/selfReferencingTypeReferenceInference.ts

@@ -0,0 +1,18 @@
+interface Box<T> {
+    __: T
+}
+
+type Recursive<T> =
+    | T
+    | Box<Recursive<T>>
+
+type InferRecursive<T> = T extends Recursive<infer R> ? R : "never!"
+
+// the type we are testing with
+type t1 = Box<string | Box<number | boolean>>
+
+type t2 = InferRecursive<t1>
+type t3 = InferRecursive<Box<string | Box<number | boolean>>> // write t1 explicitly
+
+  // Why is t2 and t3 different??
+  // They have same input type!