diff --git a/src/compiler/checker.ts b/src/compiler/checker.ts
index 8e24fee20606d..d619af3e745fd 100644
--- a/src/compiler/checker.ts
+++ b/src/compiler/checker.ts
@@ -32472,17 +32472,6 @@ namespace ts {
                     }
                 }
             }
-            const restType = getEffectiveRestType(context);
-            if (restType && restType.flags & TypeFlags.TypeParameter) {
-                // The contextual signature has a generic rest parameter. We first instantiate the contextual
-                // signature (without fixing type parameters) and assign types to contextually typed parameters.
-                const instantiatedContext = instantiateSignature(context, inferenceContext.nonFixingMapper);
-                assignContextualParameterTypes(signature, instantiatedContext);
-                // We then infer from a tuple type representing the parameters that correspond to the contextual
-                // rest parameter.
-                const restPos = getParameterCount(context) - 1;
-                inferTypes(inferenceContext.inferences, getRestTypeAtPosition(signature, restPos), restType);
-            }
         }
 
         function assignContextualParameterTypes(signature: Signature, context: Signature) {
@@ -32998,10 +32987,15 @@ namespace ts {
                     if (isContextSensitive(node)) {
                         if (contextualSignature) {
                             const inferenceContext = getInferenceContext(node);
+                            let instantiatedContextualSignature: Signature | undefined;
                             if (checkMode && checkMode & CheckMode.Inferential) {
                                 inferFromAnnotatedParameters(signature, contextualSignature, inferenceContext!);
+                                const restType = getEffectiveRestType(contextualSignature);
+                                if (restType && restType.flags & TypeFlags.TypeParameter) {
+                                    instantiatedContextualSignature = instantiateSignature(contextualSignature, inferenceContext!.nonFixingMapper);
+                                }
                             }
-                            const instantiatedContextualSignature = inferenceContext ?
+                            instantiatedContextualSignature ||= inferenceContext ?
                                 instantiateSignature(contextualSignature, inferenceContext.mapper) : contextualSignature;
                             assignContextualParameterTypes(signature, instantiatedContextualSignature);
                         }
diff --git a/tests/baselines/reference/inferredRestTypeFixedOnce.js b/tests/baselines/reference/inferredRestTypeFixedOnce.js
new file mode 100644
index 0000000000000..0142365cc0ced
--- /dev/null
+++ b/tests/baselines/reference/inferredRestTypeFixedOnce.js
@@ -0,0 +1,10 @@
+//// [inferredRestTypeFixedOnce.ts]
+function wrap<Args extends unknown[]>(_: (...args: Args) => void) {}
+wrap(({ cancelable } = {}) => {});
+
+
+//// [inferredRestTypeFixedOnce.js]
+function wrap(_) { }
+wrap(function (_a) {
+    var _b = _a === void 0 ? {} : _a, cancelable = _b.cancelable;
+});
diff --git a/tests/baselines/reference/inferredRestTypeFixedOnce.symbols b/tests/baselines/reference/inferredRestTypeFixedOnce.symbols
new file mode 100644
index 0000000000000..56bc1ab941c98
--- /dev/null
+++ b/tests/baselines/reference/inferredRestTypeFixedOnce.symbols
@@ -0,0 +1,12 @@
+=== tests/cases/compiler/inferredRestTypeFixedOnce.ts ===
+function wrap<Args extends unknown[]>(_: (...args: Args) => void) {}
+>wrap : Symbol(wrap, Decl(inferredRestTypeFixedOnce.ts, 0, 0))
+>Args : Symbol(Args, Decl(inferredRestTypeFixedOnce.ts, 0, 14))
+>_ : Symbol(_, Decl(inferredRestTypeFixedOnce.ts, 0, 38))
+>args : Symbol(args, Decl(inferredRestTypeFixedOnce.ts, 0, 42))
+>Args : Symbol(Args, Decl(inferredRestTypeFixedOnce.ts, 0, 14))
+
+wrap(({ cancelable } = {}) => {});
+>wrap : Symbol(wrap, Decl(inferredRestTypeFixedOnce.ts, 0, 0))
+>cancelable : Symbol(cancelable, Decl(inferredRestTypeFixedOnce.ts, 1, 7))
+
diff --git a/tests/baselines/reference/inferredRestTypeFixedOnce.types b/tests/baselines/reference/inferredRestTypeFixedOnce.types
new file mode 100644
index 0000000000000..718c82d919121
--- /dev/null
+++ b/tests/baselines/reference/inferredRestTypeFixedOnce.types
@@ -0,0 +1,13 @@
+=== tests/cases/compiler/inferredRestTypeFixedOnce.ts ===
+function wrap<Args extends unknown[]>(_: (...args: Args) => void) {}
+>wrap : <Args extends unknown[]>(_: (...args: Args) => void) => void
+>_ : (...args: Args) => void
+>args : Args
+
+wrap(({ cancelable } = {}) => {});
+>wrap(({ cancelable } = {}) => {}) : void
+>wrap : <Args extends unknown[]>(_: (...args: Args) => void) => void
+>({ cancelable } = {}) => {} : ({ cancelable }?: { cancelable: any; }) => void
+>cancelable : any
+>{} : {}
+
diff --git a/tests/cases/compiler/inferredRestTypeFixedOnce.ts b/tests/cases/compiler/inferredRestTypeFixedOnce.ts
new file mode 100644
index 0000000000000..2424b51a25bb6
--- /dev/null
+++ b/tests/cases/compiler/inferredRestTypeFixedOnce.ts
@@ -0,0 +1,2 @@
+function wrap<Args extends unknown[]>(_: (...args: Args) => void) {}
+wrap(({ cancelable } = {}) => {});