diff --git a/src/PowerShellEditorServices/Services/PowerShell/Host/PsesInternalHost.cs b/src/PowerShellEditorServices/Services/PowerShell/Host/PsesInternalHost.cs
index 05d9d0a92..fffed9891 100644
--- a/src/PowerShellEditorServices/Services/PowerShell/Host/PsesInternalHost.cs
+++ b/src/PowerShellEditorServices/Services/PowerShell/Host/PsesInternalHost.cs
@@ -227,10 +227,18 @@ await ExecuteDelegateAsync(
             return true;
         }
 
+        public Task StopAsync()
+        {
+            TriggerShutdown();
+            return Shutdown;
+        }
+
         public void TriggerShutdown()
         {
-            Interlocked.Exchange(ref _shuttingDown, 1);
-            _cancellationContext.CancelCurrentTaskStack();
+            if (Interlocked.Exchange(ref _shuttingDown, 1) == 0)
+            {
+                _cancellationContext.CancelCurrentTaskStack();
+            }
         }
 
         public void SetExit()
diff --git a/test/PowerShellEditorServices.Test/Language/LanguageServiceTests.cs b/test/PowerShellEditorServices.Test/Language/LanguageServiceTests.cs
index ec5d3169d..68a6db95a 100644
--- a/test/PowerShellEditorServices.Test/Language/LanguageServiceTests.cs
+++ b/test/PowerShellEditorServices.Test/Language/LanguageServiceTests.cs
@@ -27,7 +27,6 @@
 
 namespace Microsoft.PowerShell.EditorServices.Test.Language
 {
-    /*
     public class LanguageServiceTests : IDisposable
     {
         private readonly WorkspaceService workspace;
@@ -55,7 +54,7 @@ public LanguageServiceTests()
 
         public void Dispose()
         {
-            // TODO: Dispose of the host
+            _psesHost.StopAsync().GetAwaiter().GetResult();
         }
 
         [Trait("Category", "Completions")]
@@ -526,5 +525,4 @@ private List<SymbolReference> FindSymbolsInFile(ScriptRegion scriptRegion)
                     GetScriptFile(scriptRegion));
         }
     }
-    */
 }
diff --git a/test/PowerShellEditorServices.Test/PsesHostFactory.cs b/test/PowerShellEditorServices.Test/PsesHostFactory.cs
index 6f843f12d..dfe74b836 100644
--- a/test/PowerShellEditorServices.Test/PsesHostFactory.cs
+++ b/test/PowerShellEditorServices.Test/PsesHostFactory.cs
@@ -4,6 +4,7 @@
 using System;
 using System.Collections.Generic;
 using System.Collections.ObjectModel;
+using System.Globalization;
 using System.IO;
 using System.Management.Automation;
 using System.Management.Automation.Host;
@@ -61,7 +62,7 @@ public static PsesInternalHost Create(ILoggerFactory loggerFactory)
                 "PowerShell Editor Services Test Host",
                 "Test.PowerShellEditorServices",
                 new Version("1.0.0"),
-                psHost: null,
+                psHost: new NullPSHost(),
                 TestProfilePaths,
                 featureFlags: Array.Empty<string>(),
                 additionalModules: Array.Empty<string>(),
@@ -79,5 +80,45 @@ public static PsesInternalHost Create(ILoggerFactory loggerFactory)
             return psesHost;
         }
     }
+
+    internal class NullPSHost : PSHost
+    {
+        public override CultureInfo CurrentCulture => CultureInfo.CurrentCulture;
+
+        public override CultureInfo CurrentUICulture => CultureInfo.CurrentUICulture;
+
+        public override Guid InstanceId { get; } = Guid.NewGuid();
+
+        public override string Name => nameof(NullPSHost);
+
+        public override PSHostUserInterface UI { get; } = new NullPSHostUI();
+
+        public override Version Version { get; } = new Version(1, 0, 0);
+
+        public override void EnterNestedPrompt()
+        {
+            // Do nothing
+        }
+
+        public override void ExitNestedPrompt()
+        {
+            // Do nothing
+        }
+
+        public override void NotifyBeginApplication()
+        {
+            // Do nothing
+        }
+
+        public override void NotifyEndApplication()
+        {
+            // Do nothing
+        }
+
+        public override void SetShouldExit(int exitCode)
+        {
+            // Do nothing
+        }
+    }
 }