Replace Jedi with Jedi LSP (#17273)

* Remove the experiment

* News file

* Fix linting issue

* Turn JediLSP option into Jedi

* Remove commented code

* Update comment

* Always register no LS activator

* Add basic 2.7 LS support (Pylance or None)

* Skip tests on Python 2.7

* Fix unit tests

* Default to Pylance

* Fix unit tests

* Set default LS to Jedi

* Install JediLSP requirements for testing

* Change default back to none

* Typo

* Turn condition around to match comment

* Remove JediLSP requirements

* Default to None

* Install JediLSP requirements

* Install JediLSP reqs for smoke tests too

* Install JediLSP reqs in coverage step

* Undo PR check workflow changes

* Turns out it won't run if there's no 'run' keyword

* Change condition order

* Add activation service unit tests

* Remove comment

* Add NoLanguageServerExtensionActivator check

* Delete Jedi tests

* Change condition order

* Install Jedi LSP reqs in nightly coverage

Author: kimadeline

.github/workflows/nightly-coverage.yml

@@ -51,6 +51,7 @@ jobs:
           python -m pip --disable-pip-version-check install -t ./pythonFiles/lib/python --no-cache-dir --implementation py --no-deps --upgrade -r requirements.txt --no-user
           # We need to have debugpy so that tests relying on it keep passing, but we don't need install_debugpy's logic in the test phase.
           python -m pip --disable-pip-version-check install -t ./pythonFiles/lib/python --no-cache-dir --implementation py --no-deps --upgrade --pre debugpy
+          python -m pip --disable-pip-version-check install -t ./pythonFiles/lib/jedilsp --no-cache-dir --implementation py --no-deps --upgrade -r ./jedils_requirements.txt
 
       - name: Install test requirements
         run: python -m pip install --upgrade -r build/test-requirements.txt

.github/workflows/pr-check.yml

@@ -100,6 +100,10 @@ jobs:
           # We need to have debugpy so that tests relying on it keep passing, but we don't need install_debugpy's logic in the test phase.
           python -m pip --disable-pip-version-check install -t ./pythonFiles/lib/python --no-cache-dir --implementation py --no-deps --upgrade --pre debugpy
 
+      - name: Install Jedi LSP requirements
+        run: python -m pip --disable-pip-version-check install -t ./pythonFiles/lib/jedilsp --no-cache-dir --implementation py --no-deps --upgrade -r ./jedils_requirements.txt
+        if: startsWith(matrix.python, 3.)
+
       - name: Install test requirements
         run: python -m pip install --upgrade -r build/test-requirements.txt
 
@@ -388,6 +392,7 @@ jobs:
           python -m pip --disable-pip-version-check install -t ./pythonFiles/lib/python --no-cache-dir --implementation py --no-deps --upgrade -r requirements.txt --no-user
           # We need to have debugpy so that tests relying on it keep passing, but we don't need install_debugpy's logic in the test phase.
           python -m pip --disable-pip-version-check install -t ./pythonFiles/lib/python --no-cache-dir --implementation py --no-deps --upgrade --pre debugpy
+          python -m pip --disable-pip-version-check install -t ./pythonFiles/lib/jedilsp --no-cache-dir --implementation py --no-deps --upgrade -r ./jedils_requirements.txt
 
       - name: Install test requirements
         run: python -m pip install --upgrade -r build/test-requirements.txt

news/1 Enhancements/11995.md

@@ -0,0 +1 @@
+Phase out Jedi 0.17, and use Jedi behind a language server protocol as the Jedi option.

package.json

@@ -645,7 +645,6 @@
                             "pythonDeprecatePythonPath",
                             "pythonDiscoveryModule",
                             "pythonDiscoveryModuleWithoutWatcher",
-                            "pythonJediLSP",
                             "pythonSortEnvs",
                             "pythonSurveyNotification",
                             "pythonTensorboardExperiment",
@@ -666,7 +665,6 @@
                             "pythonDeprecatePythonPath",
                             "pythonDiscoveryModule",
                             "pythonDiscoveryModuleWithoutWatcher",
-                            "pythonJediLSP",
                             "pythonSortEnvs",
                             "pythonSurveyNotification",
                             "pythonTensorboardExperiment",
@@ -770,13 +768,11 @@
                     "enum": [
                         "Default",
                         "Jedi",
-                        "JediLSP",
                         "Pylance",
                         "None"
                     ],
                     "enumDescriptions": [
                         "Automatically select a language server: Pylance if installed and available, otherwise fallback to Jedi.",
-                        "Use Jedi as a language server.",
                         "Use Jedi behind the Language Server Protocol (LSP) as a language server.",
                         "Use Pylance as a language server.",
                         "Disable language server capabilities."

src/client/activation/activationService.ts

@@ -3,7 +3,7 @@
 import '../common/extensions';
 
 import { inject, injectable } from 'inversify';
-import { ConfigurationChangeEvent, Disposable, OutputChannel, Uri } from 'vscode';
+import { ConfigurationChangeEvent, ConfigurationTarget, Disposable, OutputChannel, Uri } from 'vscode';
 
 import { LSNotSupportedDiagnosticServiceId } from '../application/diagnostics/checks/lsNotSupported';
 import { IDiagnosticsService } from '../application/diagnostics/types';
@@ -56,6 +56,8 @@ export class LanguageServerExtensionActivationService
 
     private readonly workspaceService: IWorkspaceService;
 
+    private readonly configurationService: IConfigurationService;
+
     private readonly output: OutputChannel;
 
     private readonly interpreterService: IInterpreterService;
@@ -69,6 +71,7 @@ export class LanguageServerExtensionActivationService
         @inject(IPersistentStateFactory) private stateFactory: IPersistentStateFactory,
     ) {
         this.workspaceService = this.serviceContainer.get<IWorkspaceService>(IWorkspaceService);
+        this.configurationService = this.serviceContainer.get<IConfigurationService>(IConfigurationService);
         this.interpreterService = this.serviceContainer.get<IInterpreterService>(IInterpreterService);
         this.output = this.serviceContainer.get<OutputChannel>(IOutputChannel, STANDARD_OUTPUT_CHANNEL);
 
@@ -87,8 +90,8 @@ export class LanguageServerExtensionActivationService
             this.serviceContainer.get<IExtensions>(IExtensions),
             this.serviceContainer.get<IApplicationShell>(IApplicationShell),
             this.serviceContainer.get<ICommandManager>(ICommandManager),
-            this.serviceContainer.get<IWorkspaceService>(IWorkspaceService),
-            this.serviceContainer.get<IConfigurationService>(IConfigurationService),
+            this.workspaceService,
+            this.configurationService,
         );
         disposables.push(this.languageServerChangeHandler);
     }
@@ -228,6 +231,9 @@ export class LanguageServerExtensionActivationService
         key: string,
     ): Promise<RefCountedLanguageServer> {
         let serverType = this.getCurrentLanguageServerType();
+
+        this.updateLanguageServerSetting(resource);
+
         if (serverType === LanguageServerType.Microsoft) {
             const lsNotSupportedDiagnosticService = this.serviceContainer.get<IDiagnosticsService>(
                 IDiagnosticsService,
@@ -243,22 +249,13 @@ export class LanguageServerExtensionActivationService
             }
         }
 
-        if (serverType === LanguageServerType.JediLSP && interpreter && interpreter.version) {
-            if (interpreter.version.major < 3 || (interpreter.version.major === 3 && interpreter.version.minor < 6)) {
-                sendTelemetryEvent(EventName.JEDI_FALLBACK);
-                serverType = LanguageServerType.Jedi;
-            }
-        }
-
-        // If Pylance was chosen via the default and the interpreter is Python 2, fall back to
-        // Jedi. If Pylance was explicitly chosen, continue anyway, even if Pylance won't work
-        // as expected, matching pre-default behavior.
-        if (this.getCurrentLanguageServerTypeIsDefault()) {
-            if (serverType === LanguageServerType.Node && interpreter && interpreter.version) {
-                if (interpreter.version.major < 3) {
-                    sendTelemetryEvent(EventName.JEDI_FALLBACK);
-                    serverType = LanguageServerType.Jedi;
-                }
+        // If the interpreter is Python 2 and the LS setting is explicitly set to Jedi, turn it off.
+        // If set to Default, use Pylance.
+        if (interpreter && (interpreter.version?.major ?? 0) < 3) {
+            if (serverType === LanguageServerType.Jedi) {
+                serverType = LanguageServerType.None;
+            } else if (this.getCurrentLanguageServerTypeIsDefault()) {
+                serverType = LanguageServerType.Node;
             }
         }
 
@@ -348,4 +345,23 @@ export class LanguageServerExtensionActivationService
         const values = await Promise.all([...this.cache.values()]);
         values.forEach((v) => (v.clearAnalysisCache ? v.clearAnalysisCache() : noop()));
     }
+
+    private updateLanguageServerSetting(resource: Resource): void {
+        // Update settings.json value to Jedi if it's JediLSP.
+        const settings = this.workspaceService
+            .getConfiguration('python', resource)
+            .inspect<LanguageServerType>('languageServer');
+
+        let configTarget: ConfigurationTarget;
+
+        if (settings?.workspaceValue === LanguageServerType.JediLSP) {
+            configTarget = ConfigurationTarget.Workspace;
+        } else if (settings?.globalValue === LanguageServerType.JediLSP) {
+            configTarget = ConfigurationTarget.Global;
+        } else {
+            return;
+        }
+
+        this.configurationService.updateSetting('languageServer', LanguageServerType.Jedi, resource, configTarget);
+    }
 }

src/client/activation/common/defaultlanguageServer.ts

@@ -3,8 +3,7 @@
 
 import { injectable } from 'inversify';
 import { PYLANCE_EXTENSION_ID } from '../../common/constants';
-import { JediLSP } from '../../common/experiments/groups';
-import { IDefaultLanguageServer, IExperimentService, IExtensions, DefaultLSType } from '../../common/types';
+import { IDefaultLanguageServer, IExtensions, DefaultLSType } from '../../common/types';
 import { IServiceManager } from '../../ioc/types';
 import { ILSExtensionApi } from '../node/languageServerFolderService';
 import { LanguageServerType } from '../types';
@@ -19,26 +18,20 @@ class DefaultLanguageServer implements IDefaultLanguageServer {
 }
 
 export async function setDefaultLanguageServer(
-    experimentService: IExperimentService,
     extensions: IExtensions,
     serviceManager: IServiceManager,
 ): Promise<void> {
-    const lsType = await getDefaultLanguageServer(experimentService, extensions);
+    const lsType = await getDefaultLanguageServer(extensions);
     serviceManager.addSingletonInstance<IDefaultLanguageServer>(
         IDefaultLanguageServer,
         new DefaultLanguageServer(lsType),
     );
 }
 
-async function getDefaultLanguageServer(
-    experimentService: IExperimentService,
-    extensions: IExtensions,
-): Promise<DefaultLSType> {
+async function getDefaultLanguageServer(extensions: IExtensions): Promise<DefaultLSType> {
     if (extensions.getExtension<ILSExtensionApi>(PYLANCE_EXTENSION_ID)) {
         return LanguageServerType.Node;
     }
 
-    return (await experimentService.inExperiment(JediLSP.experiment))
-        ? LanguageServerType.JediLSP
-        : LanguageServerType.Jedi;
+    return LanguageServerType.Jedi;
 }

src/client/activation/jedi/manager.ts

@@ -45,7 +45,7 @@ export class JediLanguageServerManager implements ILanguageServerManager {
     constructor(
         @inject(IServiceContainer) private readonly serviceContainer: IServiceContainer,
         @inject(ILanguageServerAnalysisOptions)
-        @named(LanguageServerType.JediLSP)
+        @named(LanguageServerType.Jedi)
         private readonly analysisOptions: ILanguageServerAnalysisOptions,
         @inject(ICommandManager) commandManager: ICommandManager,
     ) {
@@ -144,7 +144,7 @@ export class JediLanguageServerManager implements ILanguageServerManager {
         const options = await this.analysisOptions.getAnalysisOptions();
         this.middleware = new LanguageClientMiddleware(
             this.serviceContainer,
-            LanguageServerType.JediLSP,
+            LanguageServerType.Jedi,
             () => this.languageServerProxy?.languageClient,
             this.lsVersion,
         );

src/client/activation/languageClientMiddleware.ts

@@ -24,7 +24,7 @@ export class LanguageClientMiddleware extends LanguageClientMiddlewareBase {
     ) {
         super(serviceContainer, serverType, sendTelemetryEvent, serverVersion);
 
-        if (serverType === LanguageServerType.None || serverType === LanguageServerType.Jedi) {
+        if (serverType === LanguageServerType.None) {
             return;
         }
 

src/client/activation/languageClientMiddlewareBase.ts

@@ -133,7 +133,7 @@ export class LanguageClientMiddlewareBase implements Middleware {
             this.eventName = EventName.PYTHON_LANGUAGE_SERVER_REQUEST;
         } else if (serverType === LanguageServerType.Node) {
             this.eventName = EventName.LANGUAGE_SERVER_REQUEST;
-        } else if (serverType === LanguageServerType.JediLSP) {
+        } else if (serverType === LanguageServerType.Jedi) {
             this.eventName = EventName.JEDI_LANGUAGE_SERVER_REQUEST;
         }
     }

src/client/activation/languageServer/deprecationPrompt.ts

@@ -103,10 +103,7 @@ export class MPLSDeprecationPrompt implements IMPLSDeprecationPrompt {
     }
 
     private async switchLanguageServer(lsType: LanguageServerType.Node | LanguageServerType.Jedi): Promise<void> {
-        let defaultType = this.defaultLanguageServer.defaultLSType;
-        if (defaultType === LanguageServerType.JediLSP) {
-            defaultType = LanguageServerType.Jedi;
-        }
+        const defaultType = this.defaultLanguageServer.defaultLSType;
 
         // If changing to the default, unset the setting instead of explicitly setting it.
         const changeTo = lsType !== defaultType ? lsType : undefined;

src/client/activation/serviceRegistry.ts

@@ -10,7 +10,6 @@ import { DownloadBetaChannelRule, DownloadDailyChannelRule } from './common/down
 import { LanguageServerDownloader } from './common/downloader';
 import { LanguageServerDownloadChannel } from './common/packageRepository';
 import { ExtensionSurveyPrompt } from './extensionSurvey';
-import { JediExtensionActivator } from './jedi';
 import { JediLanguageServerAnalysisOptions } from './jedi/analysisOptions';
 import { JediLanguageClientFactory } from './jedi/languageClientFactory';
 import { JediLanguageServerProxy } from './jedi/languageServerProxy';
@@ -130,34 +129,29 @@ export function registerTypes(serviceManager: IServiceManager, languageServerTyp
             ILanguageServerFolderService,
             NodeLanguageServerFolderService,
         );
-    } else if (languageServerType === LanguageServerType.JediLSP) {
+    } else if (languageServerType === LanguageServerType.Jedi) {
         serviceManager.add<ILanguageServerActivator>(
             ILanguageServerActivator,
             JediLanguageServerActivator,
-            LanguageServerType.JediLSP,
+            LanguageServerType.Jedi,
         );
 
         serviceManager.add<ILanguageServerAnalysisOptions>(
             ILanguageServerAnalysisOptions,
             JediLanguageServerAnalysisOptions,
-            LanguageServerType.JediLSP,
+            LanguageServerType.Jedi,
         );
 
         serviceManager.addSingleton<ILanguageClientFactory>(ILanguageClientFactory, JediLanguageClientFactory);
         serviceManager.add<ILanguageServerManager>(ILanguageServerManager, JediLanguageServerManager);
         serviceManager.add<ILanguageServerProxy>(ILanguageServerProxy, JediLanguageServerProxy);
-    } else if (languageServerType === LanguageServerType.None) {
-        serviceManager.add<ILanguageServerActivator>(
-            ILanguageServerActivator,
-            NoLanguageServerExtensionActivator,
-            LanguageServerType.None,
-        );
     }
+
     serviceManager.add<ILanguageServerActivator>(
         ILanguageServerActivator,
-        JediExtensionActivator,
-        LanguageServerType.Jedi,
-    ); // We fallback to Jedi if for some reason we're unable to use other language servers, hence register this always.
+        NoLanguageServerExtensionActivator,
+        LanguageServerType.None,
+    );
 
     serviceManager.addSingleton<IDownloadChannelRule>(
         IDownloadChannelRule,

src/client/common/configSettings.ts

@@ -293,8 +293,12 @@ export class PythonSettings implements IPythonSettings {
             userLS === 'Default' ||
             !Object.values(LanguageServerType).includes(userLS as LanguageServerType)
         ) {
-            this.languageServer = this.defaultLS?.defaultLSType ?? LanguageServerType.Jedi;
+            this.languageServer = this.defaultLS?.defaultLSType ?? LanguageServerType.None;
             this.languageServerIsDefault = true;
+        } else if (userLS === 'JediLSP') {
+            // Switch JediLSP option to Jedi.
+            this.languageServer = LanguageServerType.Jedi;
+            this.languageServerIsDefault = false;
         } else {
             this.languageServer = userLS as LanguageServerType;
             this.languageServerIsDefault = false;

src/client/common/experiments/groups.ts

@@ -10,10 +10,6 @@ export enum DeprecatePythonPath {
     experiment = 'pythonDeprecatePythonPath',
 }
 
-// Experiment to switch Jedi to use an LSP instead of direct providers
-export enum JediLSP {
-    experiment = 'pythonJediLSP',
-}
 // Experiment to show a prompt asking users to join python mailing list.
 export enum JoinMailingListPromptVariants {
     variant1 = 'pythonJoinMailingListVar1',

src/client/common/types.ts

@@ -547,7 +547,7 @@ export interface IInterpreterPathProxyService {
     get(resource: Resource): string;
 }
 
-export type DefaultLSType = LanguageServerType.Jedi | LanguageServerType.JediLSP | LanguageServerType.Node;
+export type DefaultLSType = LanguageServerType.Jedi | LanguageServerType.Node;
 
 /**
  * Interface used to retrieve the default language server.

src/client/extensionActivation.ts

@@ -14,13 +14,7 @@ import { IApplicationEnvironment, ICommandManager } from './common/application/t
 import { Commands, PYTHON, PYTHON_LANGUAGE, STANDARD_OUTPUT_CHANNEL, UseProposedApi } from './common/constants';
 import { registerTypes as installerRegisterTypes } from './common/installer/serviceRegistry';
 import { IFileSystem } from './common/platform/types';
-import {
-    IConfigurationService,
-    IDisposableRegistry,
-    IExperimentService,
-    IExtensions,
-    IOutputChannel,
-} from './common/types';
+import { IConfigurationService, IDisposableRegistry, IExtensions, IOutputChannel } from './common/types';
 import { noop } from './common/utils/misc';
 import { DebuggerTypeName } from './debugger/constants';
 import { DebugSessionEventDispatcher } from './debugger/extension/hooks/eventHandlerDispatcher';
@@ -117,10 +111,8 @@ async function activateLegacy(ext: ExtensionState): Promise<ActivationResult> {
     debugConfigurationRegisterTypes(serviceManager);
     tensorBoardRegisterTypes(serviceManager);
 
-    const experimentService = serviceContainer.get<IExperimentService>(IExperimentService);
-
     const extensions = serviceContainer.get<IExtensions>(IExtensions);
-    await setDefaultLanguageServer(experimentService, extensions, serviceManager);
+    await setDefaultLanguageServer(extensions, serviceManager);
 
     const configuration = serviceManager.get<IConfigurationService>(IConfigurationService);
     // We should start logging using the log level as soon as possible, so set it as soon as we can access the level.

src/client/jupyter/jupyterIntegration.ts

@@ -212,7 +212,7 @@ export class JupyterExtensionIntegration {
                 const interpreter = !isResource(r) ? r : undefined;
                 const client = await this.languageServerCache.get(resource, interpreter);
 
-                // Some language servers don't support the connection yet. (like Jedi until we switch to LSP)
+                // Some language servers don't support the connection yet.
                 if (client && client.connection && client.capabilities) {
                     return {
                         connection: client.connection,