Use daemon for starting jupyter notebooks

news/1 Enhancements/7242.md

@@ -0,0 +1 @@
+Improve startup performance of Jupyter by using a Python daemon.

pythonFiles/datascience/jupyter_daemon.py

@@ -32,6 +32,8 @@ def m_exec_module(self, module_name, args=[], cwd=None, env=None):
             return self._execute_and_capture_output(self._print_kernel_list)
         elif module_name == "jupyter" and args == ["kernelspec", "--version"]:
             return self._execute_and_capture_output(self._print_kernelspec_version)
+        elif module_name == "jupyter" and args[0] == "nbconvert" and args[-1] != "--version":
+            return self._execute_and_capture_output(lambda : self._convert(args))
         else:
             log.info("check base class stuff")
             return super().m_exec_module(module_name, args, cwd, env)
@@ -87,6 +89,13 @@ def _print_kernel_list(self):
         )
         sys.stdout.flush()
 
+    def _convert(self, args):
+        log.info("nbconvert")
+        from nbconvert import nbconvertapp as app
+
+        sys.argv = [""] + args
+        app.main()
+
     def _start_notebook(self, args):
         from notebook import notebookapp as app
 

src/client/common/process/pythonExecutionFactory.ts

@@ -3,6 +3,7 @@
 import { inject, injectable } from 'inversify';
 
 import { IEnvironmentActivationService } from '../../interpreter/activation/types';
+import { IInterpreterService } from '../../interpreter/contracts';
 import { WindowsStoreInterpreter } from '../../interpreter/locators/services/windowsStoreInterpreter';
 import { IWindowsStoreInterpreter } from '../../interpreter/locators/types';
 import { IServiceContainer } from '../../ioc/types';
@@ -52,14 +53,16 @@ export class PythonExecutionFactory implements IPythonExecutionFactory {
         const pythonPath = options.pythonPath ? options.pythonPath : this.configService.getSettings(options.resource).pythonPath;
         const daemonPoolKey = `${pythonPath}#${options.daemonClass || ''}#${options.daemonModule || ''}`;
         const disposables = this.serviceContainer.get<IDisposableRegistry>(IDisposableRegistry);
-        const activatedProcPromise = this.createActivatedEnvironment({ allowEnvironmentFetchExceptions: true, pythonPath: options.pythonPath, resource: options.resource });
-
+        const interpreterService = this.serviceContainer.get<IInterpreterService>(IInterpreterService);
+        const activatedProcPromise = this.createActivatedEnvironment({ allowEnvironmentFetchExceptions: true, pythonPath: pythonPath, resource: options.resource });
+        const interpreterInfoPromise = interpreterService.getInterpreterDetails(pythonPath);
         // Ensure we do not start multiple daemons for the same interpreter.
         // Cache the promise.
         const start = async () => {
+            const interpreter = await interpreterInfoPromise;
             const [activatedProc, activatedEnvVars] = await Promise.all([
                 activatedProcPromise,
-                this.activationHelper.getActivatedEnvironmentVariables(options.resource, undefined, false)
+                this.activationHelper.getActivatedEnvironmentVariables(options.resource, interpreter, true)
             ]);
 
             const daemon = new PythonDaemonExecutionServicePool({...options, pythonPath}, activatedProc!, activatedEnvVars);

src/client/datascience/activation.ts

@@ -4,35 +4,32 @@
 'use strict';
 
 import { inject, injectable } from 'inversify';
-import { TextDocument } from 'vscode';
 import { IExtensionSingleActivationService } from '../activation/types';
-import { IDocumentManager } from '../common/application/types';
 import '../common/extensions';
 import { IPythonExecutionFactory } from '../common/process/types';
 import { IDisposableRegistry } from '../common/types';
 import { debounceAsync, swallowExceptions } from '../common/utils/decorators';
 import { IInterpreterService } from '../interpreter/contracts';
 import { PythonDaemonModule } from './constants';
+import { INotebookEditor, INotebookEditorProvider } from './types';
 
 @injectable()
 export class Activation implements IExtensionSingleActivationService {
     private notebookOpened = false;
     constructor(
-        @inject(IDocumentManager) private readonly documentManager: IDocumentManager,
+        @inject(INotebookEditorProvider) private readonly notebookProvider: INotebookEditorProvider,
         @inject(IInterpreterService) private readonly interpreterService: IInterpreterService,
         @inject(IPythonExecutionFactory) private readonly factory: IPythonExecutionFactory,
         @inject(IDisposableRegistry) private readonly disposables: IDisposableRegistry
     ) {}
     public async activate(): Promise<void> {
-        this.disposables.push(this.documentManager.onDidOpenTextDocument(this.onDidOpenTextDocument, this));
+        this.disposables.push(this.notebookProvider.onDidOpenNotebookEditor(this.onDidOpenNotebookEditor, this));
         this.disposables.push(this.interpreterService.onDidChangeInterpreter(this.onDidChangeInterpreter, this));
     }
 
-    private onDidOpenTextDocument(e: TextDocument) {
-        if (e.fileName.toLowerCase().endsWith('.ipynb')) {
-            this.notebookOpened = true;
-            this.PreWarmDaemonPool().ignoreErrors();
-        }
+    private onDidOpenNotebookEditor(_: INotebookEditor) {
+        this.notebookOpened = true;
+        this.PreWarmDaemonPool().ignoreErrors();
     }
 
     private onDidChangeInterpreter() {

src/client/datascience/interactive-ipynb/nativeEditorProvider.ts

@@ -3,7 +3,7 @@
 'use strict';
 import { inject, injectable } from 'inversify';
 import * as path from 'path';
-import { TextDocument, TextEditor, Uri } from 'vscode';
+import { Event, EventEmitter, TextDocument, TextEditor, Uri } from 'vscode';
 
 import { ICommandManager, IDocumentManager, IWorkspaceService } from '../../common/application/types';
 import { JUPYTER_LANGUAGE } from '../../common/constants';
@@ -20,9 +20,13 @@ export class NativeEditorProvider implements INotebookEditorProvider, IAsyncDisp
 
     private activeEditors: Map<string, INotebookEditor> = new Map<string, INotebookEditor>();
     private executedEditors: Set<string> = new Set<string>();
+    private _onDidOpenNotebookEditor = new EventEmitter<INotebookEditor>();
     private notebookCount: number = 0;
     private openedNotebookCount: number = 0;
     private nextNumber: number = 1;
+    public get onDidOpenNotebookEditor(): Event<INotebookEditor> {
+        return this._onDidOpenNotebookEditor.event;
+    }
     constructor(
         @inject(IServiceContainer) private serviceContainer: IServiceContainer,
         @inject(IAsyncDisposableRegistry) asyncRegistry: IAsyncDisposableRegistry,
@@ -170,6 +174,7 @@ export class NativeEditorProvider implements INotebookEditorProvider, IAsyncDisp
         this.activeEditors.set(e.file.fsPath, e);
         this.disposables.push(e.saved(this.onSavedEditor.bind(this, e.file.fsPath)));
         this.openedNotebookCount += 1;
+        this._onDidOpenNotebookEditor.fire(e);
     }
 
     private onSavedEditor(oldPath: string, e: INotebookEditor) {

src/client/datascience/jupyter/jupyterCommand.ts

@@ -14,7 +14,7 @@ import {
 } from '../../common/process/types';
 import { IEnvironmentActivationService } from '../../interpreter/activation/types';
 import { IInterpreterService, PythonInterpreter } from '../../interpreter/contracts';
-import { JupyterCommands } from '../constants';
+import { JupyterCommands, PythonDaemonModule } from '../constants';
 import { IJupyterCommand, IJupyterCommandFactory } from '../types';
 
 // JupyterCommand objects represent some process that can be launched that should be guaranteed to work because it
@@ -68,9 +68,18 @@ class InterpreterJupyterCommand implements IJupyterCommand {
     protected interpreterPromise: Promise<PythonInterpreter | undefined>;
     private pythonLauncher: Promise<IPythonExecutionService>;
 
-    constructor(protected readonly moduleName: string, protected args: string[], pythonExecutionFactory: IPythonExecutionFactory, private readonly _interpreter: PythonInterpreter) {
+    constructor(protected readonly moduleName: string, protected args: string[], protected readonly pythonExecutionFactory: IPythonExecutionFactory,
+        private readonly _interpreter: PythonInterpreter, isActiveInterpreter: boolean) {
         this.interpreterPromise = Promise.resolve(this._interpreter);
-        this.pythonLauncher = pythonExecutionFactory.createActivatedEnvironment({ resource: undefined, interpreter: _interpreter, allowEnvironmentFetchExceptions: true });
+        this.pythonLauncher = this.interpreterPromise.then(interpreter => {
+            // Create a daemon only if the interpreter is the same as the current interpreter.
+            // We don't want too many daemons (one for each of the users interpreter on their machine).
+            if (isActiveInterpreter) {
+                return pythonExecutionFactory.createDaemon({ daemonModule: PythonDaemonModule, pythonPath: interpreter!.path });
+            } else {
+                return pythonExecutionFactory.createActivatedEnvironment({interpreter: this._interpreter});
+            }
+        });
     }
     public interpreter() : Promise<PythonInterpreter | undefined> {
         return this.interpreterPromise;
@@ -80,14 +89,20 @@ class InterpreterJupyterCommand implements IJupyterCommand {
         const newOptions = { ...options };
         const launcher = await this.pythonLauncher;
         const newArgs = [...this.args, ...args];
-        return launcher.execObservable(newArgs, newOptions);
+        const moduleName = newArgs[1];
+        newArgs.shift(); // Remove '-m'
+        newArgs.shift(); // Remove module name
+        return launcher.execModuleObservable(moduleName, newArgs, newOptions);
     }
 
     public async exec(args: string[], options: SpawnOptions): Promise<ExecutionResult<string>> {
         const newOptions = { ...options };
         const launcher = await this.pythonLauncher;
         const newArgs = [...this.args, ...args];
-        return launcher.exec(newArgs, newOptions);
+        const moduleName = newArgs[1];
+        newArgs.shift(); // Remove '-m'
+        newArgs.shift(); // Remove module name
+        return launcher.execModule(moduleName, newArgs, newOptions);
     }
 }
 
@@ -99,8 +114,8 @@ class InterpreterJupyterCommand implements IJupyterCommand {
  * @implements {IJupyterCommand}
  */
 class InterpreterJupyterNotebookCommand extends InterpreterJupyterCommand {
-    constructor(moduleName: string, args: string[], pythonExecutionFactory: IPythonExecutionFactory, interpreter: PythonInterpreter) {
-        super(moduleName, args, pythonExecutionFactory, interpreter);
+    constructor(moduleName: string, args: string[], pythonExecutionFactory: IPythonExecutionFactory, interpreter: PythonInterpreter, isActiveInterpreter: boolean) {
+        super(moduleName, args, pythonExecutionFactory, interpreter, isActiveInterpreter);
     }
 }
 
@@ -109,19 +124,19 @@ class InterpreterJupyterNotebookCommand extends InterpreterJupyterCommand {
 export class JupyterCommandFactory implements IJupyterCommandFactory {
 
     constructor(
-        @inject(IPythonExecutionFactory) private executionFactory: IPythonExecutionFactory,
-        @inject(IEnvironmentActivationService) private activationHelper : IEnvironmentActivationService,
-        @inject(IProcessServiceFactory) private processServiceFactory: IProcessServiceFactory,
-        @inject(IInterpreterService) private interpreterService: IInterpreterService
+        @inject(IPythonExecutionFactory) private readonly executionFactory : IPythonExecutionFactory,
+        @inject(IEnvironmentActivationService) private readonly activationHelper : IEnvironmentActivationService,
+        @inject(IProcessServiceFactory) private readonly processServiceFactory: IProcessServiceFactory,
+        @inject(IInterpreterService) private readonly interpreterService: IInterpreterService
     ) {
 
     }
 
-    public createInterpreterCommand(command: JupyterCommands, moduleName: string, args: string[], interpreter: PythonInterpreter): IJupyterCommand {
+    public createInterpreterCommand(command: JupyterCommands, moduleName: string, args: string[], interpreter: PythonInterpreter, isActiveInterpreter: boolean): IJupyterCommand {
         if (command === JupyterCommands.NotebookCommand){
-            return new InterpreterJupyterNotebookCommand(moduleName, args, this.executionFactory, interpreter);
+            return new InterpreterJupyterNotebookCommand(moduleName, args, this.executionFactory, interpreter, isActiveInterpreter);
         }
-        return new InterpreterJupyterCommand(moduleName, args, this.executionFactory, interpreter);
+        return new InterpreterJupyterCommand(moduleName, args, this.executionFactory, interpreter, isActiveInterpreter);
     }
 
     public createProcessCommand(exe: string, args: string[]): IJupyterCommand {

src/client/datascience/jupyter/jupyterCommandFinder.ts

@@ -10,13 +10,13 @@ import { IApplicationShell, IWorkspaceService } from '../../common/application/t
 import { Cancellation, createPromiseFromCancellation, wrapCancellationTokens } from '../../common/cancellation';
 import { traceError, traceInfo, traceWarning } from '../../common/logger';
 import { IFileSystem } from '../../common/platform/types';
-import { IProcessService, IProcessServiceFactory, IPythonExecutionFactory, SpawnOptions } from '../../common/process/types';
+import { IProcessService, IProcessServiceFactory, IPythonExecutionFactory, IPythonExecutionService, SpawnOptions } from '../../common/process/types';
 import { IConfigurationService, IDisposableRegistry, ILogger, IPersistentState, IPersistentStateFactory } from '../../common/types';
 import * as localize from '../../common/utils/localize';
 import { StopWatch } from '../../common/utils/stopWatch';
 import { IInterpreterService, IKnownSearchPathsForInterpreters, PythonInterpreter } from '../../interpreter/contracts';
 import { sendTelemetryEvent } from '../../telemetry';
-import { JupyterCommands, RegExpValues, Telemetry } from '../constants';
+import { JupyterCommands, PythonDaemonModule, RegExpValues, Telemetry } from '../constants';
 import { IJupyterCommand, IJupyterCommandFactory } from '../types';
 
 export enum ModuleExistsStatus {
@@ -141,11 +141,13 @@ export class JupyterCommandFinderImpl {
 
         // If the module is found on this interpreter, then we found it.
         if (interpreter && !Cancellation.isCanceled(cancelToken)) {
-            findResult = await this.doesModuleExist(command, interpreter, cancelToken);
+            const [result, activeInterpreter] = await Promise.all([this.doesModuleExist(command, interpreter, cancelToken), this.interpreterService.getActiveInterpreter(undefined)]);
+            findResult = result!;
+            const isActiveInterpreter = activeInterpreter ? activeInterpreter.path === interpreter.path : false;
             if (findResult.status === ModuleExistsStatus.FoundJupyter) {
-                findResult.command = this.commandFactory.createInterpreterCommand(command, 'jupyter', ['-m', 'jupyter', command], interpreter);
+                findResult.command = this.commandFactory.createInterpreterCommand(command, 'jupyter', ['-m', 'jupyter', command], interpreter, isActiveInterpreter);
             } else if (findResult.status === ModuleExistsStatus.Found) {
-                findResult.command = this.commandFactory.createInterpreterCommand(command, command, ['-m', command], interpreter);
+                findResult.command = this.commandFactory.createInterpreterCommand(command, command, ['-m', command], interpreter, isActiveInterpreter);
             }
         }
         return findResult;
@@ -350,13 +352,29 @@ export class JupyterCommandFinderImpl {
 
         return found;
     }
+    private async createExecutionService(interpreter: PythonInterpreter): Promise<IPythonExecutionService> {
+        const [currentInterpreter, pythonService] = await Promise.all([
+            this.interpreterService.getActiveInterpreter(undefined),
+            this.executionFactory.createActivatedEnvironment({ resource: undefined, interpreter, allowEnvironmentFetchExceptions: true })
+        ]);
+
+        // Use daemons for current interpreter, when using any other interpreter, do not use a daemon.
+        // Creating daemons for other interpreters might not be what we want.
+        // E.g. users can have dozens of pipenv or conda environments.
+        // In such cases, we'd end up creating n*3 python processes that are long lived.
+        if (!currentInterpreter || currentInterpreter.path !== interpreter.path){
+            return pythonService!;
+        }
+
+        return this.executionFactory.createDaemon({ daemonModule: PythonDaemonModule, pythonPath: interpreter.path });
+    }
     private async doesModuleExist(moduleName: string, interpreter: PythonInterpreter, cancelToken?: CancellationToken): Promise<IModuleExistsResult> {
         const result: IModuleExistsResult = {
             status: ModuleExistsStatus.NotFound
         };
         if (interpreter && interpreter !== null) {
             const newOptions: SpawnOptions = { throwOnStdErr: false, encoding: 'utf8', token: cancelToken };
-            const pythonService = await this.executionFactory.createActivatedEnvironment({ resource: undefined, interpreter, allowEnvironmentFetchExceptions: true });
+            const pythonService = await this.createExecutionService(interpreter);
 
             // For commands not 'ipykernel' first try them as jupyter commands
             if (moduleName !== JupyterCommands.KernelCreateCommand) {

src/client/datascience/jupyter/jupyterExecution.ts

@@ -59,7 +59,7 @@ export class JupyterExecutionBase implements IJupyterExecution {
         this.commandFinder = serviceContainer.get<JupyterCommandFinder>(JupyterCommandFinder);
         this.kernelService = new KernelService(this, this.commandFinder, asyncRegistry,
             processServiceFactory, interpreterService, fileSystem);
-        this.notebookStarter = new NotebookStarter(executionFactory, this, this.commandFinder,
+        this.notebookStarter = new NotebookStarter(executionFactory, this.commandFinder,
             this.kernelService, fileSystem, serviceContainer);
         this.disposableRegistry.push(this.interpreterService.onDidChangeInterpreter(() => this.onSettingsChanged()));
         this.disposableRegistry.push(this);

src/client/datascience/jupyter/jupyterSession.ts

@@ -174,7 +174,7 @@ export class JupyterSession implements IJupyterSession {
 
     private async waitForIdleOnSession(session: Session.ISession | undefined, timeout: number): Promise<void> {
         if (session && session.kernel) {
-            traceInfo(`Waiting for idle on: ${session.kernel.id} -> ${session.kernel.status}`);
+            traceInfo(`Waiting for idle on (kernel): ${session.kernel.id} -> ${session.kernel.status}`);
 
             const statusChangedPromise = new Promise(resolve => session.kernelChanged.connect((_, e) => e.newValue && e.newValue.status === 'idle' ? resolve() : undefined));
             const checkStatusPromise = new Promise(async resolve => {
@@ -191,7 +191,7 @@ export class JupyterSession implements IJupyterSession {
                 resolve();
             });
             await Promise.race([statusChangedPromise, checkStatusPromise]);
-            traceInfo(`Finished waiting for idle on: ${session.kernel.id} -> ${session.kernel.status}`);
+            traceInfo(`Finished waiting for idle on (kernel): ${session.kernel.id} -> ${session.kernel.status}`);
 
             // If we didn't make it out in ten seconds, indicate an error
             if (session.kernel && session.kernel.status === 'idle') {

src/client/datascience/jupyter/liveshare/hostJupyterServer.ts

@@ -178,7 +178,7 @@ export class HostJupyterServer
                 this.workspaceService);
 
             // Wait for it to be ready
-            traceInfo(`Waiting for idle ${this.id}`);
+            traceInfo(`Waiting for idle (session) ${this.id}`);
             const stopWatch = new StopWatch();
             const idleTimeout = configService.getSettings().datascience.jupyterLaunchTimeout;
             await notebook.waitForIdle(idleTimeout);