diff --git a/CONTRIBUTING - PYTHON_ANALYSIS.md b/CONTRIBUTING - PYTHON_ANALYSIS.md index d90e684e5083..1b67cd884e6a 100644 --- a/CONTRIBUTING - PYTHON_ANALYSIS.md +++ b/CONTRIBUTING - PYTHON_ANALYSIS.md @@ -33,6 +33,7 @@ Visual Studio 2017: 4. Delete contents of the *analysis* folder in the Python Extension folder 5. Copy *.dll, *.pdb, *.json fron *Python/BuildOutput/VsCode/raw* to *analysis* 6. In VS Code set setting *python.downloadCodeAnalysis* to *false* +7. In VS Code set setting *python.jediEnabled* to *false* ### Debugging code in Python Extension to VS Code Folow regular TypeScript debugging steps diff --git a/src/client/activation/analysis.ts b/src/client/activation/analysis.ts index 57494a338a5d..536530d74050 100644 --- a/src/client/activation/analysis.ts +++ b/src/client/activation/analysis.ts @@ -3,15 +3,14 @@ import { inject, injectable } from 'inversify'; import * as path from 'path'; -import { ExtensionContext, OutputChannel } from 'vscode'; +import { OutputChannel, Uri } from 'vscode'; import { Disposable, LanguageClient, LanguageClientOptions, ServerOptions } from 'vscode-languageclient'; -import { IApplicationShell, ICommandManager } from '../common/application/types'; +import { IApplicationShell, ICommandManager, IWorkspaceService } from '../common/application/types'; import { isTestExecution, STANDARD_OUTPUT_CHANNEL } from '../common/constants'; import { createDeferred, Deferred } from '../common/helpers'; import { IFileSystem, IPlatformService } from '../common/platform/types'; import { StopWatch } from '../common/stopWatch'; import { IConfigurationService, IExtensionContext, IOutputChannel } from '../common/types'; -import { IEnvironmentVariablesProvider } from '../common/variables/types'; import { IInterpreterService } from '../interpreter/contracts'; import { IServiceContainer } from '../ioc/types'; import { @@ -21,7 +20,7 @@ import { } from '../telemetry/constants'; import { getTelemetryReporter } from '../telemetry/telemetry'; import { AnalysisEngineDownloader } from './downloader'; -import { InterpreterDataService } from './interpreterDataService'; +import { InterpreterData, InterpreterDataService } from './interpreterDataService'; import { PlatformData } from './platformData'; import { IExtensionActivator } from './types'; @@ -42,10 +41,13 @@ export class AnalysisExtensionActivator implements IExtensionActivator { private readonly interpreterService: IInterpreterService; private readonly startupCompleted: Deferred; private readonly disposables: Disposable[] = []; + private readonly context: IExtensionContext; + private readonly workspace: IWorkspaceService; + private readonly root: Uri | undefined; private languageClient: LanguageClient | undefined; - private readonly context: ExtensionContext; private interpreterHash: string = ''; + private loadExtensionArgs: {} | undefined; constructor(@inject(IServiceContainer) private readonly services: IServiceContainer) { this.context = this.services.get(IExtensionContext); @@ -55,14 +57,22 @@ export class AnalysisExtensionActivator implements IExtensionActivator { this.fs = this.services.get(IFileSystem); this.platformData = new PlatformData(services.get(IPlatformService), this.fs); this.interpreterService = this.services.get(IInterpreterService); + this.workspace = this.services.get(IWorkspaceService); + + // Currently only a single root. Multi-root support is future. + this.root = this.workspace && this.workspace.hasWorkspaceFolders + ? this.workspace.workspaceFolders![0]!.uri : undefined; this.startupCompleted = createDeferred(); const commandManager = this.services.get(ICommandManager); + this.disposables.push(commandManager.registerCommand(loadExtensionCommand, async (args) => { if (this.languageClient) { await this.startupCompleted.promise; this.languageClient.sendRequest('python/loadExtension', args); + } else { + this.loadExtensionArgs = args; } } )); @@ -70,17 +80,18 @@ export class AnalysisExtensionActivator implements IExtensionActivator { public async activate(): Promise { this.sw.reset(); - const clientOptions = await this.getAnalysisOptions(this.context); + const clientOptions = await this.getAnalysisOptions(); if (!clientOptions) { return false; } this.disposables.push(this.interpreterService.onDidChangeInterpreter(() => this.restartLanguageServer())); - return this.startLanguageServer(this.context, clientOptions); + return this.startLanguageServer(clientOptions); } public async deactivate(): Promise { if (this.languageClient) { - await this.languageClient.stop(); + // Do not await on this + this.languageClient.stop(); } for (const d of this.disposables) { d.dispose(); @@ -100,7 +111,7 @@ export class AnalysisExtensionActivator implements IExtensionActivator { } } - private async startLanguageServer(context: ExtensionContext, clientOptions: LanguageClientOptions): Promise { + private async startLanguageServer(clientOptions: LanguageClientOptions): Promise { // Determine if we are running MSIL/Universal via dotnet or self-contained app. const reporter = getTelemetryReporter(); @@ -109,22 +120,22 @@ export class AnalysisExtensionActivator implements IExtensionActivator { const settings = this.configuration.getSettings(); if (!settings.downloadCodeAnalysis) { // Depends on .NET Runtime or SDK. Typically development-only case. - this.languageClient = this.createSimpleLanguageClient(context, clientOptions); - await this.startLanguageClient(context); + this.languageClient = this.createSimpleLanguageClient(clientOptions); + await this.startLanguageClient(); return true; } - const mscorlib = path.join(context.extensionPath, analysisEngineFolder, 'mscorlib.dll'); + const mscorlib = path.join(this.context.extensionPath, analysisEngineFolder, 'mscorlib.dll'); if (!await this.fs.fileExists(mscorlib)) { const downloader = new AnalysisEngineDownloader(this.services, analysisEngineFolder); - await downloader.downloadAnalysisEngine(context); + await downloader.downloadAnalysisEngine(this.context); reporter.sendTelemetryEvent(PYTHON_ANALYSIS_ENGINE_DOWNLOADED); } - const serverModule = path.join(context.extensionPath, analysisEngineFolder, this.platformData.getEngineExecutableName()); - this.languageClient = this.createSelfContainedLanguageClient(context, serverModule, clientOptions); + const serverModule = path.join(this.context.extensionPath, analysisEngineFolder, this.platformData.getEngineExecutableName()); + this.languageClient = this.createSelfContainedLanguageClient(serverModule, clientOptions); try { - await this.startLanguageClient(context); + await this.startLanguageClient(); return true; } catch (ex) { this.appShell.showErrorMessage(`Language server failed to start. Error ${ex}`); @@ -133,22 +144,26 @@ export class AnalysisExtensionActivator implements IExtensionActivator { } } - private async startLanguageClient(context: ExtensionContext): Promise { + private async startLanguageClient(): Promise { this.languageClient!.onReady() .then(() => { this.startupCompleted.resolve(); + if (this.loadExtensionArgs) { + this.languageClient!.sendRequest('python/loadExtension', this.loadExtensionArgs); + this.loadExtensionArgs = undefined; + } }) .catch(error => this.startupCompleted.reject(error)); - context.subscriptions.push(this.languageClient!.start()); + this.context.subscriptions.push(this.languageClient!.start()); if (isTestExecution()) { await this.startupCompleted.promise; } } - private createSimpleLanguageClient(context: ExtensionContext, clientOptions: LanguageClientOptions): LanguageClient { + private createSimpleLanguageClient(clientOptions: LanguageClientOptions): LanguageClient { const commandOptions = { stdio: 'pipe' }; - const serverModule = path.join(context.extensionPath, analysisEngineFolder, this.platformData.getEngineDllName()); + const serverModule = path.join(this.context.extensionPath, analysisEngineFolder, this.platformData.getEngineDllName()); const serverOptions: ServerOptions = { run: { command: dotNetCommand, args: [serverModule], options: commandOptions }, debug: { command: dotNetCommand, args: [serverModule, '--debug'], options: commandOptions } @@ -156,7 +171,7 @@ export class AnalysisExtensionActivator implements IExtensionActivator { return new LanguageClient(PYTHON, languageClientName, serverOptions, clientOptions); } - private createSelfContainedLanguageClient(context: ExtensionContext, serverModule: string, clientOptions: LanguageClientOptions): LanguageClient { + private createSelfContainedLanguageClient(serverModule: string, clientOptions: LanguageClientOptions): LanguageClient { const options = { stdio: 'pipe' }; const serverOptions: ServerOptions = { run: { command: serverModule, rgs: [], options: options }, @@ -165,19 +180,22 @@ export class AnalysisExtensionActivator implements IExtensionActivator { return new LanguageClient(PYTHON, languageClientName, serverOptions, clientOptions); } - private async getAnalysisOptions(context: ExtensionContext): Promise { + private async getAnalysisOptions(): Promise { // tslint:disable-next-line:no-any const properties = new Map(); + let interpreterData: InterpreterData | undefined; + let pythonPath = ''; - // Microsoft Python code analysis engine needs full path to the interpreter - const interpreterDataService = new InterpreterDataService(context, this.services); - const interpreterData = await interpreterDataService.getInterpreterData(); - if (!interpreterData) { - const appShell = this.services.get(IApplicationShell); - appShell.showWarningMessage('Unable to determine path to Python interpreter. IntelliSense will be limited.'); + try { + const interpreterDataService = new InterpreterDataService(this.context, this.services); + interpreterData = await interpreterDataService.getInterpreterData(); + } catch (ex) { + this.appShell.showErrorMessage('Unable to determine path to the Python interpreter. IntelliSense will be limited.'); } + this.interpreterHash = interpreterData ? interpreterData.hash : ''; if (interpreterData) { + pythonPath = path.dirname(interpreterData.path); // tslint:disable-next-line:no-string-literal properties['InterpreterPath'] = interpreterData.path; // tslint:disable-next-line:no-string-literal @@ -196,25 +214,17 @@ export class AnalysisExtensionActivator implements IExtensionActivator { } // tslint:disable-next-line:no-string-literal - properties['DatabasePath'] = path.join(context.extensionPath, analysisEngineFolder); - - const envProvider = this.services.get(IEnvironmentVariablesProvider); - let pythonPath = (await envProvider.getEnvironmentVariables()).PYTHONPATH; - this.interpreterHash = interpreterData ? interpreterData.hash : ''; + properties['DatabasePath'] = path.join(this.context.extensionPath, analysisEngineFolder); // Make sure paths do not contain multiple slashes so file URIs // in VS Code (Node.js) and in the language server (.NET) match. // Note: for the language server paths separator is always ; searchPaths = searchPaths.split(path.delimiter).map(p => path.normalize(p)).join(';'); - pythonPath = pythonPath ? path.normalize(pythonPath) : ''; - // tslint:disable-next-line:no-string-literal properties['SearchPaths'] = `${searchPaths};${pythonPath}`; - const selector: string[] = [PYTHON]; - // const searchExcludes = workspace.getConfiguration('search').get('exclude', null); - // const filesExcludes = workspace.getConfiguration('files').get('exclude', null); - // const watcherExcludes = workspace.getConfiguration('files').get('watcherExclude', null); + const selector = [{ language: PYTHON, scheme: 'file' }]; + const excludeFiles = this.getExcludedFiles(); // Options to control the language client return { @@ -236,8 +246,38 @@ export class AnalysisExtensionActivator implements IExtensionActivator { maxDocumentationTextLength: 0 }, asyncStartup: true, + excludeFiles: excludeFiles, testEnvironment: isTestExecution() } }; } + + private getExcludedFiles(): string[] { + const list: string[] = ['**/Lib/**', '**/site-packages/**']; + this.getVsCodeExcludeSection('search.exclude', list); + this.getVsCodeExcludeSection('files.exclude', list); + this.getVsCodeExcludeSection('files.watcherExclude', list); + this.getPythonExcludeSection('linting.ignorePatterns', list); + this.getPythonExcludeSection('workspaceSymbols.exclusionPattern', list); + return list; + } + + private getVsCodeExcludeSection(setting: string, list: string[]): void { + const states = this.workspace.getConfiguration(setting, this.root); + if (states) { + Object.keys(states) + .filter(k => (k.indexOf('*') >= 0 || k.indexOf('/') >= 0) && states[k]) + .forEach(p => list.push(p)); + } + } + + private getPythonExcludeSection(setting: string, list: string[]): void { + const pythonSettings = this.configuration.getSettings(this.root); + const paths = pythonSettings && pythonSettings.linting ? pythonSettings.linting.ignorePatterns : undefined; + if (paths && Array.isArray(paths)) { + paths + .filter(p => p && p.length > 0) + .forEach(p => list.push(p)); + } + } } diff --git a/src/client/activation/downloader.ts b/src/client/activation/downloader.ts index 3fee93e418f5..62e063c83da8 100644 --- a/src/client/activation/downloader.ts +++ b/src/client/activation/downloader.ts @@ -5,11 +5,11 @@ import * as fileSystem from 'fs'; import * as path from 'path'; import * as request from 'request'; import * as requestProgress from 'request-progress'; -import { ExtensionContext, OutputChannel, ProgressLocation, window } from 'vscode'; +import { OutputChannel, ProgressLocation, window } from 'vscode'; import { STANDARD_OUTPUT_CHANNEL } from '../common/constants'; import { createDeferred, createTemporaryFile } from '../common/helpers'; import { IFileSystem, IPlatformService } from '../common/platform/types'; -import { IOutputChannel } from '../common/types'; +import { IExtensionContext, IOutputChannel } from '../common/types'; import { IServiceContainer } from '../ioc/types'; import { HashVerifier } from './hashVerifier'; import { PlatformData } from './platformData'; @@ -35,7 +35,7 @@ export class AnalysisEngineDownloader { this.platformData = new PlatformData(this.platform, this.fs); } - public async downloadAnalysisEngine(context: ExtensionContext): Promise { + public async downloadAnalysisEngine(context: IExtensionContext): Promise { const platformString = await this.platformData.getPlatformName(); const enginePackageFileName = `${downloadBaseFileName}-${platformString}.${downloadVersion}${downloadFileExtension}`; diff --git a/src/test/activation/excludeFiles.ptvs.test.ts b/src/test/activation/excludeFiles.ptvs.test.ts new file mode 100644 index 000000000000..88c844866aa6 --- /dev/null +++ b/src/test/activation/excludeFiles.ptvs.test.ts @@ -0,0 +1,95 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +import * as assert from 'assert'; +import { Container } from 'inversify'; +import * as path from 'path'; +import { commands, ConfigurationTarget, languages, Position, TextDocument, window, workspace } from 'vscode'; +import { ConfigurationService } from '../../client/common/configuration/service'; +import '../../client/common/extensions'; +import { IConfigurationService } from '../../client/common/types'; +import { activated } from '../../client/extension'; +import { ServiceContainer } from '../../client/ioc/container'; +import { ServiceManager } from '../../client/ioc/serviceManager'; +import { IServiceContainer, IServiceManager } from '../../client/ioc/types'; +import { IsAnalysisEngineTest } from '../constants'; +import { closeActiveWindows } from '../initialize'; + +const wksPath = path.join(__dirname, '..', '..', '..', 'src', 'test', 'pythonFiles', 'exclusions'); +const fileOne = path.join(wksPath, 'one.py'); + +// tslint:disable-next-line:max-func-body-length +suite('Exclude files (Analysis Engine)', () => { + let textDocument: TextDocument; + let serviceManager: IServiceManager; + let serviceContainer: IServiceContainer; + let configService: IConfigurationService; + + suiteSetup(async function () { + if (!IsAnalysisEngineTest()) { + // tslint:disable-next-line:no-invalid-this + this.skip(); + } + }); + setup(async () => { + const cont = new Container(); + serviceContainer = new ServiceContainer(cont); + serviceManager = new ServiceManager(cont); + + serviceManager.addSingleton(IConfigurationService, ConfigurationService); + configService = serviceManager.get(IConfigurationService); + }); + suiteTeardown(closeActiveWindows); + teardown(closeActiveWindows); + + async function openFile(file: string): Promise { + textDocument = await workspace.openTextDocument(file); + await activated; + await window.showTextDocument(textDocument); + // Make sure LS completes file loading and analysis. + // In test mode it awaits for the completion before trying + // to fetch data for completion, hover.etc. + await commands.executeCommand('vscode.executeCompletionItemProvider', textDocument.uri, new Position(0, 0)); + } + + async function setSetting(name: string, value: {} | undefined): Promise { + await configService.updateSettingAsync(name, value, undefined, ConfigurationTarget.Global); + } + + test('Default exclusions', async () => { + await openFile(fileOne); + const diag = languages.getDiagnostics(); + + const main = diag.filter(d => d[0].fsPath.indexOf('one.py') >= 0); + assert.equal(main.length > 0, true); + + const subdir = diag.filter(d => d[0].fsPath.indexOf('three.py') >= 0); + assert.equal(subdir.length > 0, true); + + const node_modules = diag.filter(d => d[0].fsPath.indexOf('node.py') >= 0); + assert.equal(node_modules.length, 0); + + const lib = diag.filter(d => d[0].fsPath.indexOf('fileLib.py') >= 0); + assert.equal(lib.length, 0); + + const sitePackages = diag.filter(d => d[0].fsPath.indexOf('sitePackages.py') >= 0); + assert.equal(sitePackages.length, 0); + }); + test('Exclude subfolder', async () => { + await setSetting('linting.ignorePatterns', ['**/dir1/**']); + + await openFile(fileOne); + const diag = languages.getDiagnostics(); + + const main = diag.filter(d => d[0].fsPath.indexOf('one.py') >= 0); + assert.equal(main.length > 0, true); + + const subdir1 = diag.filter(d => d[0].fsPath.indexOf('dir1file.py') >= 0); + assert.equal(subdir1.length, 0); + + const subdir2 = diag.filter(d => d[0].fsPath.indexOf('dir2file.py') >= 0); + assert.equal(subdir2.length, 0); + + await setSetting('linting.ignorePatterns', undefined); + }); +}); diff --git a/src/test/pythonFiles/exclusions/Lib/fileLib.py b/src/test/pythonFiles/exclusions/Lib/fileLib.py new file mode 100644 index 000000000000..50000adeda40 --- /dev/null +++ b/src/test/pythonFiles/exclusions/Lib/fileLib.py @@ -0,0 +1 @@ + a \ No newline at end of file diff --git a/src/test/pythonFiles/exclusions/Lib/site-packages/sitePackages.py b/src/test/pythonFiles/exclusions/Lib/site-packages/sitePackages.py new file mode 100644 index 000000000000..dad1af98c7f5 --- /dev/null +++ b/src/test/pythonFiles/exclusions/Lib/site-packages/sitePackages.py @@ -0,0 +1 @@ + b \ No newline at end of file diff --git a/src/test/pythonFiles/exclusions/dir1/dir1file.py b/src/test/pythonFiles/exclusions/dir1/dir1file.py new file mode 100644 index 000000000000..fe453b3fcc6a --- /dev/null +++ b/src/test/pythonFiles/exclusions/dir1/dir1file.py @@ -0,0 +1 @@ + for \ No newline at end of file diff --git a/src/test/pythonFiles/exclusions/dir1/dir2/dir2file.py b/src/test/pythonFiles/exclusions/dir1/dir2/dir2file.py new file mode 100644 index 000000000000..fe453b3fcc6a --- /dev/null +++ b/src/test/pythonFiles/exclusions/dir1/dir2/dir2file.py @@ -0,0 +1 @@ + for \ No newline at end of file diff --git a/src/test/pythonFiles/exclusions/one.py b/src/test/pythonFiles/exclusions/one.py new file mode 100644 index 000000000000..8c68a1c1fee2 --- /dev/null +++ b/src/test/pythonFiles/exclusions/one.py @@ -0,0 +1 @@ + if \ No newline at end of file