Skip to content

Commit 287fd8b

Browse files
author
Mikhail Arkhipov
authored
Add missing operators in tokenizer, handle end-of-comment completion better (#1129)
* Basic tokenizer * Fixed property names * Tests, round I * Tests, round II * tokenizer test * Remove temorary change * Fix merge issue * Merge conflict * Merge conflict * Completion test * Fix last line * Fix javascript math * Make test await for results * Add license headers * Rename definitions to types * License headers * Fix typo in completion details (typo) * Fix hover test * Russian translations * Update to better translation * Fix typo * #70 How to get all parameter info when filling in a function param list * Fix #70 How to get all parameter info when filling in a function param list * Clean up * Clean imports * CR feedback * Trim whitespace for test stability * More tests * Better handle no-parameters documentation * Better handle ellipsis and Python3 * #385 Auto-Indentation doesn't work after comment * #141 Auto indentation broken when return keyword involved * Undo changes * #627 Docstrings for builtin methods are not parsed correctly * reStructuredText converter * Fix: period is not an operator * Minor fixes * Restructure * Tests * Tests * Code heuristics * Baselines * HTML handling * Lists * State machine * Baselines * Squash * no message * Whitespace difference * Update Jedi to 0.11.1 * Enable Travis * Test fixes * Undo change * Jedi 0.11 with parser * Undo changes * Undo changes * Test fixes * More tests * Tests * Fix pylint search * Handle quote escapes in strings * Escapes in strings * CR feedback * Discover pylintrc better + tests * Fix .pyenv/versions search * Fix multiple linters output * Better handle markdown underscore * Test * Fix 916: PyLint checks wrong files * Test stability * Try increase timeout * Make sure linting is enabled in tests * Try another way of waiting * Simplify * Fix clear diags on close tests * Try writing settings directly * Increase timeout * Measure test time * Measure time * Simplify * Set timeout * Better venv detection * Add test * More reliable check * Fix pylint switch key * Remove incorrect flag * Disable print * Require pylint 1.8 on CI * Fix working directory for standalone files * Use an 'elif' * Separate file for pylint root config * Remove double event listening * Remove test * Revert "Remove test" This reverts commit e240c3f. * Revert "Remove double event listening" This reverts commit af573be. * Explicitly disable linter * New buttons * PR feedback * Revert "PR feedback" This reverts commit d0c7ab5. * PR feedback * Fix end of comment completion and operator handling * Simplify * Test typo fix
1 parent ab5faf6 commit 287fd8b

File tree

8 files changed

+51
-37
lines changed

8 files changed

+51
-37
lines changed

‎.gitignore

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,3 +15,4 @@ coverage/
1515
.venv
1616
pythonFiles/experimental/ptvsd/**
1717
debug_coverage*/**
18+
analysis/**

‎src/client/extension.ts

Lines changed: 26 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -6,10 +6,12 @@ if ((Reflect as any).metadata === undefined) {
66
require('reflect-metadata');
77
}
88
import { Container } from 'inversify';
9-
import * as vscode from 'vscode';
10-
import { Disposable, Memento, OutputChannel, window } from 'vscode';
9+
import {
10+
debug, Disposable, DocumentFilter, ExtensionContext,
11+
extensions, IndentAction, languages, Memento,
12+
OutputChannel, window
13+
} from 'vscode';
1114
import { PythonSettings } from './common/configSettings';
12-
import * as settings from './common/configSettings';
1315
import { STANDARD_OUTPUT_CHANNEL } from './common/constants';
1416
import { FeatureDeprecationManager } from './common/featureDeprecationManager';
1517
import { createDeferred } from './common/helpers';
@@ -61,12 +63,12 @@ import * as tests from './unittests/main';
6163
import { registerTypes as unitTestsRegisterTypes } from './unittests/serviceRegistry';
6264
import { WorkspaceSymbols } from './workspaceSymbols/main';
6365

64-
const PYTHON: vscode.DocumentFilter = { language: 'python' };
66+
const PYTHON: DocumentFilter = { language: 'python' };
6567
const activationDeferred = createDeferred<void>();
6668
export const activated = activationDeferred.promise;
6769

6870
// tslint:disable-next-line:max-func-body-length
69-
export async function activate(context: vscode.ExtensionContext) {
71+
export async function activate(context: ExtensionContext) {
7072
const cont = new Container();
7173
const serviceManager = new ServiceManager(cont);
7274
const serviceContainer = new ServiceContainer(cont);
@@ -95,7 +97,7 @@ export async function activate(context: vscode.ExtensionContext) {
9597
serviceManager.get<ICodeExecutionManager>(ICodeExecutionManager).registerCommands();
9698

9799
const persistentStateFactory = serviceManager.get<IPersistentStateFactory>(IPersistentStateFactory);
98-
const pythonSettings = settings.PythonSettings.getInstance();
100+
const pythonSettings = PythonSettings.getInstance();
99101
// tslint:disable-next-line:no-floating-promises
100102
sendStartupTelemetry(activated, serviceContainer);
101103

@@ -125,60 +127,60 @@ export async function activate(context: vscode.ExtensionContext) {
125127

126128
// Enable indentAction
127129
// tslint:disable-next-line:no-non-null-assertion
128-
vscode.languages.setLanguageConfiguration(PYTHON.language!, {
130+
languages.setLanguageConfiguration(PYTHON.language!, {
129131
onEnterRules: [
130132
{
131133
beforeText: /^\s*(?:def|class|for|if|elif|else|while|try|with|finally|except|async)\b.*/,
132-
action: { indentAction: vscode.IndentAction.Indent }
134+
action: { indentAction: IndentAction.Indent }
133135
},
134136
{
135137
beforeText: /^\s*#.*/,
136138
afterText: /.+$/,
137-
action: { indentAction: vscode.IndentAction.None, appendText: '# ' }
139+
action: { indentAction: IndentAction.None, appendText: '# ' }
138140
},
139141
{
140142
beforeText: /^\s+(continue|break|return)\b.*/,
141143
afterText: /\s+$/,
142-
action: { indentAction: vscode.IndentAction.Outdent }
144+
action: { indentAction: IndentAction.Outdent }
143145
}
144146
]
145147
});
146148

147149
context.subscriptions.push(jediFactory);
148-
context.subscriptions.push(vscode.languages.registerRenameProvider(PYTHON, new PythonRenameProvider(serviceContainer)));
150+
context.subscriptions.push(languages.registerRenameProvider(PYTHON, new PythonRenameProvider(serviceContainer)));
149151
const definitionProvider = new PythonDefinitionProvider(jediFactory);
150-
context.subscriptions.push(vscode.languages.registerDefinitionProvider(PYTHON, definitionProvider));
151-
context.subscriptions.push(vscode.languages.registerHoverProvider(PYTHON, new PythonHoverProvider(jediFactory)));
152-
context.subscriptions.push(vscode.languages.registerReferenceProvider(PYTHON, new PythonReferenceProvider(jediFactory)));
153-
context.subscriptions.push(vscode.languages.registerCompletionItemProvider(PYTHON, new PythonCompletionItemProvider(jediFactory, serviceContainer), '.'));
154-
context.subscriptions.push(vscode.languages.registerCodeLensProvider(PYTHON, serviceContainer.get<IShebangCodeLensProvider>(IShebangCodeLensProvider)));
152+
context.subscriptions.push(languages.registerDefinitionProvider(PYTHON, definitionProvider));
153+
context.subscriptions.push(languages.registerHoverProvider(PYTHON, new PythonHoverProvider(jediFactory)));
154+
context.subscriptions.push(languages.registerReferenceProvider(PYTHON, new PythonReferenceProvider(jediFactory)));
155+
context.subscriptions.push(languages.registerCompletionItemProvider(PYTHON, new PythonCompletionItemProvider(jediFactory, serviceContainer), '.'));
156+
context.subscriptions.push(languages.registerCodeLensProvider(PYTHON, serviceContainer.get<IShebangCodeLensProvider>(IShebangCodeLensProvider)));
155157

156158
const symbolProvider = new PythonSymbolProvider(jediFactory);
157-
context.subscriptions.push(vscode.languages.registerDocumentSymbolProvider(PYTHON, symbolProvider));
159+
context.subscriptions.push(languages.registerDocumentSymbolProvider(PYTHON, symbolProvider));
158160
if (pythonSettings.devOptions.indexOf('DISABLE_SIGNATURE') === -1) {
159-
context.subscriptions.push(vscode.languages.registerSignatureHelpProvider(PYTHON, new PythonSignatureProvider(jediFactory), '(', ','));
161+
context.subscriptions.push(languages.registerSignatureHelpProvider(PYTHON, new PythonSignatureProvider(jediFactory), '(', ','));
160162
}
161163
if (pythonSettings.formatting.provider !== 'none') {
162164
const formatProvider = new PythonFormattingEditProvider(context, serviceContainer);
163-
context.subscriptions.push(vscode.languages.registerDocumentFormattingEditProvider(PYTHON, formatProvider));
164-
context.subscriptions.push(vscode.languages.registerDocumentRangeFormattingEditProvider(PYTHON, formatProvider));
165+
context.subscriptions.push(languages.registerDocumentFormattingEditProvider(PYTHON, formatProvider));
166+
context.subscriptions.push(languages.registerDocumentRangeFormattingEditProvider(PYTHON, formatProvider));
165167
}
166168

167169
const linterProvider = new LinterProvider(context, serviceContainer);
168170
context.subscriptions.push(linterProvider);
169171

170-
const jupyterExtension = vscode.extensions.getExtension('donjayamanne.jupyter');
172+
const jupyterExtension = extensions.getExtension('donjayamanne.jupyter');
171173
const lintingEngine = serviceContainer.get<ILintingEngine>(ILintingEngine);
172174
lintingEngine.linkJupiterExtension(jupyterExtension).ignoreErrors();
173175

174176
tests.activate(context, unitTestOutChannel, symbolProvider, serviceContainer);
175177

176178
context.subscriptions.push(new WorkspaceSymbols(serviceContainer));
177-
context.subscriptions.push(vscode.languages.registerOnTypeFormattingEditProvider(PYTHON, new BlockFormatProviders(), ':'));
178-
context.subscriptions.push(vscode.languages.registerOnTypeFormattingEditProvider(PYTHON, new OnEnterFormatter(), '\n'));
179+
context.subscriptions.push(languages.registerOnTypeFormattingEditProvider(PYTHON, new BlockFormatProviders(), ':'));
180+
context.subscriptions.push(languages.registerOnTypeFormattingEditProvider(PYTHON, new OnEnterFormatter(), '\n'));
179181

180182
serviceContainer.getAll<BaseConfigurationProvider>(IDebugConfigurationProvider).forEach(debugConfig => {
181-
context.subscriptions.push(vscode.debug.registerDebugConfigurationProvider(debugConfig.debugType, debugConfig));
183+
context.subscriptions.push(debug.registerDebugConfigurationProvider(debugConfig.debugType, debugConfig));
182184
});
183185
activationDeferred.resolve();
184186

‎src/client/language/tokenizer.ts

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -34,10 +34,10 @@ export class Tokenizer implements ITokenizer {
3434
// 'not', 'or', 'pass', 'print', 'raise', 'return', 'True', 'try',
3535
// 'while', 'with', 'yield'
3636
// ];
37-
private cs: ICharacterStream;
37+
private cs: ICharacterStream = new CharacterStream('');
3838
private tokens: IToken[] = [];
3939
private floatRegex = /[-+]?(?:(?:\d*\.\d+)|(?:\d+\.?))(?:[Ee][+-]?\d+)?/;
40-
private mode: TokenizerMode;
40+
private mode = TokenizerMode.Full;
4141

4242
constructor() {
4343
//this.floatRegex.compile();
@@ -287,15 +287,15 @@ export class Tokenizer implements ITokenizer {
287287
} else if (nextChar === Char.Less) {
288288
length = this.cs.lookAhead(2) === Char.Equal ? 3 : 2;
289289
} else {
290-
length = 1;
290+
length = nextChar === Char.Equal ? 2 : 1;
291291
}
292292
break;
293293

294294
case Char.Greater:
295295
if (nextChar === Char.Greater) {
296296
length = this.cs.lookAhead(2) === Char.Equal ? 3 : 2;
297297
} else {
298-
length = 1;
298+
length = nextChar === Char.Equal ? 2 : 1;
299299
}
300300
break;
301301

‎src/client/providers/completionSource.ts

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -61,7 +61,7 @@ export class CompletionSource {
6161
const sourceText = `${document.getText(leadingRange)}${itemString}`;
6262
const range = new vscode.Range(leadingRange.end, leadingRange.end.translate(0, itemString.length));
6363

64-
return await this.itemInfoSource.getItemInfoFromText(document.uri, document.fileName, range, sourceText, token);
64+
return this.itemInfoSource.getItemInfoFromText(document.uri, document.fileName, range, sourceText, token);
6565
}
6666

6767
private async getCompletionResult(document: vscode.TextDocument, position: vscode.Position, token: vscode.CancellationToken)
@@ -90,7 +90,7 @@ export class CompletionSource {
9090
source: source
9191
};
9292

93-
return await this.jediFactory.getJediProxyHandler<proxy.ICompletionResult>(document.uri).sendCommand(cmd, token);
93+
return this.jediFactory.getJediProxyHandler<proxy.ICompletionResult>(document.uri).sendCommand(cmd, token);
9494
}
9595

9696
private toVsCodeCompletions(documentPosition: DocumentPosition, data: proxy.ICompletionResult, resource: vscode.Uri): vscode.CompletionItem[] {

‎src/client/providers/providerUtilities.ts

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,10 +13,19 @@ export function getDocumentTokens(document: vscode.TextDocument, tokenizeTo: vsc
1313
export function isPositionInsideStringOrComment(document: vscode.TextDocument, position: vscode.Position): boolean {
1414
const tokenizeTo = position.translate(1, 0);
1515
const tokens = getDocumentTokens(document, tokenizeTo, TokenizerMode.CommentsAndStrings);
16-
const index = tokens.getItemContaining(document.offsetAt(position));
16+
const offset = document.offsetAt(position);
17+
let index = tokens.getItemContaining(offset);
1718
if (index >= 0) {
1819
const token = tokens.getItemAt(index);
1920
return token.type === TokenType.String || token.type === TokenType.Comment;
2021
}
22+
if (offset > 0) {
23+
// In case position is at the every end of the comment or unterminated string
24+
index = tokens.getItemContaining(offset - 1);
25+
if (index >= 0) {
26+
const token = tokens.getItemAt(index);
27+
return token.end === offset && token.type === TokenType.Comment;
28+
}
29+
}
2130
return false;
2231
}

‎src/test/autocomplete/base.test.ts

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -195,10 +195,11 @@ suite('Autocomplete', () => {
195195
new vscode.Position(3, 0), // false
196196
new vscode.Position(4, 2), // false
197197
new vscode.Position(4, 8), // false
198-
new vscode.Position(5, 4) // false
198+
new vscode.Position(5, 4), // false
199+
new vscode.Position(5, 10) // false
199200
];
200201
const expected = [
201-
false, true, false, false, false, false, false, false, false, false
202+
false, true, false, false, false, false, false, false, false, false, false
202203
];
203204
const textDocument = await vscode.workspace.openTextDocument(fileSuppress);
204205
await vscode.window.showTextDocument(textDocument);

‎src/test/language/tokenizer.test.ts

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -180,15 +180,15 @@ suite('Language.Tokenizer', () => {
180180
});
181181
test('Operators', async () => {
182182
const text = '< <> << <<= ' +
183-
'== != > >> >>= ' +
183+
'== != > >> >>= >= <=' +
184184
'+ -' +
185185
'* ** / /= //=' +
186186
'*= += -= **= ' +
187187
'& &= | |= ^ ^=';
188188
const tokens = new Tokenizer().tokenize(text);
189189
const lengths = [
190190
1, 2, 2, 3,
191-
2, 2, 1, 2, 3,
191+
2, 2, 1, 2, 3, 2, 2,
192192
1, 1,
193193
1, 2, 1, 2, 3,
194194
2, 2, 2, 3,

‎tslint.json

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,7 @@
5959
],
6060
"no-unnecessary-type-assertion": false,
6161
"no-submodule-imports": false,
62-
"no-redundant-jsdoc": false
62+
"no-redundant-jsdoc": false,
63+
"binary-expression-operand-order": false
6364
}
64-
}
65+
}

0 commit comments

Comments
 (0)