Skip to content

Commit 7d93692

Browse files
author
Mikhail Arkhipov
authored
Improve tooltip display (#725)
* Basic tokenizer * Fixed property names * Tests, round I * Tests, round II * tokenizer test * Remove temorary change * Fix merge issue * Merge conflict * Merge conflict * Completion test * Fix last line * Fix javascript math * Make test await for results * Add license headers * Rename definitions to types * License headers * Fix typo in completion details (typo) * Fix hover test * Russian translations * Update to better translation * Fix typo * #70 How to get all parameter info when filling in a function param list * Fix #70 How to get all parameter info when filling in a function param list * Clean up * Clean imports * CR feedback * Trim whitespace for test stability * More tests * Better handle no-parameters documentation * Better handle ellipsis and Python3 * #385 Auto-Indentation doesn't work after comment * #141 Auto indentation broken when return keyword involved * Undo changes * #627 Docstrings for builtin methods are not parsed correctly * reStructuredText converter * Fix: period is not an operator * Minor fixes * Restructure * Tests * Tests * Code heuristics * Baselines * HTML handling * Lists * State machine * Baselines * Squash * no message * Whitespace difference
1 parent c00f94b commit 7d93692

18 files changed

+1112
-81
lines changed

‎src/client/common/markdown/restTextConverter.ts

Lines changed: 250 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,250 @@
1+
// Copyright (c) Microsoft Corporation. All rights reserved.
2+
// Licensed under the MIT License.
3+
4+
import { EOL } from 'os';
5+
// tslint:disable-next-line:import-name
6+
import Char from 'typescript-char';
7+
import { isDecimal, isWhiteSpace } from '../../language/characters';
8+
9+
enum State {
10+
Default,
11+
Preformatted,
12+
Code
13+
}
14+
15+
export class RestTextConverter {
16+
private state: State = State.Default;
17+
private md: string[] = [];
18+
19+
// tslint:disable-next-line:cyclomatic-complexity
20+
public toMarkdown(docstring: string): string {
21+
// Translates reStructruredText (Python doc syntax) to markdown.
22+
// It only translates as much as needed to display tooltips
23+
// and documentation in the completion list.
24+
// See https://en.wikipedia.org/wiki/ReStructuredText
25+
26+
const result = this.transformLines(docstring);
27+
this.state = State.Default;
28+
this.md = [];
29+
30+
return result;
31+
}
32+
33+
public escapeMarkdown(text: string): string {
34+
// Not complete escape list so it does not interfere
35+
// with subsequent code highlighting (see above).
36+
return text
37+
.replace(/\#/g, '\\#')
38+
.replace(/\*/g, '\\*')
39+
.replace(/\_/g, '\\_');
40+
}
41+
42+
private transformLines(docstring: string): string {
43+
const lines = docstring.split(/\r?\n/);
44+
for (let i = 0; i < lines.length; i += 1) {
45+
const line = lines[i];
46+
// Avoid leading empty lines
47+
if (this.md.length === 0 && line.length === 0) {
48+
continue;
49+
}
50+
51+
switch (this.state) {
52+
case State.Default:
53+
i += this.inDefaultState(lines, i);
54+
break;
55+
case State.Preformatted:
56+
i += this.inPreformattedState(lines, i);
57+
break;
58+
case State.Code:
59+
this.inCodeState(line);
60+
break;
61+
default:
62+
break;
63+
}
64+
}
65+
66+
this.endCodeBlock();
67+
this.endPreformattedBlock();
68+
69+
return this.md.join(EOL).trim();
70+
}
71+
72+
private inDefaultState(lines: string[], i: number): number {
73+
let line = lines[i];
74+
if (line.startsWith('```')) {
75+
this.startCodeBlock();
76+
return 0;
77+
}
78+
79+
if (line.startsWith('===') || line.startsWith('---')) {
80+
return 0; // Eat standalone === or --- lines.
81+
}
82+
if (this.handleDoubleColon(line)) {
83+
return 0;
84+
}
85+
if (this.isIgnorable(line)) {
86+
return 0;
87+
}
88+
89+
if (this.handleSectionHeader(lines, i)) {
90+
return 1; // Eat line with === or ---
91+
}
92+
93+
const result = this.checkPreContent(lines, i);
94+
if (this.state !== State.Default) {
95+
return result; // Handle line in the new state
96+
}
97+
98+
line = this.cleanup(line);
99+
line = line.replace(/``/g, '`'); // Convert double backticks to single.
100+
line = this.escapeMarkdown(line);
101+
this.md.push(line);
102+
103+
return 0;
104+
}
105+
106+
private inPreformattedState(lines: string[], i: number): number {
107+
let line = lines[i];
108+
if (this.isIgnorable(line)) {
109+
return 0;
110+
}
111+
// Preformatted block terminates by a line without leading whitespace.
112+
if (line.length > 0 && !isWhiteSpace(line.charCodeAt(0)) && !this.isListItem(line)) {
113+
this.endPreformattedBlock();
114+
return -1;
115+
}
116+
117+
const prevLine = this.md.length > 0 ? this.md[this.md.length - 1] : undefined;
118+
if (line.length === 0 && prevLine && (prevLine.length === 0 || prevLine.startsWith('```'))) {
119+
return 0; // Avoid more than one empty line in a row.
120+
}
121+
122+
// Since we use HTML blocks as preformatted text
123+
// make sure we drop angle brackets since otherwise
124+
// they will render as tags and attributes
125+
line = line.replace(/</g, ' ').replace(/>/g, ' ');
126+
line = line.replace(/``/g, '`'); // Convert double backticks to single.
127+
// Keep hard line breaks for the preformatted content
128+
this.md.push(`${line} `);
129+
return 0;
130+
}
131+
132+
private inCodeState(line: string): void {
133+
const prevLine = this.md.length > 0 ? this.md[this.md.length - 1] : undefined;
134+
if (line.length === 0 && prevLine && (prevLine.length === 0 || prevLine.startsWith('```'))) {
135+
return; // Avoid more than one empty line in a row.
136+
}
137+
138+
if (line.startsWith('```')) {
139+
this.endCodeBlock();
140+
} else {
141+
this.md.push(line);
142+
}
143+
}
144+
145+
private isIgnorable(line: string): boolean {
146+
if (line.indexOf('generated/') >= 0) {
147+
return true; // Drop generated content.
148+
}
149+
const trimmed = line.trim();
150+
if (trimmed.startsWith('..') && trimmed.indexOf('::') > 0) {
151+
// Ignore lines likes .. sectionauthor:: John Doe.
152+
return true;
153+
}
154+
return false;
155+
}
156+
157+
private checkPreContent(lines: string[], i: number): number {
158+
const line = lines[i];
159+
if (i === 0 || line.trim().length === 0) {
160+
return 0;
161+
}
162+
163+
if (!isWhiteSpace(line.charCodeAt(0)) && !this.isListItem(line)) {
164+
return 0; // regular line, nothing to do here.
165+
}
166+
// Indented content is considered to be preformatted.
167+
this.startPreformattedBlock();
168+
return -1;
169+
}
170+
171+
private handleSectionHeader(lines: string[], i: number): boolean {
172+
const line = lines[i];
173+
if (i < lines.length - 1 && (lines[i + 1].startsWith('==='))) {
174+
// Section title -> heading level 3.
175+
this.md.push(`### ${this.cleanup(line)}`);
176+
return true;
177+
}
178+
if (i < lines.length - 1 && (lines[i + 1].startsWith('---'))) {
179+
// Subsection title -> heading level 4.
180+
this.md.push(`#### ${this.cleanup(line)}`);
181+
return true;
182+
}
183+
return false;
184+
}
185+
186+
private handleDoubleColon(line: string): boolean {
187+
if (!line.endsWith('::')) {
188+
return false;
189+
}
190+
// Literal blocks begin with `::`. Such as sequence like
191+
// '... as shown below::' that is followed by a preformatted text.
192+
if (line.length > 2 && !line.startsWith('..')) {
193+
// Ignore lines likes .. autosummary:: John Doe.
194+
// Trim trailing : so :: turns into :.
195+
this.md.push(line.substring(0, line.length - 1));
196+
}
197+
198+
this.startPreformattedBlock();
199+
return true;
200+
}
201+
202+
private startPreformattedBlock(): void {
203+
// Remove previous empty line so we avoid double empties.
204+
this.tryRemovePrecedingEmptyLines();
205+
// Lie about the language since we don't want preformatted text
206+
// to be colorized as Python. HTML is more 'appropriate' as it does
207+
// not colorize -- or + or keywords like 'from'.
208+
this.md.push('```html');
209+
this.state = State.Preformatted;
210+
}
211+
212+
private endPreformattedBlock(): void {
213+
if (this.state === State.Preformatted) {
214+
this.tryRemovePrecedingEmptyLines();
215+
this.md.push('```');
216+
this.state = State.Default;
217+
}
218+
}
219+
220+
private startCodeBlock(): void {
221+
// Remove previous empty line so we avoid double empties.
222+
this.tryRemovePrecedingEmptyLines();
223+
this.md.push('```python');
224+
this.state = State.Code;
225+
}
226+
227+
private endCodeBlock(): void {
228+
if (this.state === State.Code) {
229+
this.tryRemovePrecedingEmptyLines();
230+
this.md.push('```');
231+
this.state = State.Default;
232+
}
233+
}
234+
235+
private tryRemovePrecedingEmptyLines(): void {
236+
while (this.md.length > 0 && this.md[this.md.length - 1].trim().length === 0) {
237+
this.md.pop();
238+
}
239+
}
240+
241+
private isListItem(line: string): boolean {
242+
const trimmed = line.trim();
243+
const ch = trimmed.length > 0 ? trimmed.charCodeAt(0) : 0;
244+
return ch === Char.Asterisk || ch === Char.Hyphen || isDecimal(ch);
245+
}
246+
247+
private cleanup(line: string): string {
248+
return line.replace(/:mod:/g, 'module:');
249+
}
250+
}

‎src/client/language/tokenizer.ts

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -300,7 +300,7 @@ export class Tokenizer implements ITokenizer {
300300
break;
301301

302302
default:
303-
break;
303+
return false;
304304
}
305305
this.tokens.push(new Token(TokenType.Operator, this.cs.position, length));
306306
this.cs.advance(length);

0 commit comments

Comments
 (0)