Skip to content

Use a flat list for separated syntax lists. #1027

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Nov 1, 2014
Merged

Use a flat list for separated syntax lists. #1027

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion src/services/formatting/indentationTrackingWalker.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -103,7 +103,7 @@ module TypeScript.Services.Formatting {
if (isToken(element)) {
this.visitToken(<ISyntaxToken>element);
}
else if (element.kind() === SyntaxKind.List || element.kind() === SyntaxKind.SeparatedList) {
else if (element.kind() === SyntaxKind.List) {
for (var i = 0, n = element.childCount(); i < n; i++) {
this.walk(element.childAt(i));
}
Expand Down
946 changes: 383 additions & 563 deletions src/services/syntax/SyntaxGenerator.js
View file
Open in desktop

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion src/services/syntax/incrementalParser.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -777,7 +777,7 @@ module TypeScript.IncrementalParser {
function moveToFirstChildIfList(): void {
var element = pieces[currentPieceIndex].element;

if (isList(element) || isSeparatedList(element)) {
if (isList(element)) {
// We cannot ever get an empty list in our piece path. Empty lists are 'shared' and
// we make sure to filter that out before pushing any children.
// Debug.assert(childCount(element) > 0);
Expand Down
57 changes: 19 additions & 38 deletions src/services/syntax/parser.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -557,7 +557,7 @@ module TypeScript.Parser {
// when next requested.
while (true) {
// Parent must be a list or a node. All of those have a 'data' element.
Debug.assert(isNode(parent) || isList(parent) || isSeparatedList(parent));
Debug.assert(isNode(parent) || isList(parent));
var dataElement = <{ data: number }><any>parent;
if (dataElement.data) {
dataElement.data &= SyntaxConstants.NodeParsedInStrictModeMask
Expand Down Expand Up @@ -592,20 +592,6 @@ module TypeScript.Parser {
}
}
}
else if (isSeparatedList(parent)) {
var list2 = <ISyntaxNodeOrToken[]>parent;
for (var i = 0, n = list2.childCount(); i < n; i++) {
if (list2.childAt(i) === oldToken) {
if (i % 2 === 0) {
list2[i / 2] = newToken;
}
else {
list2.separators[(i - 1) / 2] = newToken;
}
return;
}
}
}

throw Errors.invalidOperation();
}
Expand Down Expand Up @@ -3786,7 +3772,7 @@ module TypeScript.Parser {
return result;
}

function parseSeparatedSyntaxList<T extends ISyntaxNodeOrToken>(currentListType: ListParsingState, skippedTokens: ISyntaxToken[]): T[] {
function parseSeparatedSyntaxList<T extends ISyntaxNodeOrToken>(currentListType: ListParsingState, skippedTokens: ISyntaxToken[]): ISeparatedSyntaxList<T> {
var savedListParsingState = listParsingState;
listParsingState |= (1 << currentListType);

Expand All @@ -3799,7 +3785,7 @@ module TypeScript.Parser {

// Returns true if we should abort parsing.
function abortParsingListOrMoveToNextToken<T extends ISyntaxNodeOrToken>(
currentListType: ListParsingState, nodes: T[], separators: ISyntaxToken[], skippedTokens: ISyntaxToken[]): boolean {
currentListType: ListParsingState, nodeAndSeparators: ISyntaxNodeOrToken[], skippedTokens: ISyntaxToken[]): boolean {
// Ok. We're at a token that is not a terminator for the list and wasn't the start of
// an item in the list. Definitely report an error for this token.
reportUnexpectedTokenDiagnostic(currentListType);
Expand All @@ -3819,30 +3805,27 @@ module TypeScript.Parser {

// Otherwise, if none of the lists we're in can capture this token, then we need to
// unilaterally skip it. Note: we've already reported an error above.
addSkippedTokenToList(nodes, separators, skippedTokens, consumeToken(currentToken()));
addSkippedTokenToList(nodeAndSeparators, skippedTokens, consumeToken(currentToken()));

// Continue parsing this list. Attach this token to whatever we've seen already.
return false;
}

function addSkippedTokenToList<T extends ISyntaxNodeOrToken>(
nodes: T[], separators: ISyntaxToken[], skippedTokens: ISyntaxToken[], skippedToken: ISyntaxToken): void {
function addSkippedTokenToList(
nodesAndSeparators: ISyntaxNodeOrToken[], skippedTokens: ISyntaxToken[], skippedToken: ISyntaxToken): void {
// Now, add this skipped token to the last item we successfully parsed in the list. Or
// add it to the list of skipped tokens if we haven't parsed anything. Our caller will
// have to deal with them.
//
// Note: we only bother doing this if we're creating a concrete syntax tree.
if (syntaxFactory.isConcrete) {
var length = nodes.length + (separators ? separators.length : 0);
var length = nodesAndSeparators.length;

for (var i = length - 1; i >= 0; i--) {
var array: ISyntaxNodeOrToken[] = separators && (i % 2 === 1) ? separators : nodes;
var arrayIndex = separators ? IntegerUtilities.integerDivide(i, 2) : i;

var item = array[arrayIndex];
var item = nodesAndSeparators[i];
var _lastToken = lastToken(item);
if (_lastToken && _lastToken.fullWidth() > 0) {
array[arrayIndex] = <T>addSkippedTokenAfterNodeOrToken(item, skippedToken);
nodesAndSeparators[i] = addSkippedTokenAfterNodeOrToken(item, skippedToken);
return;
}
}
Expand Down Expand Up @@ -3895,7 +3878,7 @@ module TypeScript.Parser {

// List wasn't complete and we didn't get an item. Figure out if we should bail out
// or skip a token and continue.
var abort = abortParsingListOrMoveToNextToken(currentListType, items, /*separators:*/ undefined, skippedTokens);
var abort = abortParsingListOrMoveToNextToken(currentListType, items, skippedTokens);
if (abort) {
break;
}
Expand All @@ -3914,9 +3897,8 @@ module TypeScript.Parser {
return result;
}

function parseSeparatedSyntaxListWorker<T extends ISyntaxNodeOrToken>(currentListType: ListParsingState, skippedTokens: ISyntaxToken[]): T[] {
var nodes: T[] = getArray();
var separators: ISyntaxToken[] = getArray();
function parseSeparatedSyntaxListWorker<T extends ISyntaxNodeOrToken>(currentListType: ListParsingState, skippedTokens: ISyntaxToken[]): ISeparatedSyntaxList<T> {
var nodesAndSeparators: ISyntaxNodeOrToken[] = getArray();

// Debug.assert(nodes.length === 0);
// Debug.assert(separators.length === 0);
Expand All @@ -3934,7 +3916,7 @@ module TypeScript.Parser {
// continue parsing.

// Debug.assert(oldItemsCount % 2 === 0);
var succeeded = tryParseExpectedListItem(currentListType, inErrorRecovery, nodes, /*processItems:*/ undefined);
var succeeded = tryParseExpectedListItem(currentListType, inErrorRecovery, nodesAndSeparators, /*processItems:*/ undefined);

if (!succeeded) {
// We weren't able to parse out a list element.
Expand All @@ -3948,7 +3930,7 @@ module TypeScript.Parser {

// List wasn't complete and we didn't get an item. Figure out if we should bail out
// or skip a token and continue.
var abort = abortParsingListOrMoveToNextToken(currentListType, nodes, separators, skippedTokens);
var abort = abortParsingListOrMoveToNextToken(currentListType, nodesAndSeparators, skippedTokens);
if (abort) {
break;
}
Expand All @@ -3973,7 +3955,7 @@ module TypeScript.Parser {
var tokenKind = _currentToken.kind();
if (tokenKind === _separatorKind || tokenKind === SyntaxKind.CommaToken) {
// Consume the last separator and continue parsing list elements.
separators.push(consumeToken(_currentToken));
nodesAndSeparators.push(consumeToken(_currentToken));
continue;
}

Expand Down Expand Up @@ -4001,7 +3983,7 @@ module TypeScript.Parser {

if (allowAutomaticSemicolonInsertion && canEatAutomaticSemicolon(/*allowWithoutNewline:*/ false)) {
var semicolonToken = eatExplicitOrAutomaticSemicolon(/*allowWithoutNewline:*/ false) || Syntax.emptyToken(SyntaxKind.SemicolonToken);
separators.push(semicolonToken);
nodesAndSeparators.push(semicolonToken);
// Debug.assert(items.length % 2 === 0);
continue;
}
Expand All @@ -4011,7 +3993,7 @@ module TypeScript.Parser {
// This time mark that we're in error recovery mode though.
//
// Note: trying to eat this token will emit the appropriate diagnostic.
separators.push(eatToken(_separatorKind));
nodesAndSeparators.push(eatToken(_separatorKind));

// Now that we're in 'error recovery' mode we cantweak some parsing rules as
// appropriate. For example, if we have:
Expand All @@ -4027,12 +4009,11 @@ module TypeScript.Parser {
inErrorRecovery = true;
}

var result = Syntax.separatedList<T>(nodes, separators);
var result = Syntax.separatedList<T>(nodesAndSeparators);

// Can't return if it has more then 0 elements. In that case, the list will have been
// copied into the SyntaxList.
returnZeroLengthArray(nodes);
returnZeroLengthArray(separators);
returnZeroLengthArray(nodesAndSeparators);

return result;
}
Expand Down
8 changes: 2 additions & 6 deletions src/services/syntax/syntaxElement.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ module TypeScript {
// nodes don't have positions or parents.
export function isShared(element: ISyntaxElement): boolean {
var kind = element.kind();
return (kind === SyntaxKind.List || kind === SyntaxKind.SeparatedList) && (<ISyntaxNodeOrToken[]>element).length === 0;
return kind === SyntaxKind.List && (<ISyntaxNodeOrToken[]>element).length === 0;
}

export function syntaxTree(element: ISyntaxElement): SyntaxTree {
Expand Down Expand Up @@ -225,10 +225,6 @@ module TypeScript {
return element && element.kind() === SyntaxKind.List;
}

export function isSeparatedList(element: ISyntaxElement): boolean {
return element && element.kind() === SyntaxKind.SeparatedList;
}

export function syntaxID(element: ISyntaxElement): number {
if (isShared(element)) {
throw Errors.invalidOperation("Should not use shared syntax element as a key.");
Expand Down Expand Up @@ -350,7 +346,7 @@ module TypeScript {
}

function data(element: ISyntaxElement): number {
Debug.assert(isNode(element) || isList(element) || isSeparatedList(element));
Debug.assert(isNode(element) || isList(element));

// Lists and nodes all have a 'data' element.
var dataElement = <{ data: number }><any>element;
Expand Down
71 changes: 10 additions & 61 deletions src/services/syntax/syntaxGenerator.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1028,7 +1028,7 @@ function getType(child: IMemberDefinition): string {
return "ISyntaxToken";
}
else if (child.isSeparatedList) {
return child.elementType + "[]";
return "ISeparatedSyntaxList<" + child.elementType + ">";
}
else if (child.isList) {
return child.elementType + "[]";
Expand Down Expand Up @@ -2180,12 +2180,12 @@ function generateRewriter(): string {
" return isToken(node) ? <ISyntaxNodeOrToken>this.visitToken(<ISyntaxToken>node) : this.visitNode(<ISyntaxNode>node);\r\n" +
" }\r\n" +
"\r\n" +
" public visitList<T extends ISyntaxNodeOrToken>(list: T[]): T[] {\r\n" +
" var newItems: T[] = undefined;\r\n" +
" public visitList<T extends ISyntaxNodeOrToken[]>(list: T): T {\r\n" +
" var newItems: T = undefined;\r\n" +
"\r\n" +
" for (var i = 0, n = list.length; i < n; i++) {\r\n" +
" var item = list[i];\r\n" +
" var newItem = <T>this.visitNodeOrToken(item);\r\n" +
" var newItem = this.visitNodeOrToken(item);\r\n" +
"\r\n" +
" if (item !== newItem && !newItems) {\r\n" +
" newItems = [];\r\n" +
Expand All @@ -2200,31 +2200,9 @@ function generateRewriter(): string {
" }\r\n" +
"\r\n" +
" // Debug.assert(!newItems || newItems.length === childCount(list));\r\n" +
" return !newItems ? list : Syntax.list<T>(newItems);\r\n" +
" return !newItems ? list : <T>Syntax.list(newItems);\r\n" +
" }\r\n" +
"\r\n" +
" public visitSeparatedList<T extends ISyntaxNodeOrToken>(list: T[]): T[] {\r\n" +
" var newItems: ISyntaxNodeOrToken[] = undefined;\r\n" +
"\r\n" +
" for (var i = 0, n = childCount(list); i < n; i++) {\r\n" +
" var item = childAt(list, i);\r\n" +
" var newItem = isToken(item) ? <ISyntaxNodeOrToken>this.visitToken(<ISyntaxToken>item) : this.visitNode(<ISyntaxNode>item);\r\n" +
"\r\n" +
" if (item !== newItem && !newItems) {\r\n" +
" newItems = [];\r\n" +
" for (var j = 0; j < i; j++) {\r\n" +
" newItems.push(childAt(list, j));\r\n" +
" }\r\n" +
" }\r\n" +
"\r\n" +
" if (newItems) {\r\n" +
" newItems.push(newItem);\r\n" +
" }\r\n" +
" }\r\n" +
"\r\n" +
" // Debug.assert(newItems === undefined || newItems.length === childCount(list));\r\n" +
" return !newItems ? list : Syntax.separatedList<T>(newItems);\r\n" +
" }\r\n";
"\r\n"

for (var i = 0; i < definitions.length; i++) {
var definition = definitions[i];
Expand Down Expand Up @@ -2256,12 +2234,9 @@ function generateRewriter(): string {
if (child.isToken) {
result += "this.visitToken(node." + child.name + ")";
}
else if (child.isList) {
else if (child.isList || child.isSeparatedList) {
result += "this.visitList(node." + child.name + ")";
}
else if (child.isSeparatedList) {
result += "this.visitSeparatedList(node." + child.name + ")";
}
else if (child.type === "SyntaxKind") {
result += "node.kind";
}
Expand Down Expand Up @@ -2318,17 +2293,7 @@ function generateWalker(): string {
" list[i].accept(this);\r\n" +
" }\r\n" +
" }\r\n" +
"\r\n" +
" public visitSeparatedList(list: ISyntaxNodeOrToken[]): void {\r\n" +
" for (var i = 0, n = separatedListChildCount(list); i < n; i++) {\r\n" +
" if (i % 2 === 0) {\r\n" +
" list[i >> 1].accept(this);\r\n" +
" }\r\n" +
" else {\r\n" +
" this.visitToken(list.separators[i >> 1]);\r\n" +
" }\r\n" +
" }\r\n" +
" }\r\n";
"\r\n";

for (var i = 0; i < definitions.length; i++) {
var definition = definitions[i];
Expand All @@ -2347,12 +2312,9 @@ function generateWalker(): string {
result += " this.visitToken(node." + child.name + ");\r\n";
}
}
else if (child.isList) {
else if (child.isList || child.isSeparatedList) {
result += " this.visitList(node." + child.name + ");\r\n";
}
else if (child.isSeparatedList) {
result += " this.visitSeparatedList(node." + child.name + ");\r\n";
}
else if (isNodeOrToken(child)) {
if (child.isOptional) {
result += " visitNodeOrToken(this, node." + child.name + ");\r\n";
Expand Down Expand Up @@ -2721,15 +2683,6 @@ function generateIsTypeScriptSpecific(): string {

result += "module TypeScript {\r\n";

result += " function isSeparatedListTypeScriptSpecific(list: ISyntaxNodeOrToken[]): boolean {\r\n"
result += " for (var i = 0, n = list.childCount(); i < n; i++) {\r\n";
result += " if (isTypeScriptSpecific(list.childAt(i))) {\r\n";
result += " return true;\r\n";
result += " }\r\n";
result += " }\r\n\r\n";
result += " return false;\r\n";
result += " }\r\n\r\n";

result += " function isListTypeScriptSpecific(list: ISyntaxNodeOrToken[]): boolean {\r\n"
result += " for (var i = 0, n = list.length; i < n; i++) {\r\n";
result += " if (isTypeScriptSpecific(list[i])) {\r\n";
Expand All @@ -2743,7 +2696,6 @@ function generateIsTypeScriptSpecific(): string {
result += " if (!element) { return false; }\r\n";
result += " if (isToken(element)) { return false; }\r\n";
result += " if (isList(element)) { return isListTypeScriptSpecific(<ISyntaxNodeOrToken[]>element); }\r\n";
result += " if (isSeparatedList(element)) { return isSeparatedListTypeScriptSpecific(<ISyntaxNodeOrToken[]>element); }\r\n\r\n";
result += " switch (element.kind()) {\r\n";

for (var i = 0; i < definitions.length; i++) {
Expand Down Expand Up @@ -2850,12 +2802,9 @@ function generateIsTypeScriptSpecificMethod(definition: ITypeDefinition): string
addedCheck = true;

if (child.isTypeScriptSpecific) {
if (child.isList) {
if (child.isList || child.isSeparatedList) {
result += getPropertyAccess(child, "node") + ".length > 0";
}
else if (child.isSeparatedList) {
result += getPropertyAccess(child, "node") + ".childCount() > 0";
}
else {
result += "!!" + getPropertyAccess(child, "node");
}
Expand Down
2 changes: 0 additions & 2 deletions src/services/syntax/syntaxKind.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,6 @@ module TypeScript {
// Variable width tokens, trivia and lists.
None,
List,
SeparatedList,
TriviaList,

// Trivia
WhitespaceTrivia,
Expand Down
Loading