Merge pull request #1027 from Microsoft/separatedList

Use a flat list for separated syntax lists.

Author: CyrusNajmabadi

src/services/formatting/indentationTrackingWalker.ts

@@ -103,7 +103,7 @@ module TypeScript.Services.Formatting {
                 if (isToken(element)) {
                     this.visitToken(<ISyntaxToken>element);
                 }
-                else if (element.kind() === SyntaxKind.List || element.kind() === SyntaxKind.SeparatedList) {
+                else if (element.kind() === SyntaxKind.List) {
                     for (var i = 0, n = element.childCount(); i < n; i++) {
                         this.walk(element.childAt(i));
                     }

src/services/syntax/SyntaxGenerator.js

@@ -777,7 +777,7 @@ module TypeScript.IncrementalParser {
         function moveToFirstChildIfList(): void {
             var element = pieces[currentPieceIndex].element;
 
-            if (isList(element) || isSeparatedList(element)) {
+            if (isList(element)) {
                 // We cannot ever get an empty list in our piece path.  Empty lists are 'shared' and
                 // we make sure to filter that out before pushing any children.
                 // Debug.assert(childCount(element) > 0);

src/services/syntax/incrementalParser.ts

@@ -557,7 +557,7 @@ module TypeScript.Parser {
             // when next requested.
             while (true) {
                 // Parent must be a list or a node.  All of those have a 'data' element.
-                Debug.assert(isNode(parent) || isList(parent) || isSeparatedList(parent));
+                Debug.assert(isNode(parent) || isList(parent));
                 var dataElement = <{ data: number }><any>parent;
                 if (dataElement.data) {
                     dataElement.data &= SyntaxConstants.NodeParsedInStrictModeMask
@@ -592,20 +592,6 @@ module TypeScript.Parser {
                     }
                 }
             }
-            else if (isSeparatedList(parent)) {
-                var list2 = <ISyntaxNodeOrToken[]>parent;
-                for (var i = 0, n = list2.childCount(); i < n; i++) {
-                    if (list2.childAt(i) === oldToken) {
-                        if (i % 2 === 0) {
-                            list2[i / 2] = newToken;
-                        }
-                        else {
-                            list2.separators[(i - 1) / 2] = newToken;
-                        }
-                        return;
-                    }
-                }
-            }
 
             throw Errors.invalidOperation();
         }
@@ -3786,7 +3772,7 @@ module TypeScript.Parser {
             return result;
         }
 
-        function parseSeparatedSyntaxList<T extends ISyntaxNodeOrToken>(currentListType: ListParsingState, skippedTokens: ISyntaxToken[]): T[] {
+        function parseSeparatedSyntaxList<T extends ISyntaxNodeOrToken>(currentListType: ListParsingState, skippedTokens: ISyntaxToken[]): ISeparatedSyntaxList<T> {
             var savedListParsingState = listParsingState;
             listParsingState |= (1 << currentListType);
 
@@ -3799,7 +3785,7 @@ module TypeScript.Parser {
 
         // Returns true if we should abort parsing.
         function abortParsingListOrMoveToNextToken<T extends ISyntaxNodeOrToken>(
-                currentListType: ListParsingState, nodes: T[], separators: ISyntaxToken[], skippedTokens: ISyntaxToken[]): boolean {
+                currentListType: ListParsingState, nodeAndSeparators: ISyntaxNodeOrToken[], skippedTokens: ISyntaxToken[]): boolean {
             // Ok.  We're at a token that is not a terminator for the list and wasn't the start of 
             // an item in the list. Definitely report an error for this token.
             reportUnexpectedTokenDiagnostic(currentListType);
@@ -3819,30 +3805,27 @@ module TypeScript.Parser {
 
             // Otherwise, if none of the lists we're in can capture this token, then we need to 
             // unilaterally skip it.  Note: we've already reported an error above.
-            addSkippedTokenToList(nodes, separators, skippedTokens, consumeToken(currentToken()));
+            addSkippedTokenToList(nodeAndSeparators, skippedTokens, consumeToken(currentToken()));
 
             // Continue parsing this list.  Attach this token to whatever we've seen already.
             return false;
         }
 
-        function addSkippedTokenToList<T extends ISyntaxNodeOrToken>(
-                nodes: T[], separators: ISyntaxToken[], skippedTokens: ISyntaxToken[], skippedToken: ISyntaxToken): void {
+        function addSkippedTokenToList(
+                nodesAndSeparators: ISyntaxNodeOrToken[], skippedTokens: ISyntaxToken[], skippedToken: ISyntaxToken): void {
             // Now, add this skipped token to the last item we successfully parsed in the list.  Or
             // add it to the list of skipped tokens if we haven't parsed anything.  Our caller will
             // have to deal with them.
             //
             // Note: we only bother doing this if we're creating a concrete syntax tree.
             if (syntaxFactory.isConcrete) {
-                var length = nodes.length + (separators ? separators.length : 0);
+                var length = nodesAndSeparators.length;
 
                 for (var i = length - 1; i >= 0; i--) {
-                    var array: ISyntaxNodeOrToken[] = separators && (i % 2 === 1) ? separators : nodes;
-                    var arrayIndex = separators ? IntegerUtilities.integerDivide(i, 2) : i;
-
-                    var item = array[arrayIndex];
+                    var item = nodesAndSeparators[i];
                     var _lastToken = lastToken(item);
                     if (_lastToken && _lastToken.fullWidth() > 0) {
-                        array[arrayIndex] = <T>addSkippedTokenAfterNodeOrToken(item, skippedToken);
+                        nodesAndSeparators[i] = addSkippedTokenAfterNodeOrToken(item, skippedToken);
                         return;
                     }
                 }
@@ -3895,7 +3878,7 @@ module TypeScript.Parser {
 
                     // List wasn't complete and we didn't get an item.  Figure out if we should bail out
                     // or skip a token and continue.
-                    var abort = abortParsingListOrMoveToNextToken(currentListType, items, /*separators:*/ undefined, skippedTokens);
+                    var abort = abortParsingListOrMoveToNextToken(currentListType, items, skippedTokens);
                     if (abort) {
                         break;
                     }
@@ -3914,9 +3897,8 @@ module TypeScript.Parser {
             return result;
         }
 
-        function parseSeparatedSyntaxListWorker<T extends ISyntaxNodeOrToken>(currentListType: ListParsingState, skippedTokens: ISyntaxToken[]): T[] {
-            var nodes: T[] = getArray();
-            var separators: ISyntaxToken[] = getArray();
+        function parseSeparatedSyntaxListWorker<T extends ISyntaxNodeOrToken>(currentListType: ListParsingState, skippedTokens: ISyntaxToken[]): ISeparatedSyntaxList<T> {
+            var nodesAndSeparators: ISyntaxNodeOrToken[] = getArray();
 
             // Debug.assert(nodes.length === 0);
             // Debug.assert(separators.length === 0);
@@ -3934,7 +3916,7 @@ module TypeScript.Parser {
                 // continue parsing.
 
                 // Debug.assert(oldItemsCount % 2 === 0);
-                var succeeded = tryParseExpectedListItem(currentListType, inErrorRecovery, nodes, /*processItems:*/ undefined);
+                var succeeded = tryParseExpectedListItem(currentListType, inErrorRecovery, nodesAndSeparators, /*processItems:*/ undefined);
 
                 if (!succeeded) {
                     // We weren't able to parse out a list element.
@@ -3948,7 +3930,7 @@ module TypeScript.Parser {
 
                     // List wasn't complete and we didn't get an item.  Figure out if we should bail out
                     // or skip a token and continue.
-                    var abort = abortParsingListOrMoveToNextToken(currentListType, nodes, separators, skippedTokens);
+                    var abort = abortParsingListOrMoveToNextToken(currentListType, nodesAndSeparators, skippedTokens);
                     if (abort) {
                         break;
                     }
@@ -3973,7 +3955,7 @@ module TypeScript.Parser {
                 var tokenKind = _currentToken.kind();
                 if (tokenKind === _separatorKind || tokenKind === SyntaxKind.CommaToken) {
                     // Consume the last separator and continue parsing list elements.
-                    separators.push(consumeToken(_currentToken));
+                    nodesAndSeparators.push(consumeToken(_currentToken));
                     continue;
                 }
 
@@ -4001,7 +3983,7 @@ module TypeScript.Parser {
 
                 if (allowAutomaticSemicolonInsertion && canEatAutomaticSemicolon(/*allowWithoutNewline:*/ false)) {
                     var semicolonToken = eatExplicitOrAutomaticSemicolon(/*allowWithoutNewline:*/ false) || Syntax.emptyToken(SyntaxKind.SemicolonToken);
-                    separators.push(semicolonToken);
+                    nodesAndSeparators.push(semicolonToken);
                     // Debug.assert(items.length % 2 === 0);
                     continue;
                 }
@@ -4011,7 +3993,7 @@ module TypeScript.Parser {
                 // This time mark that we're in error recovery mode though.
                 //
                 // Note: trying to eat this token will emit the appropriate diagnostic.
-                separators.push(eatToken(_separatorKind));
+                nodesAndSeparators.push(eatToken(_separatorKind));
 
                 // Now that we're in 'error recovery' mode we cantweak some parsing rules as 
                 // appropriate.  For example, if we have:
@@ -4027,12 +4009,11 @@ module TypeScript.Parser {
                 inErrorRecovery = true;
             }
 
-            var result = Syntax.separatedList<T>(nodes, separators);
+            var result = Syntax.separatedList<T>(nodesAndSeparators);
 
             // Can't return if it has more then 0 elements.  In that case, the list will have been
             // copied into the SyntaxList.
-            returnZeroLengthArray(nodes);
-            returnZeroLengthArray(separators);
+            returnZeroLengthArray(nodesAndSeparators);
 
             return result;
         }

src/services/syntax/parser.ts

@@ -8,7 +8,7 @@ module TypeScript {
     // nodes don't have positions or parents.
     export function isShared(element: ISyntaxElement): boolean {
         var kind = element.kind();
-        return (kind === SyntaxKind.List || kind === SyntaxKind.SeparatedList) && (<ISyntaxNodeOrToken[]>element).length === 0;
+        return kind === SyntaxKind.List && (<ISyntaxNodeOrToken[]>element).length === 0;
     }
 
     export function syntaxTree(element: ISyntaxElement): SyntaxTree {
@@ -225,10 +225,6 @@ module TypeScript {
         return element && element.kind() === SyntaxKind.List;
     }
 
-    export function isSeparatedList(element: ISyntaxElement): boolean {
-        return element && element.kind() === SyntaxKind.SeparatedList;
-    }
-
     export function syntaxID(element: ISyntaxElement): number {
         if (isShared(element)) {
             throw Errors.invalidOperation("Should not use shared syntax element as a key.");
@@ -350,7 +346,7 @@ module TypeScript {
     }
 
     function data(element: ISyntaxElement): number {
-        Debug.assert(isNode(element) || isList(element) || isSeparatedList(element));
+        Debug.assert(isNode(element) || isList(element));
 
         // Lists and nodes all have a 'data' element.
         var dataElement = <{ data: number }><any>element;

src/services/syntax/syntaxElement.ts

@@ -1028,7 +1028,7 @@ function getType(child: IMemberDefinition): string {
         return "ISyntaxToken";
     }
     else if (child.isSeparatedList) {
-        return child.elementType + "[]";
+        return "ISeparatedSyntaxList<" + child.elementType + ">";
     }
     else if (child.isList) {
         return child.elementType + "[]";
@@ -2180,12 +2180,12 @@ function generateRewriter(): string {
 "            return isToken(node) ? <ISyntaxNodeOrToken>this.visitToken(<ISyntaxToken>node) : this.visitNode(<ISyntaxNode>node);\r\n" +
 "        }\r\n" +
 "\r\n" +
-"        public visitList<T extends ISyntaxNodeOrToken>(list: T[]): T[] {\r\n" +
-"            var newItems: T[] = undefined;\r\n" +
+"        public visitList<T extends ISyntaxNodeOrToken[]>(list: T): T {\r\n" +
+"            var newItems: T = undefined;\r\n" +
 "\r\n" +
 "            for (var i = 0, n = list.length; i < n; i++) {\r\n" +
 "                var item = list[i];\r\n" +
-"                var newItem = <T>this.visitNodeOrToken(item);\r\n" +
+"                var newItem = this.visitNodeOrToken(item);\r\n" +
 "\r\n" +
 "                if (item !== newItem && !newItems) {\r\n" +
 "                    newItems = [];\r\n" +
@@ -2200,31 +2200,9 @@ function generateRewriter(): string {
 "            }\r\n" +
 "\r\n" +
 "            // Debug.assert(!newItems || newItems.length === childCount(list));\r\n" +
-"            return !newItems ? list : Syntax.list<T>(newItems);\r\n" +
+"            return !newItems ? list : <T>Syntax.list(newItems);\r\n" +
 "        }\r\n" +
-"\r\n" +
-"        public visitSeparatedList<T extends ISyntaxNodeOrToken>(list: T[]): T[] {\r\n" +
-"            var newItems: ISyntaxNodeOrToken[] = undefined;\r\n" +
-"\r\n" +
-"            for (var i = 0, n = childCount(list); i < n; i++) {\r\n" +
-"                var item = childAt(list, i);\r\n" +
-"                var newItem = isToken(item) ? <ISyntaxNodeOrToken>this.visitToken(<ISyntaxToken>item) : this.visitNode(<ISyntaxNode>item);\r\n" +
-"\r\n" +
-"                if (item !== newItem && !newItems) {\r\n" +
-"                    newItems = [];\r\n" +
-"                    for (var j = 0; j < i; j++) {\r\n" +
-"                        newItems.push(childAt(list, j));\r\n" +
-"                    }\r\n" +
-"                }\r\n" +
-"\r\n" +
-"                if (newItems) {\r\n" +
-"                    newItems.push(newItem);\r\n" +
-"                }\r\n" +
-"            }\r\n" +
-"\r\n" +
-"            // Debug.assert(newItems === undefined || newItems.length === childCount(list));\r\n" +
-"            return !newItems ? list : Syntax.separatedList<T>(newItems);\r\n" +
-"        }\r\n";
+"\r\n" 
 
     for (var i = 0; i < definitions.length; i++) {
         var definition = definitions[i];
@@ -2256,12 +2234,9 @@ function generateRewriter(): string {
             if (child.isToken) {
                 result += "this.visitToken(node." + child.name + ")";
             }
-            else if (child.isList) {
+            else if (child.isList || child.isSeparatedList) {
                 result += "this.visitList(node." + child.name + ")";
             }
-            else if (child.isSeparatedList) {
-                result += "this.visitSeparatedList(node." + child.name + ")";
-            }
             else if (child.type === "SyntaxKind") {
                 result += "node.kind";
             }
@@ -2318,17 +2293,7 @@ function generateWalker(): string {
 "                list[i].accept(this);\r\n" +
 "            }\r\n" +
 "        }\r\n" +
-"\r\n" +
-"        public visitSeparatedList(list: ISyntaxNodeOrToken[]): void {\r\n" +
-"            for (var i = 0, n = separatedListChildCount(list); i < n; i++) {\r\n" +
-"                if (i % 2 === 0) {\r\n" +
-"                    list[i >> 1].accept(this);\r\n" + 
-"                }\r\n" +
-"                else {\r\n" +
-"                    this.visitToken(list.separators[i >> 1]);\r\n" +
-"                }\r\n" +
-"            }\r\n" +
-"        }\r\n";
+"\r\n";
 
     for (var i = 0; i < definitions.length; i++) {
         var definition = definitions[i];
@@ -2347,12 +2312,9 @@ function generateWalker(): string {
                     result += "            this.visitToken(node." + child.name + ");\r\n";
                 }
             }
-            else if (child.isList) {
+            else if (child.isList || child.isSeparatedList) {
                 result += "            this.visitList(node." + child.name + ");\r\n";
             }
-            else if (child.isSeparatedList) {
-                result += "            this.visitSeparatedList(node." + child.name + ");\r\n";
-            }
             else if (isNodeOrToken(child)) {
                 if (child.isOptional) {
                     result += "            visitNodeOrToken(this, node." + child.name + ");\r\n";
@@ -2721,15 +2683,6 @@ function generateIsTypeScriptSpecific(): string {
 
     result += "module TypeScript {\r\n";
 
-    result += "    function isSeparatedListTypeScriptSpecific(list: ISyntaxNodeOrToken[]): boolean {\r\n"
-    result += "        for (var i = 0, n = list.childCount(); i < n; i++) {\r\n";
-    result += "            if (isTypeScriptSpecific(list.childAt(i))) {\r\n";
-    result += "                return true;\r\n";
-    result += "            }\r\n";
-    result += "        }\r\n\r\n";
-    result += "        return false;\r\n";
-    result += "    }\r\n\r\n";
-
     result += "    function isListTypeScriptSpecific(list: ISyntaxNodeOrToken[]): boolean {\r\n"
     result += "        for (var i = 0, n = list.length; i < n; i++) {\r\n";
     result += "            if (isTypeScriptSpecific(list[i])) {\r\n";
@@ -2743,7 +2696,6 @@ function generateIsTypeScriptSpecific(): string {
     result += "        if (!element) { return false; }\r\n";
     result += "        if (isToken(element)) { return false; }\r\n";
     result += "        if (isList(element)) { return isListTypeScriptSpecific(<ISyntaxNodeOrToken[]>element); }\r\n";
-    result += "        if (isSeparatedList(element)) { return isSeparatedListTypeScriptSpecific(<ISyntaxNodeOrToken[]>element); }\r\n\r\n";
     result += "        switch (element.kind()) {\r\n";
 
     for (var i = 0; i < definitions.length; i++) {
@@ -2850,12 +2802,9 @@ function generateIsTypeScriptSpecificMethod(definition: ITypeDefinition): string
         addedCheck = true;
 
         if (child.isTypeScriptSpecific) {
-            if (child.isList) {
+            if (child.isList || child.isSeparatedList) {
                 result += getPropertyAccess(child, "node") + ".length > 0";
             }
-            else if (child.isSeparatedList) {
-                result += getPropertyAccess(child, "node") + ".childCount() > 0";
-            }
             else {
                 result += "!!" + getPropertyAccess(child, "node");
             }

src/services/syntax/syntaxGenerator.ts

@@ -5,8 +5,6 @@ module TypeScript {
         // Variable width tokens, trivia and lists.
         None,
         List,
-        SeparatedList,
-        TriviaList,
 
         // Trivia
         WhitespaceTrivia,