Add injections and wrapping grammars for tree-sitter grammar

Author: claytonrcarter

grammars/tree-sitter-html.cson

@@ -0,0 +1,36 @@
+# This grammar is responsible for injecting the actual PHP or HTML grammars
+# For the actual PHP scopes, see tree-sitter-php.cson
+name: 'PHP'
+scopeName: 'text.html.php'
+type: 'tree-sitter'
+parser: 'tree-sitter-embedded-php'
+
+fileTypes: [
+  'aw'
+  'ctp'
+  'inc'
+  'install'
+  'module'
+  'php'
+  'php_cs'
+  'php3'
+  'php4'
+  'php5'
+  'phpt'
+  'phtml'
+  'profile'
+]
+
+firstLineRegex: [
+  '^\\s*<\\?([pP][hH][pP]|=|\\s|$)'
+]
+
+scopes:
+  'template > content:nth-child(0)': [
+    {match: /^\#!.*(?:\s|\/)php\d?(?:$|\s)/, scopes: 'comment.line.shebang.php'}
+  ],
+
+  'php': [
+    {match: '\\n', scopes: 'meta.embedded.block.php'}
+    'meta.embedded.line.php'
+  ]

grammars/tree-sitter-phpdoc.cson

@@ -0,0 +1,18 @@
+name: 'PHPDoc'
+scopeName: 'comment.block.documentation.phpdoc.php'
+type: 'tree-sitter'
+parser: 'tree-sitter-phpdoc'
+
+injectionRegex: 'phpdoc|PHPDoc'
+
+scopes:
+  tag_name: 'keyword.other.phpdoc.php'
+  type_list: 'meta.other.type.phpdoc.php'
+  'type_list > "|"': 'punctuation.separator.delimiter.php'
+  'array_type > "[]"': 'keyword.other.array.phpdoc.php'
+  primitive_type: 'keyword.other.type.php'
+  'named_type > name': 'support.class.php'
+  '* > namespace_name_as_prefix': 'support.other.namespace.php'
+  # FIXME not working
+  # '* > namespace_name_as_prefix > "\\"': 'punctuation.separator.inheritance.php'
+  '* > qualified_name > name': 'support.class.php'

lib/main.js

@@ -0,0 +1,28 @@
+exports.activate = function() {
+  if (!atom.grammars.addInjectionPoint) return
+
+  // inject source.php into text.html.php
+  atom.grammars.addInjectionPoint('text.html.php', {
+    type: 'php',
+    language () { return 'php' },
+    content (php) { return php }
+  })
+
+  // inject html into text.html.php
+  atom.grammars.addInjectionPoint('text.html.php', {
+    type: 'template',
+    language () { return 'html' },
+    content (node) { return node.descendantsOfType('content') }
+  })
+
+  // inject phpDoc comments into PHP comments
+  atom.grammars.addInjectionPoint('source.php', {
+    type: 'comment',
+    language (comment) {
+      if (comment.text.startsWith('/**') && !comment.text.startsWith('/***')) {
+        return 'phpdoc'
+      }
+    },
+    content (comment) { return comment }
+  })
+}

package-lock.json

@@ -6,6 +6,7 @@
     "atom": "*",
     "node": "*"
   },
+  "main": "lib/main",
   "homepage": "http://atom.github.io/language-php",
   "repository": {
     "type": "git",
@@ -16,7 +17,9 @@
     "url": "https://github.com/atom/language-php/issues"
   },
   "dependencies": {
+    "tree-sitter-embedded-php": "0.0.4",
     "tree-sitter-php-abc": "^0.17.0",
+    "tree-sitter-phpdoc": "0.0.4"
   },
   "devDependencies": {
     "coffeelint": "^1.10.1",

package.json

@@ -0,0 +1,101 @@
+const dedent = require("dedent");
+
+module.exports = {
+  // https://github.com/atom/atom/blob/b3d3a52d9e4eb41f33df7b91ad1f8a2657a04487/spec/tree-sitter-language-mode-spec.js#L47-L55
+  expectTokensToEqual(editor, expectedTokenLines, startingRow = 1) {
+    const lastRow = editor.getLastScreenRow();
+
+    for (let row = startingRow; row <= lastRow - startingRow; row++) {
+      const tokenLine = editor
+        .tokensForScreenRow(row)
+        .map(({ text, scopes }) => ({
+          text,
+          scopes: scopes.map((scope) =>
+            scope
+              .split(" ")
+              .map((className) => className.replace("syntax--", ""))
+              .join(".")
+          ),
+        }));
+
+      const expectedTokenLine = expectedTokenLines[row - startingRow];
+
+      expect(tokenLine.length).toEqual(expectedTokenLine.length);
+      for (let i = 0; i < tokenLine.length; i++) {
+        expect(tokenLine[i].text).toEqual(
+          expectedTokenLine[i].text,
+          `Token ${i}, row: ${row}`
+        );
+        expect(tokenLine[i].scopes).toEqual(
+          expectedTokenLine[i].scopes,
+          `Token ${i}, row: ${row}, token: '${tokenLine[i].text}'`
+        );
+      }
+    }
+  },
+
+  toHaveScopesAtPosition(posn, token, expected, includeEmbeddedScopes = false) {
+    if (expected === undefined) {
+      expected = token;
+    }
+    if (token === undefined) {
+      expected = [];
+    }
+
+    // token is not used at this time; it's just a way to keep note where we are
+    // in the line
+
+    let filterEmbeddedScopes = (scope) =>
+      includeEmbeddedScopes ||
+      (scope !== "text.html.php" &&
+        scope !== "meta.embedded.block.php" &&
+        scope !== "meta.embedded.line.php");
+
+    let actual = this.actual
+      .scopeDescriptorForBufferPosition(posn)
+      .scopes.filter(filterEmbeddedScopes);
+
+    let notExpected = actual.filter((scope) => !expected.includes(scope));
+    let notReceived = expected.filter((scope) => !actual.includes(scope));
+
+    let pass = notExpected.length === 0 && notReceived.length === 0;
+
+    if (pass) {
+      this.message = () => "Scopes matched";
+    } else {
+      let line = this.actual.getBuffer().lineForRow(posn[0]);
+      let caret = " ".repeat(posn[1]) + "^";
+
+      this.message = () =>
+        `Failure:
+  Scopes did not match at position [${posn.join(", ")}]:
+${line}
+${caret}
+  These scopes were expected but not received:
+      ${notReceived.join(", ")}
+  These scopes were received but not expected:
+      ${notExpected.join(", ")}
+      `;
+    }
+
+    return pass;
+  },
+
+  setPhpText(content) {
+    this.setText(`<?php
+${dedent(content)}
+`);
+  },
+
+  nextHighlightingUpdate(editor) {
+    return new Promise((resolve) => {
+      const subscription = editor
+        .getBuffer()
+        .getLanguageMode()
+        .onDidChangeHighlighting(() => {
+          subscription.dispose();
+          resolve();
+        });
+    });
+  },
+};

spec/tree-sitter-helpers.js

@@ -0,0 +1,74 @@
+const dedent = require("dedent");
+const {expectTokensToEqual, toHaveScopesAtPosition, nextHighlightingUpdate} = require('./tree-sitter-helpers')
+
+describe("Tree-sitter PHP grammar", () => {
+  var editor;
+
+  beforeEach(async () => {
+    atom.config.set("core.useTreeSitterParsers", true);
+    await atom.packages.activatePackage("language-php");
+    await atom.packages.activatePackage("language-html");
+    editor = await atom.workspace.open("foo.php");
+  });
+
+  beforeEach(function () {
+    this.addMatchers({ toHaveScopesAtPosition });
+  });
+
+  describe("loading the grammar", () => {
+    it('loads the wrapper HTML grammar', () => {
+      embeddingGrammar = atom.grammars.grammarForScopeName("text.html.php");
+      expect(embeddingGrammar).toBeTruthy();
+      expect(embeddingGrammar.scopeName).toBe("text.html.php");
+      expect(embeddingGrammar.constructor.name).toBe("TreeSitterGrammar");
+      // FIXME how to test that all selectors were loaded correctly? Invalid
+      // selectors may generate errors and it would be great to catch those here.
+
+      // injections
+      expect(embeddingGrammar.injectionPointsByType.template).toBeTruthy();
+      expect(embeddingGrammar.injectionPointsByType.php).toBeTruthy();
+    })
+  });
+
+  describe("shebang", () => {
+    it("recognises shebang on the first line of document", () => {
+      editor.setText(dedent`
+                  #!/usr/bin/env php
+                  <?php echo "test"; ?>
+                `);
+
+      // expect(editor).toHaveScopesAtPosition([0, 0], "#!",               ["text.html.php", "comment.line.shebang.php", "punctuation.definition.comment.php"], true);
+      expect(editor).toHaveScopesAtPosition([0, 1], "#!",               ["text.html.php", "comment.line.shebang.php",
+        // FIXME following scopes differ from TM
+        'source.html'
+      ], true);
+      expect(editor).toHaveScopesAtPosition([0, 2], "/usr/bin/env php", ["text.html.php", "comment.line.shebang.php",
+        // FIXME following scopes differ from TM
+        'source.html'
+      ], true);
+      expect(editor).toHaveScopesAtPosition([1, 0], "<?php",            ["text.html.php",
+        // "meta.embedded.line.php", "punctuation.section.embedded.begin.php"
+      ], true);
+      expect(editor).toHaveScopesAtPosition([1, 1], "<?php",            ["text.html.php", "meta.embedded.line.php", "punctuation.section.embedded.begin.php",
+        // FIXME following scopes differ from TM
+        'source.php'
+      ], true);
+    });
+
+    it("does not recognize shebang on any of the other lines", () => {
+      editor.setText(dedent`
+
+                  #!/usr/bin/env php
+                  <?php echo "test"; ?>
+                `);
+
+      expect(editor).toHaveScopesAtPosition([1, 0], "#!", ["text.html.php"], true);
+      expect(editor).toHaveScopesAtPosition([1, 1], "#!", ["text.html.php",
+        // FIXME following scopes differ from TM
+        'meta.embedded.line.php',
+        'source.php',
+        'punctuation.section.embedded.begin.php'
+      ], true);
+    });
+  });
+});