refactor(lexer): add eof token

Signed-off-by: Lexus Drumgold <[email protected]>

Author: unicornware

.dictionary.txt

@@ -16,6 +16,7 @@ iife
 jchen
 jsdoc
 kaisugi
+kleft
 lcov
 lintstagedrc
 mdast

src/enums/__tests__/token-kind.spec-d.ts

@@ -18,6 +18,12 @@ describe('unit-d:enums/TokenKind', () => {
       .toMatchTypeOf<'delimiter'>()
   })
 
+  it('should match [eof: "eof"]', () => {
+    expectTypeOf<typeof TestSubject>()
+      .toHaveProperty('eof')
+      .toMatchTypeOf<'eof'>()
+  })
+
   it('should match [markdown: "markdown"]', () => {
     expectTypeOf<typeof TestSubject>()
       .toHaveProperty('markdown')

src/enums/token-kind.ts

@@ -11,6 +11,7 @@
 enum TokenKind {
   closer = 'closer',
   delimiter = 'delimiter',
+  eof = 'eof',
   markdown = 'markdown',
   opener = 'opener',
   tag = 'tag',

src/lexer.ts

@@ -49,9 +49,9 @@ class Lexer {
    * @public
    * @readonly
    * @instance
-   * @member {Optional<Token>} head
+   * @member {Token} head
    */
-  public readonly head?: Optional<Token>
+  public readonly head: Token
 
   /**
    * Lexer options.
@@ -77,18 +77,6 @@ class Lexer {
    */
   protected readonly reader: Reader
 
-  /**
-   * Tail token.
-   *
-   * @see {@linkcode Token}
-   *
-   * @public
-   * @readonly
-   * @instance
-   * @member {Optional<Token>} tail
-   */
-  public readonly tail?: Token | undefined
-
   /**
    * Token sequence.
    *
@@ -117,9 +105,7 @@ class Lexer {
     this.options = Object.freeze(options ??= {})
     this.reader = new Reader(value, this.options.from)
     this.tokens = []
-
     this.head = this.tokenize()
-    this.tail = this.tokens.at(-1)
   }
 
   /**
@@ -333,9 +319,9 @@ class Lexer {
    * @protected
    * @instance
    *
-   * @return {Token | undefined} Head token
+   * @return {Token} Head token
    */
-  protected tokenize(): Token | undefined {
+  protected tokenize(): Token {
     while (!this.reader.eof) {
       ok(this.reader.char, 'expected character')
 
@@ -370,7 +356,10 @@ class Lexer {
       }
     }
 
-    return this.tokens[0]
+    this.enter(kinds.eof, '')
+    this.exit(kinds.eof)
+
+    return this.tokens[0]!
   }
 
   /**

src/parser.ts

@@ -62,6 +62,7 @@ import type {
 import {
   apply,
   expectEOF as eof,
+  kleft,
   opt,
   rep,
   expectSingleResult as result,
@@ -236,7 +237,7 @@ class Parser {
    * @return {P<kinds, Root>} Root parser
    */
   protected get root(): P<kinds, Root> {
-    return apply(rep(this.comment), children => {
+    return apply(kleft(rep(this.comment), tok(kinds.eof)), children => {
       const { transforms } = this.options
 
       /**