Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly
Sign up
Appearance settings

Commit b64cb0d

Browse files
refactor(lexer): add eof token
Signed-off-by: Lexus Drumgold <unicornware@flexdevelopment.llc>
1 parent f095a15 commit b64cb0d

File tree

5 files changed

+18
-20
lines changed

5 files changed

+18
-20
lines changed

‎.dictionary.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ iife
1616
jchen
1717
jsdoc
1818
kaisugi
19+
kleft
1920
lcov
2021
lintstagedrc
2122
mdast

‎src/enums/__tests__/token-kind.spec-d.ts‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,12 @@ describe('unit-d:enums/TokenKind', () => {
1818
.toMatchTypeOf<'delimiter'>()
1919
})
2020

21+
it('should match [eof: "eof"]', () => {
22+
expectTypeOf<typeof TestSubject>()
23+
.toHaveProperty('eof')
24+
.toMatchTypeOf<'eof'>()
25+
})
26+
2127
it('should match [markdown: "markdown"]', () => {
2228
expectTypeOf<typeof TestSubject>()
2329
.toHaveProperty('markdown')

‎src/enums/token-kind.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
enum TokenKind {
1212
closer = 'closer',
1313
delimiter = 'delimiter',
14+
eof = 'eof',
1415
markdown = 'markdown',
1516
opener = 'opener',
1617
tag = 'tag',

‎src/lexer.ts‎

Lines changed: 8 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -49,9 +49,9 @@ class Lexer {
4949
* @public
5050
* @readonly
5151
* @instance
52-
* @member {Optional<Token>} head
52+
* @member {Token} head
5353
*/
54-
public readonly head?: Optional<Token>
54+
public readonly head: Token
5555

5656
/**
5757
* Lexer options.
@@ -77,18 +77,6 @@ class Lexer {
7777
*/
7878
protected readonly reader: Reader
7979

80-
/**
81-
* Tail token.
82-
*
83-
* @see {@linkcode Token}
84-
*
85-
* @public
86-
* @readonly
87-
* @instance
88-
* @member {Optional<Token>} tail
89-
*/
90-
public readonly tail?: Token | undefined
91-
9280
/**
9381
* Token sequence.
9482
*
@@ -117,9 +105,7 @@ class Lexer {
117105
this.options = Object.freeze(options ??= {})
118106
this.reader = new Reader(value, this.options.from)
119107
this.tokens = []
120-
121108
this.head = this.tokenize()
122-
this.tail = this.tokens.at(-1)
123109
}
124110

125111
/**
@@ -333,9 +319,9 @@ class Lexer {
333319
* @protected
334320
* @instance
335321
*
336-
* @return {Token | undefined} Head token
322+
* @return {Token} Head token
337323
*/
338-
protected tokenize(): Token |undefined{
324+
protected tokenize(): Token {
339325
while (!this.reader.eof) {
340326
ok(this.reader.char, 'expected character')
341327

@@ -370,7 +356,10 @@ class Lexer {
370356
}
371357
}
372358

373-
return this.tokens[0]
359+
this.enter(kinds.eof, '')
360+
this.exit(kinds.eof)
361+
362+
return this.tokens[0]!
374363
}
375364

376365
/**

‎src/parser.ts‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,7 @@ import type {
6262
import {
6363
apply,
6464
expectEOF as eof,
65+
kleft,
6566
opt,
6667
rep,
6768
expectSingleResult as result,
@@ -236,7 +237,7 @@ class Parser {
236237
* @return {P<kinds, Root>} Root parser
237238
*/
238239
protected get root(): P<kinds, Root> {
239-
return apply(rep(this.comment), children => {
240+
return apply(kleft(rep(this.comment),tok(kinds.eof)), children => {
240241
const { transforms } = this.options
241242

242243
/**

0 commit comments

Comments
(0)

AltStyle によって変換されたページ (->オリジナル) /