Lexer for an experimental tiny language in Go

Question

I am implementing a lexer for an experimental language by the name "Phoenix", the language supports four primary arithmetic expression for integers only (add, subtract, multiply, divide), variable assignment and print statement.

Typical input:

# this is a comment
value = 1 + 7 / 4 * (30)
print('Value = ', value)

lexer.go:

package lexer
import (
 "bytes"
 "fmt"
 "io"
 "os"
 "phoenix/log"
 "phoenix/token"
 "unicode"
)
// Lexer type for our lexer
type Lexer struct {
 SourceFileName string
 fh *os.File
 UnScannedChar byte
 CurrentLine, CurrentCol uint64
}
// New sets up the lexer
func (l *Lexer) New(fileName string) {
 tmpFh, err := os.Open(fileName)
 l.fh = tmpFh
 if err != nil {
 log.Error(fmt.Sprintf("couldn't open file '%v'", fileName))
 os.Exit(0)
 }
 l.SourceFileName = fileName
 l.CurrentLine = 1
}
// NextChar updates Lexer.unScannedChar
func (l *Lexer) NextChar() (err error) {
 tempChar := make([]byte, 1)
 _, er := l.fh.Read(tempChar)
 if er != nil {
 if er == io.EOF {
 err = io.EOF
 return
 }
 log.Error(fmt.Sprintf("error while reading from file '%v'",
 l.SourceFileName))
 os.Exit(0)
 }
 l.UnScannedChar = tempChar[0]
 if l.UnScannedChar == '\n' {
 l.CurrentLine++
 l.CurrentCol = 0
 } else {
 l.CurrentCol++
 }
 return
}
func (l *Lexer) isIdentifierStart() bool {
 return unicode.IsLetter(rune(l.UnScannedChar)) ||
 l.UnScannedChar == '_'
}
func (l *Lexer) isIdentifierPart() bool {
 return l.isIdentifierStart() || unicode.IsNumber(rune(l.UnScannedChar))
}
func (l *Lexer) scanInteger() (newToken token.Token) {
 var buffer bytes.Buffer
 buffer.WriteString(string(l.UnScannedChar))
 l.NextChar()
 for unicode.IsDigit(rune(l.UnScannedChar)) {
 buffer.WriteString(string(l.UnScannedChar))
 l.NextChar()
 }
 newToken.Rep = token.INTEGER
 newToken.Data = buffer.String()
 return
}
func (l *Lexer) scanID() (newToken token.Token) {
 var buffer bytes.Buffer
 buffer.WriteString(string(l.UnScannedChar))
 l.NextChar()
 for l.isIdentifierPart() {
 buffer.WriteString(string(l.UnScannedChar))
 l.NextChar()
 }
 newToken.Data = buffer.String()
 switch newToken.Data {
 case "print":
 newToken.Rep = token.PRINT
 default:
 newToken.Rep = token.ID
 }
 /* we need to seek back the reading cursor as the next call of NextToken()
 will invoke NextChar(), neglecting the current unscanned character */
 l.fh.Seek(-1, os.SEEK_CUR)
 return
}
func (l *Lexer) scanString() (newToken token.Token) {
 var buffer bytes.Buffer
 l.NextChar()
 for l.UnScannedChar != '\'' {
 buffer.WriteString(string(l.UnScannedChar))
 l.NextChar()
 }
 newToken.Rep = token.STRING
 newToken.Data = buffer.String()
 return
}
// NextToken scans the Lexer.UnScannedChar and creates its correspondant token
func (l *Lexer) NextToken() (newToken token.Token) {
 err := l.NextChar()
 if err == io.EOF {
 newToken.Rep = token.EOF
 return
 }
 // scan integers
 if unicode.IsDigit(rune(l.UnScannedChar)) {
 return l.scanInteger()
 }
 // scan identifiers and print keyword
 if l.isIdentifierStart() {
 return l.scanID()
 }
 // scan strings
 if l.UnScannedChar == '\'' {
 return l.scanString()
 }
 // skip comments
 if l.UnScannedChar == '#' {
 err := l.NextChar()
 for err != io.EOF && l.UnScannedChar != '\n' {
 err = l.NextChar()
 if err == io.EOF {
 newToken.Rep = token.EOF
 return
 }
 }
 return l.NextToken()
 }
 switch l.UnScannedChar {
 case '\f', '\t', '\r', ' ':
 return l.NextToken()
 case '\n':
 newToken.Rep = token.NEWLINE
 case '(':
 newToken.Rep = token.LPAREN
 case ')':
 newToken.Rep = token.RPAREN
 case ',':
 newToken.Rep = token.COMMA
 case '+':
 newToken.Rep = token.ADD
 case '-':
 newToken.Rep = token.SUB
 case '*':
 newToken.Rep = token.MUL
 case '/':
 newToken.Rep = token.DIV
 case '=':
 newToken.Rep = token.ASSIGN
 default:
 log.Error(fmt.Sprintf(
 "at file %v, line %v: unknown character '%c'", l.SourceFileName,
 l.CurrentLine, l.UnScannedChar))
 newToken.Rep = token.UNKNOWN
 }
 return
}

token.go:

package token
const (
 SINGLEQ rune = iota
 LPAREN
 RPAREN
 COMMA
 ASSIGN
 ADD
 SUB
 MUL
 DIV
 STRING
 INTEGER
 REAL
 ID
 PRINT
 NEWLINE
 EOF
 UNKNOWN
)
type Token struct {
 Rep rune
 Data string
}
func (t Token) String() string {
 switch t.Rep {
 case SINGLEQ:
 return "'"
 case LPAREN:
 return "("
 case RPAREN:
 return ")"
 case COMMA:
 return ","
 case ASSIGN:
 return "="
 case ADD:
 return "+"
 case SUB:
 return "-"
 case MUL:
 return "*"
 case DIV:
 return "/"
 case STRING:
 return "string"
 case INTEGER:
 return "integer"
 case ID:
 return "identifier"
 case PRINT:
 return "print"
 case NEWLINE:
 return "newline"
 case EOF:
 return "EOF"
 }
 return "unknown"
}

I need the lexer to be as fast as possible and the code should be flexible to implement more extensions/features for the language.

So how can it be improved?

Answer 1

This will not be a full review, but rather a couple of improvements you can make. Firstly, why do all those string <-> []byte conversions when writing into a buffer? You can just do e.g.

buffer.Write([]byte{l.UnScannedChar})

instead of

buffer.WriteString(string(l.UnScannedChar))

Also, since you're only using file's Read and Seek methods, why not change os.File to io.ReadSeeker?

Answer 2

You should definitely not call os.Exit() from any library function return the error instead. I would not log unhandled errors either.

New has nothing to do with the Lexer as input, it is more like a constructor, so it could be a "static" method returning a Lexer instance.