top-down lexer in rust

Question

the other day, i decided to build a top-down lexer in rust, just for fun.
this is what i have so far:

use std::str::Chars;
use std::env;
use std::fs;
#[derive(Debug, Clone)]
enum Op {
 Plus,
 Minus,
 Multiply,
 Divide,
 Modulo,
 Equal,
 Not,
 NotEqual,
 Greater,
 GreaterOrEqual,
 Less,
 LessOrEqual,
}
#[derive(Debug, Clone)]
enum TokenKind {
 Opr(Op),
 Ident(String),
 Str(String),
 Num(String),
}
#[derive(Debug, Clone)]
struct Lexer<'a> {
 // string containing entire program
 source: &'a str,
 // the peekable character
 peek: Option<char>,
 // the characters iterator
 chars: Chars<'a>,
 // utf-8 position in the string
 pos: usize,
 // the bottom two are for error reporting/debugging
 row: usize,
 col: usize
}
#[allow(unused)]
impl<'a> Lexer<'a> {
 fn new(source: &'a str) -> Self {
 let mut chars = source.chars();
 Self {
 source,
 peek: chars.next(),
 chars,
 pos: 0,
 row: 1,
 col: 1
 }
 }
 #[inline]
 fn position(&self) -> (usize, usize) {
 (self.row, self.col)
 }
 #[inline]
 fn is_over(&self) -> bool {
 self.peek().is_none()
 }
 #[inline]
 fn bump(&mut self) {
 if let Some(ch) = self.chars.next() {
 self.pos += ch.len_utf8();
 self.col += 1;
 if ch == '\n' {
 self.col = 0;
 self.row += 1;
 }
 self.peek = Some(ch);
 } else {
 self.peek = None;
 }
 }
 #[inline]
 fn peek(&self) -> Option<char> {
 self.peek
 }
 #[inline]
 fn unicode_escape(&mut self, ch: char) -> Option<char> {
 // e.g. \n \t \u{...}
 let res = match ch {
 'n' => '\n',
 't' => '\t',
 'r' => '\r',
 'u' => todo!("implement unicode character codes"),
 'x' => todo!("implement hex character codes"),
 other => other
 };
 Some(res)
 }
 fn trim_ident(&mut self) -> TokenKind {
 let start_pos = self.pos;
 while let Some(s) = self.peek() {
 if !(s.is_alphanumeric() || s == '_') {
 break;
 }
 self.bump();
 }
 TokenKind::Ident(self.source[start_pos..self.pos].to_owned())
 }
 fn trim_number(&mut self) -> TokenKind {
 let start_pos = self.pos;
 while let Some(s) = self.peek() {
 if !s.is_numeric() {
 break;
 }
 self.bump();
 }
 TokenKind::Num(self.source[start_pos..self.pos].to_owned())
 }
 fn trim_string(&mut self) -> TokenKind {
 self.bump();
 let start_pos = self.pos;
 while let Some(s) = self.peek() {
 if s == '"' {
 break
 }
 if s == '\\' {
 self.bump();
 }
 self.bump();
 }
 if self.is_over() {
 todo!("Could not lex string");
 }
 self.bump();
 TokenKind::Str(self.source[start_pos..self.pos-1].to_owned())
 }
 
 #[inline]
 fn trim_comment(&mut self) {
 while let Some(s) = self.peek() {
 if s == '\n' {
 self.row = 0;
 self.col += 1;
 break
 }
 self.bump();
 }
 }
 #[inline]
 fn trim_block_comment(&mut self) {
 let mut recursion: usize = 1;
 while let Some(s) = self.next() {
 match s {
 '\n' => {
 self.row = 0;
 self.col += 1;
 },
 '/' => {
 if let Some('*') = self.peek() {
 self.bump();
 recursion += 1;
 }
 },
 '*' => {
 if let Some('/') = self.peek() {
 self.bump();
 recursion -= 1;
 }
 }
 _ => {},
 }
 if recursion == 0 {
 break
 }
 }
 }
 #[inline]
 fn trim_whitespace(&mut self) {
 while let Some(s) = self.peek() {
 if !s.is_whitespace() {
 break;
 }
 self.bump();
 }
 }
 
 #[inline]
 fn next_char(&mut self) -> Option<char> {
 let peek = self.peek;
 self.bump();
 peek
 }
}
impl<'a> Iterator for Lexer<'a> {
 type Item = TokenKind;
 fn next(&mut self) -> Option<TokenKind> {
 loop {
 self.trim_whitespace();
 return match self.peek()? {
 x if x.is_alphabetic() => return Some(self.trim_ident()),
 x if x.is_numeric() => return Some(self.trim_number()),
 '"' => {
 Some(self.trim_string())
 },
 '+' => {
 self.bump();
 Some(TokenKind::Opr(Op::Plus))
 },
 '-' => {
 self.bump();
 Some(TokenKind::Opr(Op::Minus))
 },
 '*' => {
 self.bump();
 Some(TokenKind::Opr(Op::Multiply))
 },
 '/' => {
 self.bump();
 if let Some('/') = self.peek() {
 self.trim_comment();
 continue;
 }
 if let Some('*') = self.peek() {
 self.trim_block_comment();
 continue;
 }
 Some(TokenKind::Opr(Op::Divide))
 },
 '%' => {
 self.bump();
 Some(TokenKind::Opr(Op::Modulo))
 },
 '=' => {
 self.bump();
 Some(TokenKind::Opr(Op::Equal))
 },
 '!' => {
 self.bump();
 if let Some('=') = self.peek() {
 self.bump();
 Some(TokenKind::Opr(Op::NotEqual))
 } else {
 Some(TokenKind::Opr(Op::Not))
 }
 },
 '>' => {
 self.bump();
 if let Some('=') = self.peek() {
 self.bump();
 Some(TokenKind::Opr(Op::GreaterOrEqual))
 } else {
 Some(TokenKind::Opr(Op::Greater))
 }
 },
 '<' => {
 self.bump();
 if let Some('=') = self.peek() {
 self.bump();
 Some(TokenKind::Opr(Op::LessOrEqual))
 } else {
 Some(TokenKind::Opr(Op::Less))
 }
 },
 _ => None
 };
 }
 }
}
macro_rules! elapsed {
 ($($code:tt)*) => {
 {
 let start = std::time::Instant::now();
 $($code)*;
 println!("Time elapsed is: {:?}", start.elapsed());
 }
 }
}
fn main() {
 for (i, arg) in env::args().enumerate() {
 if i == 0 { continue }
 let file = fs::read_to_string(arg).expect("failed to read file");
 let lexer: Lexer<'_> = Lexer::new(&file);
 elapsed!(for tok in lexer {
 //println!("{tok:?}");
 })
 }
}

HOWEVER i am sure there are many ways i could improve this code, readability and performance-wise. for instance, the loop {} in Lexer::next should be removed, but that messes up comments. also, as a side note, whenever i see other people implement lexers, their .peek() method clones the chars iterator and then calls .next(). my version stores peek as a separate value. can you tell me which is better?

EXAMPLES

"Hello, World!" -> Str("Hello, World!")
36 -> Num("36")
/* /* nested block comment */ */ -> nothing
foo_bar_baz_23 -> Ident("foo_bar_baz_23")
+ - / * % = ! != < > <= >= -> Opr(Plus) Opr(Minus) Opr(Divide) Opr(Multiply) Opr(Modulo) ... 
```

• pacmaninbw: Since no one has answered the question yet you can still edit the code to correct any errors.
• ggorlen: Sample inputs and outputs would be nice (driver code/test suite).
• John doe: why is nobody answering this question?

Answer 1

Instead of using .chars() and keeping track of pos I suggest using char_indices() which gives you an iterator over tuples of the position and char.

Instead of carefully tracking row and col in bump as part of the lexer, I'd create a custom iterator on top of char_indices() which iterates over something like:

struct CharInfo {
 ch: char,
 pos: usize,
 row: usize,
 col: usize
}

Instead of having a peek member in the lexer, I'd use a .peekable() on the CharInfo iterator to have an iterator that I could peek into.

Alternately, I might not use a peeking technique here. In this case your iterator is trivially cloneable. You can rewind the iterator by using a combination of .clone() and overwriting.

For example, with a peeking approach, you might use next_if to conditionally pull chars out of while they are whitespace.

loop {
 if self.chars.peek().map_or(false, |x| x.ch.is_whitespace()) {
 self.next(); // consume whitespace
 }
}

With a cloning approach, you might do:

loop {
 let current = self.chars.clone();
 if !self.chars.next().map_or(false, |x| x.ch.is_whitespace()) {
 self.chars = current;
 break;
 } 
}

But either way, you might get good mileage out of iterator methods (which you can use because you use a peekable iterator), this includes the builtin methods, methods from itertools or possibly other crates.