Add callback to skip

[mysteriouslyseeing]

Is it possible to add a callback to skip?

This would be useful if you are trying to count newlines, and you also want to skip them:

use logos::{Lexer, Logos, Skip};

#[derive(Debug, Clone, Copy, Default)]
struct Extras {
    line_num: usize,
}

#[derive(Logos)]
#[logos(skip r" \r")]
// Is there a way of adding a callback here?
#[logos(skip(regex = r"\n", callback = newline_callback))]
#[logos(extras = Extras)]
enum Token {
    #[regex("[a-zA-Z]", |lexer| { lexer.slice().chars().next().unwrap() })]
    Letter(char),
    #[regex("[0-9]", |lexer| { lexer.slice().parse::<u8>().unwrap() })]
    Number(u8),
}

fn newline_callback(lexer: &mut Lexer<Token>) -> Skip {
    lexer.extras.line_num += 1;
    Skip
}

I know you could add another variant like Token::Ignored, or attach the newline regex to another variant, but both those solutions feel messy, and it'd be nice to have all the skip logic in one place.

[jeertmans]

Hi @mysteriouslyseeing!

As you said, this is totally possible to do this, but you need to do that with an enum variant.

If you have some time, you can try adding support for a callback for skip, this shouldn't be hard.

All you need (I think), is to modify the parser to also read optional callbacks

logos/logos-codegen/src/parser/mod.rs

Lines 135 to 143 in 727becf

	("skip", |parser, span, value| match value {
	NestedValue::Literal(lit) => {
	if let Some(literal) = parser.parse_literal(Lit::new(lit)) {
	parser.skips.push(literal);
	}
	}
	_ => {
	parser.err("Expected: #[logos(skip \"regex literal\")]", span);
	}

You can take inspiration for what is done for parsing token / regex definitions:

logos/logos-codegen/src/parser/mod.rs

Lines 211 to 254 in 727becf

	/// Parse attribute definition of a token:
	///
	/// + `#[token(literal[, callback])]`
	/// + `#[regex(literal[, callback])]`
	pub fn parse_definition(&mut self, attr: &mut Attribute) -> Option<Definition> {
	let mut nested = self.parse_attr(attr)?;
	let literal = match nested.parsed::<Lit>()? {
	Ok(lit) => self.parse_literal(lit)?,
	Err(err) => {
	self.err(err.to_string(), err.span());
	return None;
	}
	};
	let mut def = Definition::new(literal);
	for (position, next) in nested.enumerate() {
	match next {
	Nested::Unexpected(tokens) => {
	self.err("Unexpected token in attribute", tokens.span());
	}
	Nested::Unnamed(tokens) => match position {
	0 => def.callback = self.parse_callback(tokens),
	_ => {
	self.err(
	"\
	Expected a named argument at this position\n\
	\n\
	hint: If you are trying to define a callback here use: callback = ...\
	",
	tokens.span(),
	);
	}
	},
	Nested::Named(name, value) => {
	def.named_attr(name, value, self);
	}
	}
	}
	Some(def)
	}

[mysteriouslyseeing]

I've had a crack at it here

[mysteriouslyseeing]

Looking at erroring, it seems a bit odd that there isn't a callback for the case when no match was found. I think it just uses Error::default(), when it would be nice if the error could gather information from the lexer, such as span and line number. Maybe a #[logos(error_callback = callback)] would also make sense?

[mysteriouslyseeing]

I've done an implementation of the above here

[jeertmans]

I've done an implementation of the above here

This is a branch, do you have a concrete example I could read? :-)

[mysteriouslyseeing]

Sure, this is an example a possible error syntax:

fn main() {
    let input = "1 * 2 - 8 % 4";
    for res in Token::lexer(input) {
        match res {
            Ok(_) => (),
            Err(e) => eprintln!("{e}"),
        }
    }
}

use std::{fmt::Display, ops::Range};


use logos::Logos;

#[derive(Debug, Clone, Default, PartialEq)]
struct Error {
    message: String,
    index: Range<usize>,
}

impl Display for Error {
    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
        write!{f,
            "Error at {:?}: {}",
            self.index,
            self.message,
        }
    }
}

#[derive(Logos, Debug)]
#[logos(error = Error)]
#[logos(error_callback = |lex| Error {
    message: format!("Invalid character '{}'", lex.slice()),
    index: lex.span(),
})]
#[logos(skip(r" "))]
enum Token {
    #[regex(r"[\*/+-]")]
    Operator,
    #[regex("[0-9]+")]
    Number
}

which prints Error at 10..11: Invalid character '%'

That example without the enhancement, while not impossible to accomplish, is significantly more clunky - you have to call Iterator::next manually so you don't consume the lexer.

[jeertmans]

Ok this makes sense, I think this would be a great new feature!

[mysteriouslyseeing]

Should I make a new issue/pull request for this?

[jeertmans]

Should I make a new issue/pull request for this?

Yes please :-)