use std::borrow::Cow;
use std::char;
use std::str;
use std::string;
use self::Token::*;
#[derive(Eq, PartialEq, Debug)]
pub enum Token<'a> {
Whitespace(&'a str),
Newline,
Comment(&'a str),
Equals,
Period,
Comma,
Colon,
Plus,
LeftBrace,
RightBrace,
LeftBracket,
RightBracket,
Keylike(&'a str),
String { src: &'a str, val: Cow<'a, str> },
}
#[derive(Eq, PartialEq, Debug)]
pub enum Error {
InvalidCharInString(usize, char),
InvalidEscape(usize, char),
InvalidHexEscape(usize, char),
InvalidEscapeValue(usize, u32),
NewlineInString(usize),
Unexpected(usize, char),
UnterminatedString(usize),
NewlineInTableKey(usize),
EmptyTableKey(usize),
Wanted { at: usize, expected: &'static str, found: &'static str },
}
#[derive(Clone)]
pub struct Tokenizer<'a> {
input: &'a str,
chars: CrlfFold<'a>,
}
#[derive(Clone)]
struct CrlfFold<'a> {
chars: str::CharIndices<'a>,
}
#[derive(Debug)]
enum MaybeString {
NotEscaped(usize),
Owned(string::String),
}
impl<'a> Tokenizer<'a> {
pub fn new(input: &'a str) -> Tokenizer<'a> {
let mut t = Tokenizer {
input: input,
chars: CrlfFold {
chars: input.char_indices(),
},
};
// Eat utf-8 BOM
t.eatc('\u{feff}');
return t
}
pub fn next(&mut self) -> Result