Adjust spans for more complex types

src/de.rs

@@ -821,16 +821,20 @@ impl<'a> Deserializer<'a> {
                 Value { e: E::Boolean(false), start: start, end: end }
             }
             Some((span, Token::Keylike(key))) => self.number_or_date(span, key)?,
-            Some((_, Token::Plus)) => self.number_leading_plus()?,
+            Some((span, Token::Plus)) => self.number_leading_plus(span)?,
-            Some((Span { start, end }, Token::LeftBrace)) => {
+            Some((Span { start, .. }, Token::LeftBrace)) => {
-                self.inline_table().map(|table| Value {
+                self.inline_table().map(|(Span { end, .. }, table)| Value {
                     e: E::InlineTable(table),
                     start: start,
                     end: end
                 })?
             }
-            Some((Span { start, end }, Token::LeftBracket)) => {
+            Some((Span { start, .. }, Token::LeftBracket)) => {
-                self.array().map(|array| Value { e: E::Array(array), start: start, end: end })?
+                self.array().map(|(Span { end, .. }, array)| Value {
+                    e: E::Array(array),
+                    start: start,
+                    end: end
+                })?
             }
             Some(token) => {
                 return Err(self.error(at, ErrorKind::Wanted {
@@ -882,11 +886,13 @@ impl<'a> Deserializer<'a> {
         }
     }
 
-    fn number_leading_plus(&mut self) -> Result<Value<'a>, Error> {
+    fn number_leading_plus(&mut self, Span { start, .. }: Span) -> Result<Value<'a>, Error> {
-        let start = self.tokens.current();
+        let start_token = self.tokens.current();
         match self.next()? {
-            Some((span, Token::Keylike(s))) => self.number(span, s),
+            Some((Span { end, .. }, Token::Keylike(s))) => {
-            _ => Err(self.error(start, ErrorKind::NumberInvalid)),
+                self.number(Span { start: start, end: end }, s)
+            },
+            _ => Err(self.error(start_token, ErrorKind::NumberInvalid)),
         }
     }
 
@@ -1042,11 +1048,11 @@ impl<'a> Deserializer<'a> {
 
     // TODO(#140): shouldn't buffer up this entire table in memory, it'd be
     // great to defer parsing everything until later.
-    fn inline_table(&mut self) -> Result<Vec<(Cow<'a, str>, Value<'a>)>, Error> {
+    fn inline_table(&mut self) -> Result<(Span, Vec<(Cow<'a, str>, Value<'a>)>), Error> {
         let mut ret = Vec::new();
         self.eat_whitespace()?;
-        if self.eat(Token::RightBrace)? {
+        if let Some(span) = self.eat_spanned(Token::RightBrace)? {
-            return Ok(ret)
+            return Ok((span, ret))
         }
         loop {
             let key = self.table_key()?;
@@ -1056,8 +1062,8 @@ impl<'a> Deserializer<'a> {
             ret.push((key, self.value()?));
 
             self.eat_whitespace()?;
-            if self.eat(Token::RightBrace)? {
+            if let Some(span) = self.eat_spanned(Token::RightBrace)? {
-                return Ok(ret)
+                return Ok((span, ret))
             }
             self.expect(Token::Comma)?;
             self.eat_whitespace()?;
@@ -1066,7 +1072,7 @@ impl<'a> Deserializer<'a> {
 
     // TODO(#140): shouldn't buffer up this entire array in memory, it'd be
     // great to defer parsing everything until later.
-    fn array(&mut self) -> Result<Vec<Value<'a>>, Error> {
+    fn array(&mut self) -> Result<(Span, Vec<Value<'a>>), Error> {
         let mut ret = Vec::new();
 
         let intermediate = |me: &mut Deserializer| {
@@ -1081,8 +1087,8 @@ impl<'a> Deserializer<'a> {
 
         loop {
             intermediate(self)?;
-            if self.eat(Token::RightBracket)? {
+            if let Some(span) = self.eat_spanned(Token::RightBracket)? {
-                return Ok(ret)
+                return Ok((span, ret))
             }
             let at = self.tokens.current();
             let value = self.value()?;
@@ -1098,8 +1104,8 @@ impl<'a> Deserializer<'a> {
             }
         }
         intermediate(self)?;
-        self.expect(Token::RightBracket)?;
+        let span = self.expect_spanned(Token::RightBracket)?;
-        Ok(ret)
+        Ok((span, ret))
     }
 
     fn table_key(&mut self) -> Result<Cow<'a, str>, Error> {
@@ -1122,10 +1128,18 @@ impl<'a> Deserializer<'a> {
         self.tokens.eat(expected).map_err(|e| self.token_error(e))
     }
 
+    fn eat_spanned(&mut self, expected: Token<'a>) -> Result<Option<Span>, Error> {
+        self.tokens.eat_spanned(expected).map_err(|e| self.token_error(e))
+    }
+
     fn expect(&mut self, expected: Token<'a>) -> Result<(), Error> {
         self.tokens.expect(expected).map_err(|e| self.token_error(e))
     }
 
+    fn expect_spanned(&mut self, expected: Token<'a>) -> Result<Span, Error> {
+        self.tokens.expect_spanned(expected).map_err(|e| self.token_error(e))
+    }
+
     fn next(&mut self) -> Result<Option<(Span, Token<'a>)>, Error> {
         self.tokens.next().map_err(|e| self.token_error(e))
     }

src/tokens.rs

@@ -118,21 +118,34 @@ impl<'a> Tokenizer<'a> {
     }
 
     pub fn eat(&mut self, expected: Token<'a>) -> Result<bool, Error> {
-        match self.peek()? {
+        self.eat_spanned(expected).map(|s| s.is_some())
-            Some((_, ref found)) if expected == *found => {}
-            Some(_) => return Ok(false),
-            None => return Ok(false),
     }
+
+    /// Eat a value, returning it's span if it was consumed.
+    pub fn eat_spanned(&mut self, expected: Token<'a>) -> Result<Option<Span>, Error> {
+        let span = match self.peek()? {
+            Some((span, ref found)) if expected == *found => span,
+            Some(_) => return Ok(None),
+            None => return Ok(None),
+        };
+
         drop(self.next());
-        Ok(true)
+        Ok(Some(span))
     }
 
     pub fn expect(&mut self, expected: Token<'a>) -> Result<(), Error> {
+        // ignore span
+        let _ = self.expect_spanned(expected)?;
+        Ok(())
+    }
+
+    /// Expect the given token returning its span.
+    pub fn expect_spanned(&mut self, expected: Token<'a>) -> Result<Span, Error> {
         let current = self.current();
         match self.next()? {
-            Some((_, found)) => {
+            Some((span, found)) => {
                 if expected == found {
-                    Ok(())
+                    Ok(span)
                 } else {
                     Err(Error::Wanted {
                         at: current,

test-suite/tests/spanned.rs

@@ -23,8 +23,21 @@ fn test_spanned_field() {
 
     good::<String>("foo = \"foo\"", "\"foo\"");
     good::<u32>("foo = 42", "42");
+    // leading plus
+    good::<u32>("foo = +42", "+42");
+    // table
     good::<HashMap<String, u32>>(
         "foo = {\"foo\" = 42, \"bar\" = 42}",
         "{\"foo\" = 42, \"bar\" = 42}"
     );
+    // array
+    good::<Vec<u32>>(
+        "foo = [0, 1, 2, 3, 4]",
+        "[0, 1, 2, 3, 4]"
+    );
+    // datetime
+    good::<String>(
+        "foo = \"1997-09-09T09:09:09Z\"",
+        "\"1997-09-09T09:09:09Z\""
+    );
 }