Merge pull request #201 from vitiral/triple

demonstrate tripple quote problem
This commit is contained in:
Alex Crichton 2017-07-28 15:18:18 -05:00 committed by GitHub
commit 29578669f2
3 changed files with 244 additions and 65 deletions

View file

@ -201,6 +201,7 @@ enum State<'a> {
parent: &'a State<'a>, parent: &'a State<'a>,
first: &'a Cell<bool>, first: &'a Cell<bool>,
type_: &'a Cell<Option<&'static str>>, type_: &'a Cell<Option<&'static str>>,
len: Option<usize>,
}, },
End, End,
} }
@ -210,6 +211,7 @@ pub struct SerializeSeq<'a: 'b, 'b> {
ser: &'b mut Serializer<'a>, ser: &'b mut Serializer<'a>,
first: Cell<bool>, first: Cell<bool>,
type_: Cell<Option<&'static str>>, type_: Cell<Option<&'static str>>,
len: Option<usize>,
} }
#[doc(hidden)] #[doc(hidden)]
@ -377,12 +379,12 @@ impl<'a> Serializer<'a> {
fn _emit_key(&mut self, state: &State) -> Result<(), Error> { fn _emit_key(&mut self, state: &State) -> Result<(), Error> {
match *state { match *state {
State::End => Ok(()), State::End => Ok(()),
State::Array { parent, first, type_ } => { State::Array { parent, first, type_, len } => {
assert!(type_.get().is_some()); assert!(type_.get().is_some());
if first.get() { if first.get() {
self._emit_key(parent)?; self._emit_key(parent)?;
} }
self.emit_array(first) self.emit_array(first, len)
} }
State::Table { parent, first, table_emitted, key } => { State::Table { parent, first, table_emitted, key } => {
if table_emitted.get() { if table_emitted.get() {
@ -399,9 +401,16 @@ impl<'a> Serializer<'a> {
} }
} }
fn emit_array(&mut self, first: &Cell<bool>) -> Result<(), Error> { fn emit_array(&mut self, first: &Cell<bool>, len: Option<usize>) -> Result<(), Error> {
match self.settings.array { match (len, &self.settings.array) {
Some(ref a) => { (Some(0...1), _) | (_, &None) => {
if first.get() {
self.dst.push_str("[")
} else {
self.dst.push_str(", ")
}
},
(_, &Some(ref a)) => {
if first.get() { if first.get() {
self.dst.push_str("[\n") self.dst.push_str("[\n")
} else { } else {
@ -411,13 +420,6 @@ impl<'a> Serializer<'a> {
self.dst.push_str(" "); self.dst.push_str(" ");
} }
}, },
None => {
if first.get() {
self.dst.push_str("[")
} else {
self.dst.push_str(", ")
}
},
} }
Ok(()) Ok(())
} }
@ -450,53 +452,137 @@ impl<'a> Serializer<'a> {
if ok { if ok {
drop(write!(self.dst, "{}", key)); drop(write!(self.dst, "{}", key));
} else { } else {
self.emit_str(key)?; self.emit_str(key, true)?;
} }
Ok(()) Ok(())
} }
fn emit_str(&mut self, value: &str) -> Result<(), Error> { fn emit_str(&mut self, value: &str, is_key: bool) -> Result<(), Error> {
let do_pretty = if self.settings.pretty_string { #[derive(PartialEq)]
value.contains('\n') enum Type {
NewlineTripple,
OnelineTripple,
OnelineSingle,
}
enum Repr {
/// represent as a literal string (using '')
Literal(String, Type),
/// represent the std way (using "")
Std(Type),
}
fn do_pretty(value: &str) -> Repr {
// For doing pretty prints we store in a new String
// because there are too many cases where pretty cannot
// work. We need to determine:
// - if we are a "multi-line" pretty (if there are \n)
// - if ['''] appears if multi or ['] if single
// - if there are any invalid control characters
//
// Doing it any other way would require multiple passes
// to determine if a pretty string works or not.
let mut out = String::with_capacity(value.len() * 2);
let mut ty = Type::OnelineSingle;
// found consecutive single quotes
let mut max_found_singles = 0;
let mut found_singles = 0;
let mut can_be_pretty = true;
for ch in value.chars() {
if can_be_pretty {
if ch == '\'' {
found_singles += 1;
if found_singles >= 3 {
can_be_pretty = false;
}
} else { } else {
false if found_singles > max_found_singles {
max_found_singles = found_singles;
}
found_singles = 0
}
match ch {
'\t' => {},
'\n' => ty = Type::NewlineTripple,
// note that the following are invalid: \b \f \r
c if c < '\u{1f}' => can_be_pretty = false, // Invalid control character
_ => {}
}
out.push(ch);
} else {
// the string cannot be represented as pretty,
// still check if it should be multiline
if ch == '\n' {
ty = Type::NewlineTripple;
}
}
}
if !can_be_pretty {
debug_assert!(ty != Type::OnelineTripple);
return Repr::Std(ty);
}
if found_singles > max_found_singles {
max_found_singles = found_singles;
}
debug_assert!(max_found_singles < 3);
if ty == Type::OnelineSingle && max_found_singles >= 1 {
// no newlines, but must use ''' because it has ' in it
ty = Type::OnelineTripple;
}
Repr::Literal(out, ty)
}
let repr = if !is_key && self.settings.pretty_string {
do_pretty(value)
} else {
Repr::Std(Type::OnelineSingle)
}; };
if do_pretty { match repr {
drop(write!(self.dst, "'''\n")); Repr::Literal(literal, ty) => {
} else { // A pretty string
drop(write!(self.dst, "\"")); match ty {
Type::NewlineTripple => self.dst.push_str("'''\n"),
Type::OnelineTripple => self.dst.push_str("'''"),
Type::OnelineSingle => self.dst.push('\''),
}
self.dst.push_str(&literal);
match ty {
Type::OnelineSingle => self.dst.push('\''),
_ => self.dst.push_str("'''"),
}
},
Repr::Std(ty) => {
match ty {
Type::NewlineTripple => self.dst.push_str("\"\"\"\n"),
Type::OnelineSingle => self.dst.push('"'),
_ => unreachable!(),
} }
for ch in value.chars() { for ch in value.chars() {
match ch { match ch {
'\u{8}' => drop(write!(self.dst, "\\b")), '\u{8}' => self.dst.push_str("\\b"),
'\u{9}' => drop(write!(self.dst, "\\t")), '\u{9}' => self.dst.push_str("\\t"),
'\u{a}' => { '\u{a}' => {
if do_pretty { match ty {
drop(write!(self.dst, "\n")); Type::NewlineTripple => self.dst.push('\n'),
} else { Type::OnelineSingle => self.dst.push_str("\\n"),
drop(write!(self.dst, "\\n")); _ => unreachable!(),
} }
}, },
'\u{c}' => drop(write!(self.dst, "\\f")), '\u{c}' => self.dst.push_str("\\f"),
'\u{d}' => drop(write!(self.dst, "\\r")), '\u{d}' => self.dst.push_str("\\r"),
'\u{22}' => { '\u{22}' => self.dst.push_str("\\\""),
if do_pretty { '\u{5c}' => self.dst.push_str("\\\\"),
drop(write!(self.dst, "\"")) c if c < '\u{1f}' => drop(write!(self.dst, "\\u{:04X}", ch as u32)),
} else { ch => self.dst.push(ch),
drop(write!(self.dst, "\\\"")) }
}
match ty {
Type::NewlineTripple => self.dst.push_str("\"\"\""),
Type::OnelineSingle => self.dst.push('"'),
_ => unreachable!(),
} }
}, },
'\u{5c}' => drop(write!(self.dst, "\\\\")),
c if c < '\u{1f}' => {
drop(write!(self.dst, "\\u{:04X}", ch as u32))
}
ch => drop(write!(self.dst, "{}", ch)),
}
}
if do_pretty {
drop(write!(self.dst, "'''"));
} else {
drop(write!(self.dst, "\""));
} }
Ok(()) Ok(())
} }
@ -651,7 +737,7 @@ impl<'a, 'b> ser::Serializer for &'b mut Serializer<'a> {
fn serialize_str(mut self, value: &str) -> Result<(), Self::Error> { fn serialize_str(mut self, value: &str) -> Result<(), Self::Error> {
self.emit_key("string")?; self.emit_key("string")?;
self.emit_str(value)?; self.emit_str(value, false)?;
if let State::Table { .. } = self.state { if let State::Table { .. } = self.state {
self.dst.push_str("\n"); self.dst.push_str("\n");
} }
@ -709,13 +795,14 @@ impl<'a, 'b> ser::Serializer for &'b mut Serializer<'a> {
Err(Error::UnsupportedType) Err(Error::UnsupportedType)
} }
fn serialize_seq(mut self, _len: Option<usize>) fn serialize_seq(mut self, len: Option<usize>)
-> Result<Self::SerializeSeq, Self::Error> { -> Result<Self::SerializeSeq, Self::Error> {
self.array_type("array")?; self.array_type("array")?;
Ok(SerializeSeq { Ok(SerializeSeq {
ser: self, ser: self,
first: Cell::new(true), first: Cell::new(true),
type_: Cell::new(None), type_: Cell::new(None),
len: len,
}) })
} }
@ -788,6 +875,7 @@ impl<'a, 'b> ser::SerializeSeq for SerializeSeq<'a, 'b> {
parent: &self.ser.state, parent: &self.ser.state,
first: &self.first, first: &self.first,
type_: &self.type_, type_: &self.type_,
len: self.len,
}, },
settings: self.ser.settings.clone(), settings: self.ser.settings.clone(),
})?; })?;
@ -799,16 +887,16 @@ impl<'a, 'b> ser::SerializeSeq for SerializeSeq<'a, 'b> {
match self.type_.get() { match self.type_.get() {
Some("table") => return Ok(()), Some("table") => return Ok(()),
Some(_) => { Some(_) => {
match self.ser.settings.array { match (self.len, &self.ser.settings.array) {
Some(ref a) => { (Some(0...1), _) | (_, &None) => {
self.ser.dst.push_str("]");
},
(_, &Some(ref a)) => {
if a.trailing_comma { if a.trailing_comma {
self.ser.dst.push_str(","); self.ser.dst.push_str(",");
} }
self.ser.dst.push_str("\n]"); self.ser.dst.push_str("\n]");
}, },
None => {
self.ser.dst.push_str("]");
}
} }
} }
None => { None => {

View file

@ -41,11 +41,12 @@ fn disable_pretty() {
const PRETTY_STD: &'static str = "\ const PRETTY_STD: &'static str = "\
[example] [example]
array = [ array = [
\"item 1\", 'item 1',
\"item 2\", 'item 2',
] ]
empty = [] empty = []
oneline = \"this has no newlines.\" one = ['one']
oneline = 'this has no newlines.'
text = ''' text = '''
this is the first line this is the first line
this is the second line this is the second line
@ -67,15 +68,21 @@ fn pretty_std() {
const PRETTY_INDENT_2: &'static str = "\ const PRETTY_INDENT_2: &'static str = "\
[example] [example]
array = [ array = [
\"item 1\", 'item 1',
\"item 2\", 'item 2',
] ]
empty = [] empty = []
oneline = \"this has no newlines.\" one = ['one']
oneline = 'this has no newlines.'
text = ''' text = '''
this is the first line this is the first line
this is the second line this is the second line
''' '''
three = [
'one',
'two',
'three',
]
"; ";
#[test] #[test]
@ -88,6 +95,7 @@ fn pretty_indent_2() {
serializer.pretty_array_indent(2); serializer.pretty_array_indent(2);
value.serialize(&mut serializer).unwrap(); value.serialize(&mut serializer).unwrap();
} }
println!(">> Result:\n{}", result);
assert_eq!(toml, &result); assert_eq!(toml, &result);
} }
@ -166,3 +174,41 @@ fn pretty_no_string() {
} }
assert_eq!(toml, &result); assert_eq!(toml, &result);
} }
const PRETTY_TRICKY: &'static str = r##"[example]
f = "\f"
glass = '''
Nothing too unusual, except that I can eat glass in:
- Greek: Μπορώ να φάω σπασμένα γυαλιά χωρίς να πάθω τίποτα.
- Polish: Mogę jeść szkło, i mi nie szkodzi.
- Hindi: , .
- Japanese:
'''
r = "\r"
r_newline = """
\r
"""
single = '''this is a single line but has '' cuz it's tricky'''
single_tricky = "single line with ''' in it"
tabs = '''
this is pretty standard
except for some tabs right here
'''
text = """
this is the first line.
This has a ''' in it and \"\"\" cuz it's tricky yo
Also ' and \" because why not
this is the third line
"""
"##;
#[test]
fn pretty_tricky() {
let toml = PRETTY_TRICKY;
let value: toml::Value = toml::from_str(toml).unwrap();
let mut result = String::with_capacity(128);
value.serialize(&mut toml::Serializer::pretty(&mut result)).unwrap();
println!("EXPECTED:\n{}", toml);
println!("\nRESULT:\n{}", result);
assert_eq!(toml, &result);
}

View file

@ -1,7 +1,9 @@
extern crate toml; extern crate toml;
extern crate serde;
extern crate serde_json; extern crate serde_json;
use toml::Value as Toml; use toml::{Value as Toml, to_string_pretty};
use serde::ser::Serialize;
use serde_json::Value as Json; use serde_json::Value as Json;
fn to_json(toml: toml::Value) -> Json { fn to_json(toml: toml::Value) -> Json {
@ -40,10 +42,52 @@ fn to_json(toml: toml::Value) -> Json {
} }
} }
fn run(toml: &str, json: &str) { fn run_pretty(toml: Toml) {
println!("parsing:\n{}", toml); // Assert toml == json
let toml: Toml = toml.parse().unwrap(); println!("### pretty round trip parse.");
let json: Json = json.parse().unwrap();
// standard pretty
let toml_raw = to_string_pretty(&toml).expect("to string");
let toml2 = toml_raw.parse().expect("from string");
assert_eq!(toml, toml2);
// pretty with indent 2
let mut result = String::with_capacity(128);
{
let mut serializer = toml::Serializer::pretty(&mut result);
serializer.pretty_array_indent(2);
toml.serialize(&mut serializer).expect("to string");
}
assert_eq!(toml, result.parse().expect("from str"));
result.clear();
{
let mut serializer = toml::Serializer::new(&mut result);
serializer.pretty_array_trailing_comma(false);
toml.serialize(&mut serializer).expect("to string");
}
assert_eq!(toml, result.parse().expect("from str"));
result.clear();
{
let mut serializer = toml::Serializer::pretty(&mut result);
serializer.pretty_string(false);
toml.serialize(&mut serializer).expect("to string");
assert_eq!(toml, toml2);
}
assert_eq!(toml, result.parse().expect("from str"));
result.clear();
{
let mut serializer = toml::Serializer::pretty(&mut result);
serializer.pretty_array(false);
toml.serialize(&mut serializer).expect("to string");
assert_eq!(toml, toml2);
}
assert_eq!(toml, result.parse().expect("from str"));
}
fn run(toml_raw: &str, json_raw: &str) {
println!("parsing:\n{}", toml_raw);
let toml: Toml = toml_raw.parse().unwrap();
let json: Json = json_raw.parse().unwrap();
// Assert toml == json // Assert toml == json
let toml_json = to_json(toml.clone()); let toml_json = to_json(toml.clone());
@ -56,6 +100,7 @@ fn run(toml: &str, json: &str) {
println!("round trip parse: {}", toml); println!("round trip parse: {}", toml);
let toml2 = toml.to_string().parse().unwrap(); let toml2 = toml.to_string().parse().unwrap();
assert_eq!(toml, toml2); assert_eq!(toml, toml2);
run_pretty(toml);
} }
macro_rules! test( ($name:ident, $toml:expr, $json:expr) => ( macro_rules! test( ($name:ident, $toml:expr, $json:expr) => (