diff --git a/Changelog.md b/Changelog.md index f2528b0c..962e4e57 100644 --- a/Changelog.md +++ b/Changelog.md @@ -15,6 +15,10 @@ ### New Features +- [#806]: Perform normalization of line end during unescape process. + +[#806]: https://github.com/tafia/quick-xml/issues/806 + ### Bug Fixes ### Misc Changes diff --git a/benches/microbenches.rs b/benches/microbenches.rs index 2f4ece04..bf64f131 100644 --- a/benches/microbenches.rs +++ b/benches/microbenches.rs @@ -327,18 +327,18 @@ fn unescaping(c: &mut Criterion) { group.bench_function("mixed", |b| { let text = -"Lorem ipsum dolor sit amet, &consectetur adipiscing elit, sed do eiusmod tempor incididunt -ut labore et dolore magna aliqua. Hac habitasse platea dictumst vestibulum rhoncus est pellentesque. -Risus ultricies "tristique nulla aliquet enim tortor" at. Fermentum odio eu feugiat pretium -nibh ipsum. Volutpat sed cras ornare arcu dui. Scelerisque fermentum dui faucibus in ornare quam. Arcu -cursus euismod quis <viverra nibh cras pulvinar mattis. Sed viverra tellus in hac habitasse platea. +"Lorem ipsum dolor sit amet, &consectetur adipiscing elit, sed do eiusmod tempor incididunt\r +ut labore et dolore magna aliqua. Hac habitasse platea dictumst vestibulum rhoncus est pellentesque.\r +Risus ultricies "tristique nulla aliquet enim tortor" at. Fermentum odio eu feugiat pretium\r +nibh ipsum. Volutpat sed cras ornare arcu dui. Scelerisque fermentum dui faucibus in ornare quam. Arcu\r +cursus euismod quis <viverra nibh cras pulvinar mattis. Sed viverra tellus in hac habitasse platea.\r Quis commodo odio aenean sed. Cursus in hac habitasse platea dictumst quisque sagittis purus. -Neque convallis a cras semper auctor. Sit amet mauris commodo quis imperdiet massa. Ac ut consequat -semper viverra nam libero justo # laoreet sit. Adipiscing commodo elit at imperdiet dui accumsan. -Enim lobortis scelerisque fermentum dui faucibus in ornare. Natoque penatibus et magnis dis parturient -montes nascetur ridiculus mus. At lectus urna !duis convallis convallis tellus id interdum. Libero -volutpat sed cras ornare arcu dui vivamus arcu. Cursus in hac habitasse platea dictumst quisque sagittis +Neque convallis a cras semper auctor. Sit amet mauris commodo quis imperdiet massa. Ac ut consequat\r +semper viverra nam libero justo # laoreet sit. Adipiscing commodo elit at imperdiet dui accumsan.\r +Enim lobortis scelerisque fermentum dui faucibus in ornare. Natoque penatibus et magnis dis parturient\r +montes nascetur ridiculus mus. At lectus urna !duis convallis convallis tellus id interdum. Libero\r +volutpat sed cras ornare arcu dui vivamus arcu. Cursus in hac habitasse platea dictumst quisque sagittis\r purus. Consequat id porta nibh venenatis cras sed felis."; b.iter(|| { diff --git a/src/escape.rs b/src/escape.rs index 76c7f7e6..9f7a6439 100644 --- a/src/escape.rs +++ b/src/escape.rs @@ -1,6 +1,6 @@ //! Manage xml character escapes -use memchr::memchr2_iter; +use memchr::{memchr2_iter, memchr3_iter}; use std::borrow::Cow; use std::num::ParseIntError; use std::ops::Range; @@ -266,7 +266,9 @@ where unescaped = Some(String::with_capacity(raw.len())); } let unescaped = unescaped.as_mut().expect("initialized"); - unescaped.push_str(&raw[last_end..start]); + for normalized in normalize_line_end_iter(&raw[last_end..start]) { + unescaped.push_str(normalized); + } // search for character correctness let pat = &raw[start + 1..end]; @@ -290,14 +292,81 @@ where if let Some(mut unescaped) = unescaped { if let Some(raw) = raw.get(last_end..) { - unescaped.push_str(raw); + for normalized in normalize_line_end_iter(raw) { + unescaped.push_str(normalized); + } } Ok(Cow::Owned(unescaped)) } else { - Ok(Cow::Borrowed(raw)) + let mut norm_iter = normalize_line_end_iter(raw); + let first = norm_iter.next(); + match first { + Some(normalized) if normalized.len() != raw.len() => { + let mut s = String::with_capacity(raw.len()); + s.push_str(normalized); + for normalized in normalize_line_end_iter(raw) { + s.push_str(normalized); + } + Ok(s.into()) + } + _ => Ok(raw.into()), + } } } +/// Normalize the line end in input, replace \r or \r\n with \n, return iterator that can given normalized str. +/// link to [line end spec]https://www.w3.org/TR/xml11/#sec-line-ends somewhere near to the code that handles normalization also would be useful. Well, actually, it describes the 5 combinations of LOF characters, that should be normalized to \n: + +/// \r\n +/// \r\u0085 (UTF-8: \r\xC2\x85) +/// \r +/// \u0085 (UTF-8: \xC2\x85) +/// \u2028 (UTF-8: \xE2\x80\xA8) +/// +/// The reason to use iterator is to avoid allocation during normalizing line end. +/// If nothing to normalize of the input, it will give back whole input in the first iteration. Caller can know +/// there is nothing to normalize, so that it needs not to do normalization thus avoid allocation. +#[inline] +fn normalize_line_end_iter(input: &str) -> impl Iterator { + let bytes = input.as_bytes(); + let len = input.len(); + let mut cursor = 0; + let mut iter = memchr3_iter(b'\r', b'\xC2', b'\xE2', bytes); + let mut temp = None; + std::iter::from_fn(move || { + if let Some(v) = temp.take() { + return Some(v); + } + loop { + if let Some(p) = iter.next() { + if p < cursor { + // already normalized in previous iteration, this position is invalid + continue; + } + let skips = match &bytes[p..] { + [b'\r', b'\n', ..] => 2, + [b'\r', b'\xC2', b'\x85', ..] => 3, + [b'\r', ..] => 1, + [b'\xC2', b'\x85', ..] => 2, + [b'\xE2', b'\x80', b'\xA8', ..] => 3, + _ => continue, + }; + // normalized + temp = Some("\n"); + let start = cursor; + cursor = p + skips; + break Some(&input[start..p]); + } else if cursor < len { + let start = cursor; + cursor = len; + break Some(&input[start..]); + } else { + break None; + } + } + }) +} + /// Resolves predefined XML entities or all HTML5 entities depending on the feature /// [`escape-html`](https://docs.rs/quick-xml/latest/quick_xml/#escape-html). /// diff --git a/tests/escape.rs b/tests/escape.rs index 894c97c2..90066e00 100644 --- a/tests/escape.rs +++ b/tests/escape.rs @@ -75,6 +75,45 @@ fn unescape() { ); } +#[test] +fn unescape_line_end() { + let unchanged = escape::unescape("test\n"); + // assert_eq does not check that Cow is borrowed, but we explicitly use Cow + // because it influences diff + // TODO: use assert_matches! when stabilized and other features will bump MSRV + assert_eq!(unchanged, Ok(Cow::Borrowed("test\n"))); + assert!(matches!(unchanged, Ok(Cow::Borrowed(_)))); + + assert_eq!( + escape::unescape("<&test'\r"\r\n>\r\n\r"), + Ok("<&test'\n\"\n>\n\n".into()) + ); + assert_eq!(escape::unescape("0\r\r\n"), Ok("0\n\n".into())); + assert_eq!(escape::unescape("\r0\n\r\r"), Ok("\n0\n\n\n".into())); + assert_eq!( + escape::unescape("\r\n&foo;\n"), + Err(EscapeError::UnrecognizedEntity(3..6, "foo".into())) + ); + + assert_eq!( + escape::unescape("<&test'\u{0085}\r\r\u{0085}\u{2028}"\r\n>\r\n\r"), + Ok("<&test'\n\n\n\n\"\n>\n\n".into()) + ); + assert_eq!( + escape::unescape("0\r\r\n\u{0085}"), + Ok("0\n\n\n".into()) + ); + assert_eq!( + escape::unescape("\r0\n\r\r\u{2028}"), + Ok("\n0\n\n\n\n".into()) + ); + assert_eq!(escape::unescape("\r\r\u{0085}\n\n"), Ok("\n\n\n\n".into())); + assert_eq!( + escape::unescape("\r\n&foo;\n\u{2028}"), + Err(EscapeError::UnrecognizedEntity(3..6, "foo".into())) + ); +} + /// XML allows any number of leading zeroes. That is not explicitly mentioned /// in the specification, but enforced by the conformance test suite /// (https://www.w3.org/XML/Test/) diff --git a/tests/reader-attributes.rs b/tests/reader-attributes.rs index 8d51b22a..23c53491 100644 --- a/tests/reader-attributes.rs +++ b/tests/reader-attributes.rs @@ -1,7 +1,10 @@ use std::borrow::Cow; use quick_xml::events::attributes::Attribute; -use quick_xml::events::{BytesEnd, Event::*}; +use quick_xml::events::{ + BytesEnd, + Event::*, +}; use quick_xml::name::QName; use quick_xml::reader::Reader; @@ -159,3 +162,31 @@ fn equal_sign_in_value() { e => panic!("Expecting Empty event, got {:?}", e), } } + +#[test] +fn line_ends() { + const XML: &str = "\r\r\u{0085}\n\n\u{0085}\u{2028}value3\r\r\u{0085}\n\n\u{0085}\u{2028}value4\r\r\u{0085}\n\n\u{0085}\u{2028}"; + let mut reader = Reader::from_str(XML); + match reader.read_event().unwrap() { + Start(event) => { + let mut iter = event.attributes(); + let a = iter.next().unwrap().unwrap(); + #[cfg(not(feature = "encoding"))] + assert_eq!( + a.unescape_value().unwrap(), + "\n\n\n\n\n\nvalue1\n\n\n\n\n\nvalue2\n\n\n\n\n\n" + ); + assert_eq!( + a.decode_and_unescape_value(reader.decoder()).unwrap(), + "\n\n\n\n\n\nvalue1\n\n\n\n\n\nvalue2\n\n\n\n\n\n" + ); + } + event => panic!("Expected Start, found {:?}", event), + } + match reader.read_event().unwrap() { + Text(event) => { + assert_eq!(event.unescape().unwrap(), "\n\n\n\n\n\nvalue3\n\n\n\n\n\nvalue4\n\n\n\n\n\n") + } + event => panic!("Expected Text, found {:?}", event), + } +} diff --git a/tests/serde-se.rs b/tests/serde-se.rs index 39f6e66a..1ac36450 100644 --- a/tests/serde-se.rs +++ b/tests/serde-se.rs @@ -1955,9 +1955,10 @@ mod with_root { 3"); serialize_as!(tuple: // Use to_string() to get owned type that is required for deserialization - ("<\"&'>".to_string(), "with\t\r\n spaces", 3usize) + // Note: \r cannot include in whitespace character because we performs line end normalization. + ("<\"&'>".to_string(), "with\t\n spaces", 3usize) => "<\"&'>\ - with\t\r\n spaces\ + with\t\n spaces\ 3"); serialize_as!(tuple_struct: Tuple(42.0, "answer")