Import xml-rs 0.8.3
Bug: 191601801
Test: m libxml
Change-Id: Ic95f410632d24503e78b3f57543d0cdb13886f2f
diff --git a/src/reader/config.rs b/src/reader/config.rs
new file mode 100644
index 0000000..5b4cd32
--- /dev/null
+++ b/src/reader/config.rs
@@ -0,0 +1,181 @@
+//! Contains parser configuration structure.
+use std::io::Read;
+use std::collections::HashMap;
+
+use reader::EventReader;
+
+/// Parser configuration structure.
+///
+/// This structure contains various configuration options which affect
+/// behavior of the parser.
+#[derive(Clone, PartialEq, Eq, Debug)]
+pub struct ParserConfig {
+ /// Whether or not should whitespace in textual events be removed. Default is false.
+ ///
+ /// When true, all standalone whitespace will be removed (this means no
+ /// `Whitespace` events will ve emitted), and leading and trailing whitespace
+ /// from `Character` events will be deleted. If after trimming `Characters`
+ /// event will be empty, it will also be omitted from output stream. This is
+ /// possible, however, only if `whitespace_to_characters` or
+ /// `cdata_to_characters` options are set.
+ ///
+ /// This option does not affect CDATA events, unless `cdata_to_characters`
+ /// option is also set. In that case CDATA content will also be trimmed.
+ pub trim_whitespace: bool,
+
+ /// Whether or not should whitespace be converted to characters.
+ /// Default is false.
+ ///
+ /// If true, instead of `Whitespace` events `Characters` events with the
+ /// same content will be emitted. If `trim_whitespace` is also true, these
+ /// events will be trimmed to nothing and, consequently, not emitted.
+ pub whitespace_to_characters: bool,
+
+ /// Whether or not should CDATA be converted to characters.
+ /// Default is false.
+ ///
+ /// If true, instead of `CData` events `Characters` events with the same
+ /// content will be emitted. If `trim_whitespace` is also true, these events
+ /// will be trimmed. If corresponding CDATA contained nothing but whitespace,
+ /// this event will be omitted from the stream.
+ pub cdata_to_characters: bool,
+
+ /// Whether or not should comments be omitted. Default is true.
+ ///
+ /// If true, `Comment` events will not be emitted at all.
+ pub ignore_comments: bool,
+
+ /// Whether or not should sequential `Characters` events be merged.
+ /// Default is true.
+ ///
+ /// If true, multiple sequential `Characters` events will be merged into
+ /// a single event, that is, their data will be concatenated.
+ ///
+ /// Multiple sequential `Characters` events are only possible if either
+ /// `cdata_to_characters` or `ignore_comments` are set. Otherwise character
+ /// events will always be separated by other events.
+ pub coalesce_characters: bool,
+
+ /// A map of extra entities recognized by the parser. Default is an empty map.
+ ///
+ /// By default the XML parser recognizes the entities defined in the XML spec. Sometimes,
+ /// however, it is convenient to make the parser recognize additional entities which
+ /// are also not available through the DTD definitions (especially given that at the moment
+ /// DTD parsing is not supported).
+ pub extra_entities: HashMap<String, String>,
+
+ /// Whether or not the parser should ignore the end of stream. Default is false.
+ ///
+ /// By default the parser will either error out when it encounters a premature end of
+ /// stream or complete normally if the end of stream was expected. If you want to continue
+ /// reading from a stream whose input is supplied progressively, you can set this option to true.
+ /// In this case the parser will allow you to invoke the next() method even if a supposed end
+ /// of stream has happened.
+ ///
+ /// Note that support for this functionality is incomplete; for example, the parser will fail if
+ /// the premature end of stream happens inside PCDATA. Therefore, use this option at your own risk.
+ pub ignore_end_of_stream: bool,
+
+ /// Whether or not non-unicode entity references get replaced with the replacement character
+ ///
+ /// When true, any decimal or hexadecimal character reference that cannot be converted from a
+ /// u32 to a char using [std::char::from_u32](https://doc.rust-lang.org/std/char/fn.from_u32.html)
+ /// will be converted into the unicode REPLACEMENT CHARACTER (U+FFFD).
+ pub replace_unknown_entity_references: bool,
+
+ /// Whether or not whitespace at the root level of the document is ignored. Default is true.
+ ///
+ /// By default any whitespace that is not enclosed within at least one level of elements will be
+ /// ignored. Setting this value to false will cause root level whitespace events to be emitted.
+ pub ignore_root_level_whitespace: bool,
+}
+
+impl ParserConfig {
+ /// Returns a new config with default values.
+ ///
+ /// You can tweak default values using builder-like pattern:
+ ///
+ /// ```rust
+ /// use xml::reader::ParserConfig;
+ ///
+ /// let config = ParserConfig::new()
+ /// .trim_whitespace(true)
+ /// .ignore_comments(true)
+ /// .coalesce_characters(false);
+ /// ```
+ pub fn new() -> ParserConfig {
+ ParserConfig {
+ trim_whitespace: false,
+ whitespace_to_characters: false,
+ cdata_to_characters: false,
+ ignore_comments: true,
+ coalesce_characters: true,
+ extra_entities: HashMap::new(),
+ ignore_end_of_stream: false,
+ replace_unknown_entity_references: false,
+ ignore_root_level_whitespace: true,
+ }
+ }
+
+ /// Creates an XML reader with this configuration.
+ ///
+ /// This is a convenience method for configuring and creating a reader at the same time:
+ ///
+ /// ```rust
+ /// use xml::reader::ParserConfig;
+ ///
+ /// let mut source: &[u8] = b"...";
+ ///
+ /// let reader = ParserConfig::new()
+ /// .trim_whitespace(true)
+ /// .ignore_comments(true)
+ /// .coalesce_characters(false)
+ /// .create_reader(&mut source);
+ /// ```
+ ///
+ /// This method is exactly equivalent to calling `EventReader::new_with_config()` with
+ /// this configuration object.
+ #[inline]
+ pub fn create_reader<R: Read>(self, source: R) -> EventReader<R> {
+ EventReader::new_with_config(source, self)
+ }
+
+ /// Adds a new entity mapping and returns an updated config object.
+ ///
+ /// This is a convenience method for adding external entities mappings to the XML parser.
+ /// An example:
+ ///
+ /// ```rust
+ /// use xml::reader::ParserConfig;
+ ///
+ /// let mut source: &[u8] = b"...";
+ ///
+ /// let reader = ParserConfig::new()
+ /// .add_entity("nbsp", " ")
+ /// .add_entity("copy", "©")
+ /// .add_entity("reg", "®")
+ /// .create_reader(&mut source);
+ /// ```
+ pub fn add_entity<S: Into<String>, T: Into<String>>(mut self, entity: S, value: T) -> ParserConfig {
+ self.extra_entities.insert(entity.into(), value.into());
+ self
+ }
+}
+
+impl Default for ParserConfig {
+ #[inline]
+ fn default() -> ParserConfig {
+ ParserConfig::new()
+ }
+}
+
+gen_setters! { ParserConfig,
+ trim_whitespace: val bool,
+ whitespace_to_characters: val bool,
+ cdata_to_characters: val bool,
+ ignore_comments: val bool,
+ coalesce_characters: val bool,
+ ignore_end_of_stream: val bool,
+ replace_unknown_entity_references: val bool,
+ ignore_root_level_whitespace: val bool
+}
diff --git a/src/reader/error.rs b/src/reader/error.rs
new file mode 100644
index 0000000..92378e6
--- /dev/null
+++ b/src/reader/error.rs
@@ -0,0 +1,121 @@
+
+use std::io;
+use std::borrow::Cow;
+use std::fmt;
+use std::error;
+use std::str;
+
+use util;
+use common::{Position, TextPosition};
+
+#[derive(Debug)]
+pub enum ErrorKind {
+ Syntax(Cow<'static, str>),
+ Io(io::Error),
+ Utf8(str::Utf8Error),
+ UnexpectedEof,
+}
+
+/// An XML parsing error.
+///
+/// Consists of a 2D position in a document and a textual message describing the error.
+#[derive(Clone, PartialEq, Eq, Debug)]
+pub struct Error {
+ pos: TextPosition,
+ kind: ErrorKind,
+}
+
+impl fmt::Display for Error {
+ fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+ write!(f, "{} {}", self.pos, self.msg())
+ }
+}
+
+impl Position for Error {
+ #[inline]
+ fn position(&self) -> TextPosition { self.pos }
+}
+
+impl Error {
+ /// Returns a reference to a message which is contained inside this error.
+ #[inline]
+ pub fn msg(&self) -> &str {
+ use self::ErrorKind::*;
+ match self.kind {
+ UnexpectedEof => &"Unexpected EOF",
+ Utf8(ref reason) => error_description(reason),
+ Io(ref io_error) => error_description(io_error),
+ Syntax(ref msg) => msg.as_ref(),
+ }
+ }
+
+ pub fn kind(&self) -> &ErrorKind { &self.kind }
+}
+
+impl error::Error for Error {
+ #[inline]
+ fn description(&self) -> &str { self.msg() }
+}
+
+impl<'a, P, M> From<(&'a P, M)> for Error where P: Position, M: Into<Cow<'static, str>> {
+ fn from(orig: (&'a P, M)) -> Self {
+ Error{
+ pos: orig.0.position(),
+ kind: ErrorKind::Syntax(orig.1.into())
+ }
+ }
+}
+
+impl From<util::CharReadError> for Error {
+ fn from(e: util::CharReadError) -> Self {
+ use util::CharReadError::*;
+ Error{
+ pos: TextPosition::new(),
+ kind: match e {
+ UnexpectedEof => ErrorKind::UnexpectedEof,
+ Utf8(reason) => ErrorKind::Utf8(reason),
+ Io(io_error) => ErrorKind::Io(io_error),
+ }
+ }
+ }
+}
+
+impl From<io::Error> for Error {
+ fn from(e: io::Error) -> Self {
+ Error {
+ pos: TextPosition::new(),
+ kind: ErrorKind::Io(e),
+ }
+ }
+}
+
+impl Clone for ErrorKind {
+ fn clone(&self) -> Self {
+ use self::ErrorKind::*;
+ match *self {
+ UnexpectedEof => UnexpectedEof,
+ Utf8(ref reason) => Utf8(reason.clone()),
+ Io(ref io_error) => Io(io::Error::new(io_error.kind(), error_description(io_error))),
+ Syntax(ref msg) => Syntax(msg.clone()),
+ }
+ }
+}
+impl PartialEq for ErrorKind {
+ fn eq(&self, other: &ErrorKind) -> bool {
+ use self::ErrorKind::*;
+ match (self, other) {
+ (&UnexpectedEof, &UnexpectedEof) => true,
+ (&Utf8(ref left), &Utf8(ref right)) => left == right,
+ (&Io(ref left), &Io(ref right)) =>
+ left.kind() == right.kind() &&
+ error_description(left) == error_description(right),
+ (&Syntax(ref left), &Syntax(ref right)) =>
+ left == right,
+
+ (_, _) => false,
+ }
+ }
+}
+impl Eq for ErrorKind {}
+
+fn error_description(e: &error::Error) -> &str { e.description() }
diff --git a/src/reader/events.rs b/src/reader/events.rs
new file mode 100644
index 0000000..46d7621
--- /dev/null
+++ b/src/reader/events.rs
@@ -0,0 +1,219 @@
+//! Contains `XmlEvent` datatype, instances of which are emitted by the parser.
+
+use std::fmt;
+use std::borrow::Cow;
+
+use name::OwnedName;
+use attribute::OwnedAttribute;
+use common::XmlVersion;
+use namespace::Namespace;
+
+/// An element of an XML input stream.
+///
+/// Items of this enum are emitted by `reader::EventReader`. They correspond to different
+/// elements of an XML document.
+#[derive(PartialEq, Clone)]
+pub enum XmlEvent {
+ /// Corresponds to XML document declaration.
+ ///
+ /// This event is always emitted before any other event. It is emitted
+ /// even if the actual declaration is not present in the document.
+ StartDocument {
+ /// XML version.
+ ///
+ /// If XML declaration is not present, defaults to `Version10`.
+ version: XmlVersion,
+
+ /// XML document encoding.
+ ///
+ /// If XML declaration is not present or does not contain `encoding` attribute,
+ /// defaults to `"UTF-8"`. This field is currently used for no other purpose than
+ /// informational.
+ encoding: String,
+
+ /// XML standalone declaration.
+ ///
+ /// If XML document is not present or does not contain `standalone` attribute,
+ /// defaults to `None`. This field is currently used for no other purpose than
+ /// informational.
+ standalone: Option<bool>
+ },
+
+ /// Denotes to the end of the document stream.
+ ///
+ /// This event is always emitted after any other event (except `Error`). After it
+ /// is emitted for the first time, it will always be emitted on next event pull attempts.
+ EndDocument,
+
+ /// Denotes an XML processing instruction.
+ ///
+ /// This event contains a processing instruction target (`name`) and opaque `data`. It
+ /// is up to the application to process them.
+ ProcessingInstruction {
+ /// Processing instruction target.
+ name: String,
+
+ /// Processing instruction content.
+ data: Option<String>
+ },
+
+ /// Denotes a beginning of an XML element.
+ ///
+ /// This event is emitted after parsing opening tags or after parsing bodiless tags. In the
+ /// latter case `EndElement` event immediately follows.
+ StartElement {
+ /// Qualified name of the element.
+ name: OwnedName,
+
+ /// A list of attributes associated with the element.
+ ///
+ /// Currently attributes are not checked for duplicates (TODO)
+ attributes: Vec<OwnedAttribute>,
+
+ /// Contents of the namespace mapping at this point of the document.
+ namespace: Namespace,
+ },
+
+ /// Denotes an end of an XML element.
+ ///
+ /// This event is emitted after parsing closing tags or after parsing bodiless tags. In the
+ /// latter case it is emitted immediately after corresponding `StartElement` event.
+ EndElement {
+ /// Qualified name of the element.
+ name: OwnedName
+ },
+
+ /// Denotes CDATA content.
+ ///
+ /// This event contains unparsed data. No unescaping will be performed.
+ ///
+ /// It is possible to configure a parser to emit `Characters` event instead of `CData`. See
+ /// `pull::ParserConfiguration` structure for more information.
+ CData(String),
+
+ /// Denotes a comment.
+ ///
+ /// It is possible to configure a parser to ignore comments, so this event will never be emitted.
+ /// See `pull::ParserConfiguration` structure for more information.
+ Comment(String),
+
+ /// Denotes character data outside of tags.
+ ///
+ /// Contents of this event will always be unescaped, so no entities like `<` or `&` or `{`
+ /// will appear in it.
+ ///
+ /// It is possible to configure a parser to trim leading and trailing whitespace for this event.
+ /// See `pull::ParserConfiguration` structure for more information.
+ Characters(String),
+
+ /// Denotes a chunk of whitespace outside of tags.
+ ///
+ /// It is possible to configure a parser to emit `Characters` event instead of `Whitespace`.
+ /// See `pull::ParserConfiguration` structure for more information. When combined with whitespace
+ /// trimming, it will eliminate standalone whitespace from the event stream completely.
+ Whitespace(String)
+}
+
+impl fmt::Debug for XmlEvent {
+ fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+ match *self {
+ XmlEvent::StartDocument { ref version, ref encoding, ref standalone } =>
+ write!(f, "StartDocument({}, {}, {:?})", version, *encoding, *standalone),
+ XmlEvent::EndDocument =>
+ write!(f, "EndDocument"),
+ XmlEvent::ProcessingInstruction { ref name, ref data } =>
+ write!(f, "ProcessingInstruction({}{})", *name, match *data {
+ Some(ref data) => format!(", {}", data),
+ None => String::new()
+ }),
+ XmlEvent::StartElement { ref name, ref attributes, namespace: Namespace(ref namespace) } =>
+ write!(f, "StartElement({}, {:?}{})", name, namespace, if attributes.is_empty() {
+ String::new()
+ } else {
+ let attributes: Vec<String> = attributes.iter().map(
+ |a| format!("{} -> {}", a.name, a.value)
+ ).collect();
+ format!(", [{}]", attributes.join(", "))
+ }),
+ XmlEvent::EndElement { ref name } =>
+ write!(f, "EndElement({})", name),
+ XmlEvent::Comment(ref data) =>
+ write!(f, "Comment({})", data),
+ XmlEvent::CData(ref data) =>
+ write!(f, "CData({})", data),
+ XmlEvent::Characters(ref data) =>
+ write!(f, "Characters({})", data),
+ XmlEvent::Whitespace(ref data) =>
+ write!(f, "Whitespace({})", data)
+ }
+ }
+}
+
+impl XmlEvent {
+ /// Obtains a writer event from this reader event.
+ ///
+ /// This method is useful for streaming processing of XML documents where the output
+ /// is also an XML document. With this method it is possible to process some events
+ /// while passing other events through to the writer unchanged:
+ ///
+ /// ```rust
+ /// use std::str;
+ ///
+ /// use xml::{EventReader, EventWriter};
+ /// use xml::reader::XmlEvent as ReaderEvent;
+ /// use xml::writer::XmlEvent as WriterEvent;
+ ///
+ /// let mut input: &[u8] = b"<hello>world</hello>";
+ /// let mut output: Vec<u8> = Vec::new();
+ ///
+ /// {
+ /// let mut reader = EventReader::new(&mut input);
+ /// let mut writer = EventWriter::new(&mut output);
+ ///
+ /// for e in reader {
+ /// match e.unwrap() {
+ /// ReaderEvent::Characters(s) =>
+ /// writer.write(WriterEvent::characters(&s.to_uppercase())).unwrap(),
+ /// e => if let Some(e) = e.as_writer_event() {
+ /// writer.write(e).unwrap()
+ /// }
+ /// }
+ /// }
+ /// }
+ ///
+ /// assert_eq!(
+ /// str::from_utf8(&output).unwrap(),
+ /// r#"<?xml version="1.0" encoding="UTF-8"?><hello>WORLD</hello>"#
+ /// );
+ /// ```
+ ///
+ /// Note that this API may change or get additions in future to improve its ergonomics.
+ pub fn as_writer_event<'a>(&'a self) -> Option<::writer::events::XmlEvent<'a>> {
+ match *self {
+ XmlEvent::StartDocument { version, ref encoding, standalone } =>
+ Some(::writer::events::XmlEvent::StartDocument {
+ version: version,
+ encoding: Some(encoding),
+ standalone: standalone
+ }),
+ XmlEvent::ProcessingInstruction { ref name, ref data } =>
+ Some(::writer::events::XmlEvent::ProcessingInstruction {
+ name: name,
+ data: data.as_ref().map(|s| &s[..])
+ }),
+ XmlEvent::StartElement { ref name, ref attributes, ref namespace } =>
+ Some(::writer::events::XmlEvent::StartElement {
+ name: name.borrow(),
+ attributes: attributes.iter().map(|a| a.borrow()).collect(),
+ namespace: Cow::Borrowed(namespace)
+ }),
+ XmlEvent::EndElement { ref name } =>
+ Some(::writer::events::XmlEvent::EndElement { name: Some(name.borrow()) }),
+ XmlEvent::Comment(ref data) => Some(::writer::events::XmlEvent::Comment(data)),
+ XmlEvent::CData(ref data) => Some(::writer::events::XmlEvent::CData(data)),
+ XmlEvent::Characters(ref data) => Some(::writer::events::XmlEvent::Characters(data)),
+ XmlEvent::Whitespace(ref data) => Some(::writer::events::XmlEvent::Characters(data)),
+ _ => None
+ }
+ }
+}
diff --git a/src/reader/lexer.rs b/src/reader/lexer.rs
new file mode 100644
index 0000000..ffaab57
--- /dev/null
+++ b/src/reader/lexer.rs
@@ -0,0 +1,866 @@
+//! Contains simple lexer for XML documents.
+//!
+//! This module is for internal use. Use `xml::pull` module to do parsing.
+
+use std::fmt;
+use std::collections::VecDeque;
+use std::io::Read;
+use std::result;
+use std::borrow::Cow;
+
+use common::{Position, TextPosition, is_whitespace_char, is_name_char};
+use reader::Error;
+use util;
+
+/// `Token` represents a single lexeme of an XML document. These lexemes
+/// are used to perform actual parsing.
+#[derive(Copy, Clone, PartialEq, Eq, Debug)]
+pub enum Token {
+ /// `<?`
+ ProcessingInstructionStart,
+ /// `?>`
+ ProcessingInstructionEnd,
+ /// `<!DOCTYPE
+ DoctypeStart,
+ /// `<`
+ OpeningTagStart,
+ /// `</`
+ ClosingTagStart,
+ /// `>`
+ TagEnd,
+ /// `/>`
+ EmptyTagEnd,
+ /// `<!--`
+ CommentStart,
+ /// `-->`
+ CommentEnd,
+ /// A chunk of characters, used for errors recovery.
+ Chunk(&'static str),
+ /// Any non-special character except whitespace.
+ Character(char),
+ /// Whitespace character.
+ Whitespace(char),
+ /// `=`
+ EqualsSign,
+ /// `'`
+ SingleQuote,
+ /// `"`
+ DoubleQuote,
+ /// `<![CDATA[`
+ CDataStart,
+ /// `]]>`
+ CDataEnd,
+ /// `&`
+ ReferenceStart,
+ /// `;`
+ ReferenceEnd,
+}
+
+impl fmt::Display for Token {
+ fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+ match *self {
+ Token::Chunk(s) => write!(f, "{}", s),
+ Token::Character(c) | Token::Whitespace(c) => write!(f, "{}", c),
+ other => write!(f, "{}", match other {
+ Token::OpeningTagStart => "<",
+ Token::ProcessingInstructionStart => "<?",
+ Token::DoctypeStart => "<!DOCTYPE",
+ Token::ClosingTagStart => "</",
+ Token::CommentStart => "<!--",
+ Token::CDataStart => "<![CDATA[",
+ Token::TagEnd => ">",
+ Token::EmptyTagEnd => "/>",
+ Token::ProcessingInstructionEnd => "?>",
+ Token::CommentEnd => "-->",
+ Token::CDataEnd => "]]>",
+ Token::ReferenceStart => "&",
+ Token::ReferenceEnd => ";",
+ Token::EqualsSign => "=",
+ Token::SingleQuote => "'",
+ Token::DoubleQuote => "\"",
+ _ => unreachable!()
+ })
+ }
+ }
+}
+
+impl Token {
+ pub fn as_static_str(&self) -> Option<&'static str> {
+ match *self {
+ Token::OpeningTagStart => Some("<"),
+ Token::ProcessingInstructionStart => Some("<?"),
+ Token::DoctypeStart => Some("<!DOCTYPE"),
+ Token::ClosingTagStart => Some("</"),
+ Token::CommentStart => Some("<!--"),
+ Token::CDataStart => Some("<![CDATA["),
+ Token::TagEnd => Some(">"),
+ Token::EmptyTagEnd => Some("/>"),
+ Token::ProcessingInstructionEnd => Some("?>"),
+ Token::CommentEnd => Some("-->"),
+ Token::CDataEnd => Some("]]>"),
+ Token::ReferenceStart => Some("&"),
+ Token::ReferenceEnd => Some(";"),
+ Token::EqualsSign => Some("="),
+ Token::SingleQuote => Some("'"),
+ Token::DoubleQuote => Some("\""),
+ Token::Chunk(s) => Some(s),
+ _ => None
+ }
+ }
+
+ // using String.push_str(token.to_string()) is simply way too slow
+ pub fn push_to_string(&self, target: &mut String) {
+ match self.as_static_str() {
+ Some(s) => { target.push_str(s); }
+ None => {
+ match *self {
+ Token::Character(c) | Token::Whitespace(c) => target.push(c),
+ _ => unreachable!()
+ }
+ }
+ }
+ }
+
+ /// Returns `true` if this token contains data that can be interpreted
+ /// as a part of the text. Surprisingly, this also means '>' and '=' and '"' and "'" and '-->'.
+ #[inline]
+ pub fn contains_char_data(&self) -> bool {
+ match *self {
+ Token::Whitespace(_) | Token::Chunk(_) | Token::Character(_) | Token::CommentEnd |
+ Token::TagEnd | Token::EqualsSign | Token::DoubleQuote | Token::SingleQuote => true,
+ _ => false
+ }
+ }
+
+ /// Returns `true` if this token corresponds to a white space character.
+ #[inline]
+ pub fn is_whitespace(&self) -> bool {
+ match *self {
+ Token::Whitespace(_) => true,
+ _ => false
+ }
+ }
+}
+
+enum State {
+ /// Triggered on '<'
+ TagStarted,
+ /// Triggered on '<!'
+ CommentOrCDataOrDoctypeStarted,
+ /// Triggered on '<!-'
+ CommentStarted,
+ /// Triggered on '<!D' up to '<!DOCTYPE'
+ DoctypeStarted(DoctypeStartedSubstate),
+ /// Triggered after DoctypeStarted to handle sub elements
+ DoctypeFinishing(u8),
+ /// Triggered on '<![' up to '<![CDATA'
+ CDataStarted(CDataStartedSubstate),
+ /// Triggered on '?'
+ ProcessingInstructionClosing,
+ /// Triggered on '/'
+ EmptyTagClosing,
+ /// Triggered on '-' up to '--'
+ CommentClosing(ClosingSubstate),
+ /// Triggered on ']' up to ']]'
+ CDataClosing(ClosingSubstate),
+ /// Default state
+ Normal
+}
+
+#[derive(Copy, Clone)]
+enum ClosingSubstate {
+ First, Second
+}
+
+#[derive(Copy, Clone)]
+enum DoctypeStartedSubstate {
+ D, DO, DOC, DOCT, DOCTY, DOCTYP
+}
+
+#[derive(Copy, Clone)]
+enum CDataStartedSubstate {
+ E, C, CD, CDA, CDAT, CDATA
+}
+
+/// `Result` represents lexing result. It is either a token or an error message.
+pub type Result = result::Result<Option<Token>, Error>;
+
+/// Helps to set up a dispatch table for lexing large unambigous tokens like
+/// `<![CDATA[` or `<!DOCTYPE `.
+macro_rules! dispatch_on_enum_state(
+ ($_self:ident, $s:expr, $c:expr, $is:expr,
+ $($st:ident; $stc:expr ; $next_st:ident ; $chunk:expr),+;
+ $end_st:ident ; $end_c:expr ; $end_chunk:expr ; $e:expr) => (
+ match $s {
+ $(
+ $st => match $c {
+ $stc => $_self.move_to($is($next_st)),
+ _ => $_self.handle_error($chunk, $c)
+ },
+ )+
+ $end_st => match $c {
+ $end_c => $e,
+ _ => $_self.handle_error($end_chunk, $c)
+ }
+ }
+ )
+);
+
+/// `Lexer` is a lexer for XML documents, which implements pull API.
+///
+/// Main method is `next_token` which accepts an `std::io::Read` instance and
+/// tries to read the next lexeme from it.
+///
+/// When `skip_errors` flag is set, invalid lexemes will be returned as `Chunk`s.
+/// When it is not set, errors will be reported as `Err` objects with a string message.
+/// By default this flag is not set. Use `enable_errors` and `disable_errors` methods
+/// to toggle the behavior.
+pub struct Lexer {
+ pos: TextPosition,
+ head_pos: TextPosition,
+ char_queue: VecDeque<char>,
+ st: State,
+ skip_errors: bool,
+ inside_comment: bool,
+ inside_token: bool,
+ eof_handled: bool
+}
+
+impl Position for Lexer {
+ #[inline]
+ /// Returns the position of the last token produced by the lexer
+ fn position(&self) -> TextPosition { self.pos }
+}
+
+impl Lexer {
+ /// Returns a new lexer with default state.
+ pub fn new() -> Lexer {
+ Lexer {
+ pos: TextPosition::new(),
+ head_pos: TextPosition::new(),
+ char_queue: VecDeque::with_capacity(4), // TODO: check size
+ st: State::Normal,
+ skip_errors: false,
+ inside_comment: false,
+ inside_token: false,
+ eof_handled: false
+ }
+ }
+
+ /// Enables error handling so `next_token` will return `Some(Err(..))`
+ /// upon invalid lexeme.
+ #[inline]
+ pub fn enable_errors(&mut self) { self.skip_errors = false; }
+
+ /// Disables error handling so `next_token` will return `Some(Chunk(..))`
+ /// upon invalid lexeme with this lexeme content.
+ #[inline]
+ pub fn disable_errors(&mut self) { self.skip_errors = true; }
+
+ /// Enables special handling of some lexemes which should be done when we're parsing comment
+ /// internals.
+ #[inline]
+ pub fn inside_comment(&mut self) { self.inside_comment = true; }
+
+ /// Disables the effect of `inside_comment()` method.
+ #[inline]
+ pub fn outside_comment(&mut self) { self.inside_comment = false; }
+
+ /// Reset the eof handled flag of the lexer.
+ #[inline]
+ pub fn reset_eof_handled(&mut self) { self.eof_handled = false; }
+
+ /// Tries to read the next token from the buffer.
+ ///
+ /// It is possible to pass different instaces of `BufReader` each time
+ /// this method is called, but the resulting behavior is undefined in this case.
+ ///
+ /// Return value:
+ /// * `Err(reason) where reason: reader::Error` - when an error occurs;
+ /// * `Ok(None)` - upon end of stream is reached;
+ /// * `Ok(Some(token)) where token: Token` - in case a complete-token has been read from the stream.
+ pub fn next_token<B: Read>(&mut self, b: &mut B) -> Result {
+ // Already reached end of buffer
+ if self.eof_handled {
+ return Ok(None);
+ }
+
+ if !self.inside_token {
+ self.pos = self.head_pos;
+ self.inside_token = true;
+ }
+
+ // Check if we have saved a char or two for ourselves
+ while let Some(c) = self.char_queue.pop_front() {
+ match try!(self.read_next_token(c)) {
+ Some(t) => {
+ self.inside_token = false;
+ return Ok(Some(t));
+ }
+ None => {} // continue
+ }
+ }
+
+ loop {
+ // TODO: this should handle multiple encodings
+ let c = match try!(util::next_char_from(b)) {
+ Some(c) => c, // got next char
+ None => break, // nothing to read left
+ };
+
+ match try!(self.read_next_token(c)) {
+ Some(t) => {
+ self.inside_token = false;
+ return Ok(Some(t));
+ }
+ None => {
+ // continue
+ }
+ }
+ }
+
+ // Handle end of stream
+ self.eof_handled = true;
+ self.pos = self.head_pos;
+ match self.st {
+ State::TagStarted | State::CommentOrCDataOrDoctypeStarted |
+ State::CommentStarted | State::CDataStarted(_)| State::DoctypeStarted(_) |
+ State::CommentClosing(ClosingSubstate::Second) |
+ State::DoctypeFinishing(_) =>
+ Err(self.error("Unexpected end of stream")),
+ State::ProcessingInstructionClosing =>
+ Ok(Some(Token::Character('?'))),
+ State::EmptyTagClosing =>
+ Ok(Some(Token::Character('/'))),
+ State::CommentClosing(ClosingSubstate::First) =>
+ Ok(Some(Token::Character('-'))),
+ State::CDataClosing(ClosingSubstate::First) =>
+ Ok(Some(Token::Character(']'))),
+ State::CDataClosing(ClosingSubstate::Second) =>
+ Ok(Some(Token::Chunk("]]"))),
+ State::Normal =>
+ Ok(None)
+ }
+ }
+
+ #[inline]
+ fn error<M: Into<Cow<'static, str>>>(&self, msg: M) -> Error {
+ (self, msg).into()
+ }
+
+ #[inline]
+ fn read_next_token(&mut self, c: char) -> Result {
+ let res = self.dispatch_char(c);
+ if self.char_queue.is_empty() {
+ if c == '\n' {
+ self.head_pos.new_line();
+ } else {
+ self.head_pos.advance(1);
+ }
+ }
+ res
+ }
+
+ fn dispatch_char(&mut self, c: char) -> Result {
+ match self.st {
+ State::Normal => self.normal(c),
+ State::TagStarted => self.tag_opened(c),
+ State::CommentOrCDataOrDoctypeStarted => self.comment_or_cdata_or_doctype_started(c),
+ State::CommentStarted => self.comment_started(c),
+ State::CDataStarted(s) => self.cdata_started(c, s),
+ State::DoctypeStarted(s) => self.doctype_started(c, s),
+ State::DoctypeFinishing(d) => self.doctype_finishing(c, d),
+ State::ProcessingInstructionClosing => self.processing_instruction_closing(c),
+ State::EmptyTagClosing => self.empty_element_closing(c),
+ State::CommentClosing(s) => self.comment_closing(c, s),
+ State::CDataClosing(s) => self.cdata_closing(c, s)
+ }
+ }
+
+ #[inline]
+ fn move_to(&mut self, st: State) -> Result {
+ self.st = st;
+ Ok(None)
+ }
+
+ #[inline]
+ fn move_to_with(&mut self, st: State, token: Token) -> Result {
+ self.st = st;
+ Ok(Some(token))
+ }
+
+ #[inline]
+ fn move_to_with_unread(&mut self, st: State, cs: &[char], token: Token) -> Result {
+ self.char_queue.extend(cs.iter().cloned());
+ self.move_to_with(st, token)
+ }
+
+ fn handle_error(&mut self, chunk: &'static str, c: char) -> Result {
+ self.char_queue.push_back(c);
+ if self.skip_errors || (self.inside_comment && chunk != "--") { // FIXME: looks hacky
+ self.move_to_with(State::Normal, Token::Chunk(chunk))
+ } else {
+ Err(self.error(format!("Unexpected token '{}' before '{}'", chunk, c)))
+ }
+ }
+
+ /// Encountered a char
+ fn normal(&mut self, c: char) -> Result {
+ match c {
+ '<' => self.move_to(State::TagStarted),
+ '>' => Ok(Some(Token::TagEnd)),
+ '/' => self.move_to(State::EmptyTagClosing),
+ '=' => Ok(Some(Token::EqualsSign)),
+ '"' => Ok(Some(Token::DoubleQuote)),
+ '\'' => Ok(Some(Token::SingleQuote)),
+ '?' => self.move_to(State::ProcessingInstructionClosing),
+ '-' => self.move_to(State::CommentClosing(ClosingSubstate::First)),
+ ']' => self.move_to(State::CDataClosing(ClosingSubstate::First)),
+ '&' => Ok(Some(Token::ReferenceStart)),
+ ';' => Ok(Some(Token::ReferenceEnd)),
+ _ if is_whitespace_char(c) => Ok(Some(Token::Whitespace(c))),
+ _ => Ok(Some(Token::Character(c)))
+ }
+ }
+
+ /// Encountered '<'
+ fn tag_opened(&mut self, c: char) -> Result {
+ match c {
+ '?' => self.move_to_with(State::Normal, Token::ProcessingInstructionStart),
+ '/' => self.move_to_with(State::Normal, Token::ClosingTagStart),
+ '!' => self.move_to(State::CommentOrCDataOrDoctypeStarted),
+ _ if is_whitespace_char(c) => self.move_to_with_unread(State::Normal, &[c], Token::OpeningTagStart),
+ _ if is_name_char(c) => self.move_to_with_unread(State::Normal, &[c], Token::OpeningTagStart),
+ _ => self.handle_error("<", c)
+ }
+ }
+
+ /// Encountered '<!'
+ fn comment_or_cdata_or_doctype_started(&mut self, c: char) -> Result {
+ match c {
+ '-' => self.move_to(State::CommentStarted),
+ '[' => self.move_to(State::CDataStarted(CDataStartedSubstate::E)),
+ 'D' => self.move_to(State::DoctypeStarted(DoctypeStartedSubstate::D)),
+ _ => self.handle_error("<!", c)
+ }
+ }
+
+ /// Encountered '<!-'
+ fn comment_started(&mut self, c: char) -> Result {
+ match c {
+ '-' => self.move_to_with(State::Normal, Token::CommentStart),
+ _ => self.handle_error("<!-", c)
+ }
+ }
+
+ /// Encountered '<!['
+ fn cdata_started(&mut self, c: char, s: CDataStartedSubstate) -> Result {
+ use self::CDataStartedSubstate::{E, C, CD, CDA, CDAT, CDATA};
+ dispatch_on_enum_state!(self, s, c, State::CDataStarted,
+ E ; 'C' ; C ; "<![",
+ C ; 'D' ; CD ; "<![C",
+ CD ; 'A' ; CDA ; "<![CD",
+ CDA ; 'T' ; CDAT ; "<![CDA",
+ CDAT ; 'A' ; CDATA ; "<![CDAT";
+ CDATA ; '[' ; "<![CDATA" ; self.move_to_with(State::Normal, Token::CDataStart)
+ )
+ }
+
+ /// Encountered '<!D'
+ fn doctype_started(&mut self, c: char, s: DoctypeStartedSubstate) -> Result {
+ use self::DoctypeStartedSubstate::{D, DO, DOC, DOCT, DOCTY, DOCTYP};
+ dispatch_on_enum_state!(self, s, c, State::DoctypeStarted,
+ D ; 'O' ; DO ; "<!D",
+ DO ; 'C' ; DOC ; "<!DO",
+ DOC ; 'T' ; DOCT ; "<!DOC",
+ DOCT ; 'Y' ; DOCTY ; "<!DOCT",
+ DOCTY ; 'P' ; DOCTYP ; "<!DOCTY";
+ DOCTYP ; 'E' ; "<!DOCTYP" ; self.move_to_with(State::DoctypeFinishing(1), Token::DoctypeStart)
+ )
+ }
+
+ /// State used while awaiting the closing bracket for the <!DOCTYPE tag
+ fn doctype_finishing(&mut self, c: char, d: u8) -> Result {
+ match c {
+ '<' => self.move_to(State::DoctypeFinishing(d + 1)),
+ '>' if d == 1 => self.move_to_with(State::Normal, Token::TagEnd),
+ '>' => self.move_to(State::DoctypeFinishing(d - 1)),
+ _ => Ok(None),
+ }
+ }
+
+ /// Encountered '?'
+ fn processing_instruction_closing(&mut self, c: char) -> Result {
+ match c {
+ '>' => self.move_to_with(State::Normal, Token::ProcessingInstructionEnd),
+ _ => self.move_to_with_unread(State::Normal, &[c], Token::Character('?')),
+ }
+ }
+
+ /// Encountered '/'
+ fn empty_element_closing(&mut self, c: char) -> Result {
+ match c {
+ '>' => self.move_to_with(State::Normal, Token::EmptyTagEnd),
+ _ => self.move_to_with_unread(State::Normal, &[c], Token::Character('/')),
+ }
+ }
+
+ /// Encountered '-'
+ fn comment_closing(&mut self, c: char, s: ClosingSubstate) -> Result {
+ match s {
+ ClosingSubstate::First => match c {
+ '-' => self.move_to(State::CommentClosing(ClosingSubstate::Second)),
+ _ => self.move_to_with_unread(State::Normal, &[c], Token::Character('-'))
+ },
+ ClosingSubstate::Second => match c {
+ '>' => self.move_to_with(State::Normal, Token::CommentEnd),
+ // double dash not followed by a greater-than is a hard error inside comment
+ _ if self.inside_comment => self.handle_error("--", c),
+ // nothing else except comment closing starts with a double dash, and comment
+ // closing can never be after another dash, and also we're outside of a comment,
+ // therefore it is safe to push only the last read character to the list of unread
+ // characters and pass the double dash directly to the output
+ _ => self.move_to_with_unread(State::Normal, &[c], Token::Chunk("--"))
+ }
+ }
+ }
+
+ /// Encountered ']'
+ fn cdata_closing(&mut self, c: char, s: ClosingSubstate) -> Result {
+ match s {
+ ClosingSubstate::First => match c {
+ ']' => self.move_to(State::CDataClosing(ClosingSubstate::Second)),
+ _ => self.move_to_with_unread(State::Normal, &[c], Token::Character(']'))
+ },
+ ClosingSubstate::Second => match c {
+ '>' => self.move_to_with(State::Normal, Token::CDataEnd),
+ _ => self.move_to_with_unread(State::Normal, &[']', c], Token::Character(']'))
+ }
+ }
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use common::{Position};
+ use std::io::{BufReader, Cursor};
+
+ use super::{Lexer, Token};
+
+ macro_rules! assert_oks(
+ (for $lex:ident and $buf:ident ; $($e:expr)+) => ({
+ $(
+ assert_eq!(Ok(Some($e)), $lex.next_token(&mut $buf));
+ )+
+ })
+ );
+
+ macro_rules! assert_err(
+ (for $lex:ident and $buf:ident expect row $r:expr ; $c:expr, $s:expr) => ({
+ let err = $lex.next_token(&mut $buf);
+ assert!(err.is_err());
+ let err = err.unwrap_err();
+ assert_eq!($r as u64, err.position().row);
+ assert_eq!($c as u64, err.position().column);
+ assert_eq!($s, err.msg());
+ })
+ );
+
+ macro_rules! assert_none(
+ (for $lex:ident and $buf:ident) => (
+ assert_eq!(Ok(None), $lex.next_token(&mut $buf));
+ )
+ );
+
+ fn make_lex_and_buf(s: &str) -> (Lexer, BufReader<Cursor<Vec<u8>>>) {
+ (Lexer::new(), BufReader::new(Cursor::new(s.to_owned().into_bytes())))
+ }
+
+ #[test]
+ fn simple_lexer_test() {
+ let (mut lex, mut buf) = make_lex_and_buf(
+ r#"<a p='q'> x<b z="y">d </b></a><p/> <?nm ?> <!-- a c --> "#
+ );
+
+ assert_oks!(for lex and buf ;
+ Token::OpeningTagStart
+ Token::Character('a')
+ Token::Whitespace(' ')
+ Token::Character('p')
+ Token::EqualsSign
+ Token::SingleQuote
+ Token::Character('q')
+ Token::SingleQuote
+ Token::TagEnd
+ Token::Whitespace(' ')
+ Token::Character('x')
+ Token::OpeningTagStart
+ Token::Character('b')
+ Token::Whitespace(' ')
+ Token::Character('z')
+ Token::EqualsSign
+ Token::DoubleQuote
+ Token::Character('y')
+ Token::DoubleQuote
+ Token::TagEnd
+ Token::Character('d')
+ Token::Whitespace('\t')
+ Token::ClosingTagStart
+ Token::Character('b')
+ Token::TagEnd
+ Token::ClosingTagStart
+ Token::Character('a')
+ Token::TagEnd
+ Token::OpeningTagStart
+ Token::Character('p')
+ Token::EmptyTagEnd
+ Token::Whitespace(' ')
+ Token::ProcessingInstructionStart
+ Token::Character('n')
+ Token::Character('m')
+ Token::Whitespace(' ')
+ Token::ProcessingInstructionEnd
+ Token::Whitespace(' ')
+ Token::CommentStart
+ Token::Whitespace(' ')
+ Token::Character('a')
+ Token::Whitespace(' ')
+ Token::Character('c')
+ Token::Whitespace(' ')
+ Token::CommentEnd
+ Token::Whitespace(' ')
+ Token::ReferenceStart
+ Token::Character('n')
+ Token::Character('b')
+ Token::Character('s')
+ Token::Character('p')
+ Token::ReferenceEnd
+ );
+ assert_none!(for lex and buf);
+ }
+
+ #[test]
+ fn special_chars_test() {
+ let (mut lex, mut buf) = make_lex_and_buf(
+ r#"?x!+ // -| ]z]]"#
+ );
+
+ assert_oks!(for lex and buf ;
+ Token::Character('?')
+ Token::Character('x')
+ Token::Character('!')
+ Token::Character('+')
+ Token::Whitespace(' ')
+ Token::Character('/')
+ Token::Character('/')
+ Token::Whitespace(' ')
+ Token::Character('-')
+ Token::Character('|')
+ Token::Whitespace(' ')
+ Token::Character(']')
+ Token::Character('z')
+ Token::Chunk("]]")
+ );
+ assert_none!(for lex and buf);
+ }
+
+ #[test]
+ fn cdata_test() {
+ let (mut lex, mut buf) = make_lex_and_buf(
+ r#"<a><![CDATA[x y ?]]> </a>"#
+ );
+
+ assert_oks!(for lex and buf ;
+ Token::OpeningTagStart
+ Token::Character('a')
+ Token::TagEnd
+ Token::CDataStart
+ Token::Character('x')
+ Token::Whitespace(' ')
+ Token::Character('y')
+ Token::Whitespace(' ')
+ Token::Character('?')
+ Token::CDataEnd
+ Token::Whitespace(' ')
+ Token::ClosingTagStart
+ Token::Character('a')
+ Token::TagEnd
+ );
+ assert_none!(for lex and buf);
+ }
+
+ #[test]
+ fn doctype_test() {
+ let (mut lex, mut buf) = make_lex_and_buf(
+ r#"<a><!DOCTYPE ab xx z> "#
+ );
+ assert_oks!(for lex and buf ;
+ Token::OpeningTagStart
+ Token::Character('a')
+ Token::TagEnd
+ Token::DoctypeStart
+ Token::TagEnd
+ Token::Whitespace(' ')
+ );
+ assert_none!(for lex and buf)
+ }
+
+ #[test]
+ fn doctype_with_internal_subset_test() {
+ let (mut lex, mut buf) = make_lex_and_buf(
+ r#"<a><!DOCTYPE ab[<!ELEMENT ba> ]> "#
+ );
+ assert_oks!(for lex and buf ;
+ Token::OpeningTagStart
+ Token::Character('a')
+ Token::TagEnd
+ Token::DoctypeStart
+ Token::TagEnd
+ Token::Whitespace(' ')
+ );
+ assert_none!(for lex and buf)
+ }
+
+ #[test]
+ fn end_of_stream_handling_ok() {
+ macro_rules! eof_check(
+ ($data:expr ; $token:expr) => ({
+ let (mut lex, mut buf) = make_lex_and_buf($data);
+ assert_oks!(for lex and buf ; $token);
+ assert_none!(for lex and buf);
+ })
+ );
+ eof_check!("?" ; Token::Character('?'));
+ eof_check!("/" ; Token::Character('/'));
+ eof_check!("-" ; Token::Character('-'));
+ eof_check!("]" ; Token::Character(']'));
+ eof_check!("]]" ; Token::Chunk("]]"));
+ }
+
+ #[test]
+ fn end_of_stream_handling_error() {
+ macro_rules! eof_check(
+ ($data:expr; $r:expr, $c:expr) => ({
+ let (mut lex, mut buf) = make_lex_and_buf($data);
+ assert_err!(for lex and buf expect row $r ; $c, "Unexpected end of stream");
+ assert_none!(for lex and buf);
+ })
+ );
+ eof_check!("<" ; 0, 1);
+ eof_check!("<!" ; 0, 2);
+ eof_check!("<!-" ; 0, 3);
+ eof_check!("<![" ; 0, 3);
+ eof_check!("<![C" ; 0, 4);
+ eof_check!("<![CD" ; 0, 5);
+ eof_check!("<![CDA" ; 0, 6);
+ eof_check!("<![CDAT" ; 0, 7);
+ eof_check!("<![CDATA" ; 0, 8);
+ eof_check!("--" ; 0, 2);
+ }
+
+ #[test]
+ fn error_in_comment_or_cdata_prefix() {
+ let (mut lex, mut buf) = make_lex_and_buf("<!x");
+ assert_err!(for lex and buf expect row 0 ; 0,
+ "Unexpected token '<!' before 'x'"
+ );
+
+ let (mut lex, mut buf) = make_lex_and_buf("<!x");
+ lex.disable_errors();
+ assert_oks!(for lex and buf ;
+ Token::Chunk("<!")
+ Token::Character('x')
+ );
+ assert_none!(for lex and buf);
+ }
+
+ #[test]
+ fn error_in_comment_started() {
+ let (mut lex, mut buf) = make_lex_and_buf("<!-\t");
+ assert_err!(for lex and buf expect row 0 ; 0,
+ "Unexpected token '<!-' before '\t'"
+ );
+
+ let (mut lex, mut buf) = make_lex_and_buf("<!-\t");
+ lex.disable_errors();
+ assert_oks!(for lex and buf ;
+ Token::Chunk("<!-")
+ Token::Whitespace('\t')
+ );
+ assert_none!(for lex and buf);
+ }
+
+ #[test]
+ fn error_in_comment_two_dashes_not_at_end() {
+ let (mut lex, mut buf) = make_lex_and_buf("--x");
+ lex.inside_comment();
+ assert_err!(for lex and buf expect row 0; 0,
+ "Unexpected token '--' before 'x'"
+ );
+
+ let (mut lex, mut buf) = make_lex_and_buf("--x");
+ assert_oks!(for lex and buf ;
+ Token::Chunk("--")
+ Token::Character('x')
+ );
+ }
+
+ macro_rules! check_case(
+ ($chunk:expr, $app:expr; $data:expr; $r:expr, $c:expr, $s:expr) => ({
+ let (mut lex, mut buf) = make_lex_and_buf($data);
+ assert_err!(for lex and buf expect row $r ; $c, $s);
+
+ let (mut lex, mut buf) = make_lex_and_buf($data);
+ lex.disable_errors();
+ assert_oks!(for lex and buf ;
+ Token::Chunk($chunk)
+ Token::Character($app)
+ );
+ assert_none!(for lex and buf);
+ })
+ );
+
+ #[test]
+ fn error_in_cdata_started() {
+ check_case!("<![", '['; "<![[" ; 0, 0, "Unexpected token '<![' before '['");
+ check_case!("<![C", '['; "<![C[" ; 0, 0, "Unexpected token '<![C' before '['");
+ check_case!("<![CD", '['; "<![CD[" ; 0, 0, "Unexpected token '<![CD' before '['");
+ check_case!("<![CDA", '['; "<![CDA[" ; 0, 0, "Unexpected token '<![CDA' before '['");
+ check_case!("<![CDAT", '['; "<![CDAT[" ; 0, 0, "Unexpected token '<![CDAT' before '['");
+ check_case!("<![CDATA", '|'; "<![CDATA|" ; 0, 0, "Unexpected token '<![CDATA' before '|'");
+ }
+
+ #[test]
+ fn error_in_doctype_started() {
+ check_case!("<!D", 'a'; "<!Da" ; 0, 0, "Unexpected token '<!D' before 'a'");
+ check_case!("<!DO", 'b'; "<!DOb" ; 0, 0, "Unexpected token '<!DO' before 'b'");
+ check_case!("<!DOC", 'c'; "<!DOCc" ; 0, 0, "Unexpected token '<!DOC' before 'c'");
+ check_case!("<!DOCT", 'd'; "<!DOCTd" ; 0, 0, "Unexpected token '<!DOCT' before 'd'");
+ check_case!("<!DOCTY", 'e'; "<!DOCTYe" ; 0, 0, "Unexpected token '<!DOCTY' before 'e'");
+ check_case!("<!DOCTYP", 'f'; "<!DOCTYPf" ; 0, 0, "Unexpected token '<!DOCTYP' before 'f'");
+ }
+
+
+
+ #[test]
+ fn issue_98_cdata_ending_with_right_bracket() {
+ let (mut lex, mut buf) = make_lex_and_buf(
+ r#"<![CDATA[Foo [Bar]]]>"#
+ );
+
+ assert_oks!(for lex and buf ;
+ Token::CDataStart
+ Token::Character('F')
+ Token::Character('o')
+ Token::Character('o')
+ Token::Whitespace(' ')
+ Token::Character('[')
+ Token::Character('B')
+ Token::Character('a')
+ Token::Character('r')
+ Token::Character(']')
+ Token::CDataEnd
+ );
+ assert_none!(for lex and buf);
+ }
+}
diff --git a/src/reader/mod.rs b/src/reader/mod.rs
new file mode 100644
index 0000000..90f5b52
--- /dev/null
+++ b/src/reader/mod.rs
@@ -0,0 +1,129 @@
+//! Contains high-level interface for a pull-based XML parser.
+//!
+//! The most important type in this module is `EventReader`, which provides an iterator
+//! view for events in XML document.
+
+use std::io::{Read};
+use std::result;
+
+use common::{Position, TextPosition};
+
+pub use self::config::ParserConfig;
+pub use self::events::XmlEvent;
+
+use self::parser::PullParser;
+
+mod lexer;
+mod parser;
+mod config;
+mod events;
+
+mod error;
+pub use self::error::{Error, ErrorKind};
+
+/// A result type yielded by `XmlReader`.
+pub type Result<T> = result::Result<T, Error>;
+
+/// A wrapper around an `std::io::Read` instance which provides pull-based XML parsing.
+pub struct EventReader<R: Read> {
+ source: R,
+ parser: PullParser
+}
+
+impl<R: Read> EventReader<R> {
+ /// Creates a new reader, consuming the given stream.
+ #[inline]
+ pub fn new(source: R) -> EventReader<R> {
+ EventReader::new_with_config(source, ParserConfig::new())
+ }
+
+ /// Creates a new reader with the provded configuration, consuming the given stream.
+ #[inline]
+ pub fn new_with_config(source: R, config: ParserConfig) -> EventReader<R> {
+ EventReader { source: source, parser: PullParser::new(config) }
+ }
+
+ /// Pulls and returns next XML event from the stream.
+ ///
+ /// If returned event is `XmlEvent::Error` or `XmlEvent::EndDocument`, then
+ /// further calls to this method will return this event again.
+ #[inline]
+ pub fn next(&mut self) -> Result<XmlEvent> {
+ self.parser.next(&mut self.source)
+ }
+
+ pub fn source(&self) -> &R { &self.source }
+ pub fn source_mut(&mut self) -> &mut R { &mut self.source }
+
+ /// Unwraps this `EventReader`, returning the underlying reader.
+ ///
+ /// Note that this operation is destructive; unwrapping the reader and wrapping it
+ /// again with `EventReader::new()` will create a fresh reader which will attempt
+ /// to parse an XML document from the beginning.
+ pub fn into_inner(self) -> R {
+ self.source
+ }
+}
+
+impl<B: Read> Position for EventReader<B> {
+ /// Returns the position of the last event produced by the reader.
+ #[inline]
+ fn position(&self) -> TextPosition {
+ self.parser.position()
+ }
+}
+
+impl<R: Read> IntoIterator for EventReader<R> {
+ type Item = Result<XmlEvent>;
+ type IntoIter = Events<R>;
+
+ fn into_iter(self) -> Events<R> {
+ Events { reader: self, finished: false }
+ }
+}
+
+/// An iterator over XML events created from some type implementing `Read`.
+///
+/// When the next event is `xml::event::Error` or `xml::event::EndDocument`, then
+/// it will be returned by the iterator once, and then it will stop producing events.
+pub struct Events<R: Read> {
+ reader: EventReader<R>,
+ finished: bool
+}
+
+impl<R: Read> Events<R> {
+ /// Unwraps the iterator, returning the internal `EventReader`.
+ #[inline]
+ pub fn into_inner(self) -> EventReader<R> {
+ self.reader
+ }
+
+ pub fn source(&self) -> &R { &self.reader.source }
+ pub fn source_mut(&mut self) -> &mut R { &mut self.reader.source }
+
+}
+
+impl<R: Read> Iterator for Events<R> {
+ type Item = Result<XmlEvent>;
+
+ #[inline]
+ fn next(&mut self) -> Option<Result<XmlEvent>> {
+ if self.finished && !self.reader.parser.is_ignoring_end_of_stream() { None }
+ else {
+ let ev = self.reader.next();
+ match ev {
+ Ok(XmlEvent::EndDocument) | Err(_) => self.finished = true,
+ _ => {}
+ }
+ Some(ev)
+ }
+ }
+}
+
+impl<'r> EventReader<&'r [u8]> {
+ /// A convenience method to create an `XmlReader` from a string slice.
+ #[inline]
+ pub fn from_str(source: &'r str) -> EventReader<&'r [u8]> {
+ EventReader::new(source.as_bytes())
+ }
+}
diff --git a/src/reader/parser/inside_cdata.rs b/src/reader/parser/inside_cdata.rs
new file mode 100644
index 0000000..3269fb4
--- /dev/null
+++ b/src/reader/parser/inside_cdata.rs
@@ -0,0 +1,32 @@
+use reader::events::XmlEvent;
+use reader::lexer::Token;
+
+use super::{Result, PullParser, State};
+
+impl PullParser {
+ pub fn inside_cdata(&mut self, t: Token) -> Option<Result> {
+ match t {
+ Token::CDataEnd => {
+ self.lexer.enable_errors();
+ let event = if self.config.cdata_to_characters {
+ None
+ } else {
+ let data = self.take_buf();
+ Some(Ok(XmlEvent::CData(data)))
+ };
+ self.into_state(State::OutsideTag, event)
+ }
+
+ Token::Whitespace(_) => {
+ t.push_to_string(&mut self.buf);
+ None
+ }
+
+ _ => {
+ self.inside_whitespace = false;
+ t.push_to_string(&mut self.buf);
+ None
+ }
+ }
+ }
+}
diff --git a/src/reader/parser/inside_closing_tag_name.rs b/src/reader/parser/inside_closing_tag_name.rs
new file mode 100644
index 0000000..1d8074a
--- /dev/null
+++ b/src/reader/parser/inside_closing_tag_name.rs
@@ -0,0 +1,34 @@
+use namespace;
+
+use reader::lexer::Token;
+
+use super::{Result, PullParser, State, QualifiedNameTarget, ClosingTagSubstate};
+
+impl PullParser {
+ pub fn inside_closing_tag_name(&mut self, t: Token, s: ClosingTagSubstate) -> Option<Result> {
+ match s {
+ ClosingTagSubstate::CTInsideName => self.read_qualified_name(t, QualifiedNameTarget::ClosingTagNameTarget, |this, token, name| {
+ match name.prefix_ref() {
+ Some(prefix) if prefix == namespace::NS_XML_PREFIX ||
+ prefix == namespace::NS_XMLNS_PREFIX =>
+ // TODO: {:?} is bad, need something better
+ Some(self_error!(this; "'{:?}' cannot be an element name prefix", name.prefix)),
+ _ => {
+ this.data.element_name = Some(name.clone());
+ match token {
+ Token::Whitespace(_) => this.into_state_continue(State::InsideClosingTag(ClosingTagSubstate::CTAfterName)),
+ Token::TagEnd => this.emit_end_element(),
+ _ => Some(self_error!(this; "Unexpected token inside closing tag: {}", token))
+ }
+ }
+ }
+ }),
+ ClosingTagSubstate::CTAfterName => match t {
+ Token::Whitespace(_) => None, // Skip whitespace
+ Token::TagEnd => self.emit_end_element(),
+ _ => Some(self_error!(self; "Unexpected token inside closing tag: {}", t))
+ }
+ }
+ }
+
+}
diff --git a/src/reader/parser/inside_comment.rs b/src/reader/parser/inside_comment.rs
new file mode 100644
index 0000000..fc98320
--- /dev/null
+++ b/src/reader/parser/inside_comment.rs
@@ -0,0 +1,32 @@
+use reader::events::XmlEvent;
+use reader::lexer::Token;
+
+use super::{Result, PullParser, State};
+
+impl PullParser {
+ pub fn inside_comment(&mut self, t: Token) -> Option<Result> {
+ match t {
+ // Double dash is illegal inside a comment
+ Token::Chunk(ref s) if &s[..] == "--" => Some(self_error!(self; "Unexpected token inside a comment: --")),
+
+ Token::CommentEnd if self.config.ignore_comments => {
+ self.lexer.outside_comment();
+ self.into_state_continue(State::OutsideTag)
+ }
+
+ Token::CommentEnd => {
+ self.lexer.outside_comment();
+ let data = self.take_buf();
+ self.into_state_emit(State::OutsideTag, Ok(XmlEvent::Comment(data)))
+ }
+
+ _ if self.config.ignore_comments => None, // Do not modify buffer if ignoring the comment
+
+ _ => {
+ t.push_to_string(&mut self.buf);
+ None
+ }
+ }
+ }
+
+}
diff --git a/src/reader/parser/inside_declaration.rs b/src/reader/parser/inside_declaration.rs
new file mode 100644
index 0000000..af39d10
--- /dev/null
+++ b/src/reader/parser/inside_declaration.rs
@@ -0,0 +1,151 @@
+
+use common::XmlVersion;
+
+use reader::events::XmlEvent;
+use reader::lexer::Token;
+
+use super::{
+ Result, PullParser, State, DeclarationSubstate, QualifiedNameTarget,
+ DEFAULT_VERSION, DEFAULT_ENCODING
+};
+
+impl PullParser {
+ // TODO: remove redundancy via macros or extra methods
+ pub fn inside_declaration(&mut self, t: Token, s: DeclarationSubstate) -> Option<Result> {
+ macro_rules! unexpected_token(
+ ($this:expr; $t:expr) => (Some($this.error(format!("Unexpected token inside XML declaration: {}", $t))));
+ ($t:expr) => (unexpected_token!(self; $t));
+ );
+
+ #[inline]
+ fn emit_start_document(this: &mut PullParser) -> Option<Result> {
+ this.parsed_declaration = true;
+ let version = this.data.take_version();
+ let encoding = this.data.take_encoding();
+ let standalone = this.data.take_standalone();
+ this.into_state_emit(State::OutsideTag, Ok(XmlEvent::StartDocument {
+ version: version.unwrap_or(DEFAULT_VERSION),
+ encoding: encoding.unwrap_or(DEFAULT_ENCODING.into()),
+ standalone: standalone
+ }))
+ }
+
+ match s {
+ DeclarationSubstate::BeforeVersion => match t {
+ Token::Whitespace(_) => None, // continue
+ Token::Character('v') => self.into_state_continue(State::InsideDeclaration(DeclarationSubstate::InsideVersion)),
+ _ => unexpected_token!(t)
+ },
+
+ DeclarationSubstate::InsideVersion => self.read_qualified_name(t, QualifiedNameTarget::AttributeNameTarget, |this, token, name| {
+ match &name.local_name[..] {
+ "ersion" if name.namespace.is_none() =>
+ this.into_state_continue(State::InsideDeclaration(
+ if token == Token::EqualsSign {
+ DeclarationSubstate::InsideVersionValue
+ } else {
+ DeclarationSubstate::AfterVersion
+ }
+ )),
+ _ => unexpected_token!(this; name)
+ }
+ }),
+
+ DeclarationSubstate::AfterVersion => match t {
+ Token::Whitespace(_) => None,
+ Token::EqualsSign => self.into_state_continue(State::InsideDeclaration(DeclarationSubstate::InsideVersionValue)),
+ _ => unexpected_token!(t)
+ },
+
+ DeclarationSubstate::InsideVersionValue => self.read_attribute_value(t, |this, value| {
+ this.data.version = match &value[..] {
+ "1.0" => Some(XmlVersion::Version10),
+ "1.1" => Some(XmlVersion::Version11),
+ _ => None
+ };
+ if this.data.version.is_some() {
+ this.into_state_continue(State::InsideDeclaration(DeclarationSubstate::AfterVersionValue))
+ } else {
+ Some(self_error!(this; "Unexpected XML version value: {}", value))
+ }
+ }),
+
+ DeclarationSubstate::AfterVersionValue => match t {
+ Token::Whitespace(_) => None, // skip whitespace
+ Token::Character('e') => self.into_state_continue(State::InsideDeclaration(DeclarationSubstate::InsideEncoding)),
+ Token::Character('s') => self.into_state_continue(State::InsideDeclaration(DeclarationSubstate::InsideStandaloneDecl)),
+ Token::ProcessingInstructionEnd => emit_start_document(self),
+ _ => unexpected_token!(t)
+ },
+
+ DeclarationSubstate::InsideEncoding => self.read_qualified_name(t, QualifiedNameTarget::AttributeNameTarget, |this, token, name| {
+ match &name.local_name[..] {
+ "ncoding" if name.namespace.is_none() =>
+ this.into_state_continue(State::InsideDeclaration(
+ if token == Token::EqualsSign { DeclarationSubstate::InsideEncodingValue } else { DeclarationSubstate::AfterEncoding }
+ )),
+ _ => unexpected_token!(this; name)
+ }
+ }),
+
+ DeclarationSubstate::AfterEncoding => match t {
+ Token::Whitespace(_) => None,
+ Token::EqualsSign => self.into_state_continue(State::InsideDeclaration(DeclarationSubstate::InsideEncodingValue)),
+ _ => unexpected_token!(t)
+ },
+
+ DeclarationSubstate::InsideEncodingValue => self.read_attribute_value(t, |this, value| {
+ this.data.encoding = Some(value);
+ this.into_state_continue(State::InsideDeclaration(DeclarationSubstate::BeforeStandaloneDecl))
+ }),
+
+ DeclarationSubstate::BeforeStandaloneDecl => match t {
+ Token::Whitespace(_) => None, // skip whitespace
+ Token::Character('s') => self.into_state_continue(State::InsideDeclaration(DeclarationSubstate::InsideStandaloneDecl)),
+ Token::ProcessingInstructionEnd => emit_start_document(self),
+ _ => unexpected_token!(t)
+ },
+
+ DeclarationSubstate::InsideStandaloneDecl => self.read_qualified_name(t, QualifiedNameTarget::AttributeNameTarget, |this, token, name| {
+ match &name.local_name[..] {
+ "tandalone" if name.namespace.is_none() =>
+ this.into_state_continue(State::InsideDeclaration(
+ if token == Token::EqualsSign {
+ DeclarationSubstate::InsideStandaloneDeclValue
+ } else {
+ DeclarationSubstate::AfterStandaloneDecl
+ }
+ )),
+ _ => unexpected_token!(this; name)
+ }
+ }),
+
+ DeclarationSubstate::AfterStandaloneDecl => match t {
+ Token::Whitespace(_) => None,
+ Token::EqualsSign => self.into_state_continue(State::InsideDeclaration(DeclarationSubstate::InsideStandaloneDeclValue)),
+ _ => unexpected_token!(t)
+ },
+
+ DeclarationSubstate::InsideStandaloneDeclValue => self.read_attribute_value(t, |this, value| {
+ let standalone = match &value[..] {
+ "yes" => Some(true),
+ "no" => Some(false),
+ _ => None
+ };
+ if standalone.is_some() {
+ this.data.standalone = standalone;
+ this.into_state_continue(State::InsideDeclaration(DeclarationSubstate::AfterStandaloneDeclValue))
+ } else {
+ Some(self_error!(this; "Invalid standalone declaration value: {}", value))
+ }
+ }),
+
+ DeclarationSubstate::AfterStandaloneDeclValue => match t {
+ Token::Whitespace(_) => None, // skip whitespace
+ Token::ProcessingInstructionEnd => emit_start_document(self),
+ _ => unexpected_token!(t)
+ }
+ }
+ }
+
+}
diff --git a/src/reader/parser/inside_doctype.rs b/src/reader/parser/inside_doctype.rs
new file mode 100644
index 0000000..8dcf367
--- /dev/null
+++ b/src/reader/parser/inside_doctype.rs
@@ -0,0 +1,16 @@
+use reader::lexer::Token;
+
+use super::{Result, PullParser, State};
+
+impl PullParser {
+ pub fn inside_doctype(&mut self, t: Token) -> Option<Result> {
+ match t {
+ Token::TagEnd => {
+ self.lexer.enable_errors();
+ self.into_state_continue(State::OutsideTag)
+ }
+
+ _ => None
+ }
+ }
+}
diff --git a/src/reader/parser/inside_opening_tag.rs b/src/reader/parser/inside_opening_tag.rs
new file mode 100644
index 0000000..533874f
--- /dev/null
+++ b/src/reader/parser/inside_opening_tag.rs
@@ -0,0 +1,108 @@
+use common::is_name_start_char;
+use attribute::OwnedAttribute;
+use namespace;
+
+use reader::lexer::Token;
+
+use super::{Result, PullParser, State, OpeningTagSubstate, QualifiedNameTarget};
+
+impl PullParser {
+ pub fn inside_opening_tag(&mut self, t: Token, s: OpeningTagSubstate) -> Option<Result> {
+ macro_rules! unexpected_token(($t:expr) => (Some(self_error!(self; "Unexpected token inside opening tag: {}", $t))));
+ match s {
+ OpeningTagSubstate::InsideName => self.read_qualified_name(t, QualifiedNameTarget::OpeningTagNameTarget, |this, token, name| {
+ match name.prefix_ref() {
+ Some(prefix) if prefix == namespace::NS_XML_PREFIX ||
+ prefix == namespace::NS_XMLNS_PREFIX =>
+ Some(self_error!(this; "'{:?}' cannot be an element name prefix", name.prefix)),
+ _ => {
+ this.data.element_name = Some(name.clone());
+ match token {
+ Token::TagEnd => this.emit_start_element(false),
+ Token::EmptyTagEnd => this.emit_start_element(true),
+ Token::Whitespace(_) => this.into_state_continue(State::InsideOpeningTag(OpeningTagSubstate::InsideTag)),
+ _ => unreachable!()
+ }
+ }
+ }
+ }),
+
+ OpeningTagSubstate::InsideTag => match t {
+ Token::Whitespace(_) => None, // skip whitespace
+ Token::Character(c) if is_name_start_char(c) => {
+ self.buf.push(c);
+ self.into_state_continue(State::InsideOpeningTag(OpeningTagSubstate::InsideAttributeName))
+ }
+ Token::TagEnd => self.emit_start_element(false),
+ Token::EmptyTagEnd => self.emit_start_element(true),
+ _ => unexpected_token!(t)
+ },
+
+ OpeningTagSubstate::InsideAttributeName => self.read_qualified_name(t, QualifiedNameTarget::AttributeNameTarget, |this, token, name| {
+ this.data.attr_name = Some(name);
+ match token {
+ Token::Whitespace(_) => this.into_state_continue(State::InsideOpeningTag(OpeningTagSubstate::AfterAttributeName)),
+ Token::EqualsSign => this.into_state_continue(State::InsideOpeningTag(OpeningTagSubstate::InsideAttributeValue)),
+ _ => unreachable!()
+ }
+ }),
+
+ OpeningTagSubstate::AfterAttributeName => match t {
+ Token::Whitespace(_) => None,
+ Token::EqualsSign => self.into_state_continue(State::InsideOpeningTag(OpeningTagSubstate::InsideAttributeValue)),
+ _ => unexpected_token!(t)
+ },
+
+ OpeningTagSubstate::InsideAttributeValue => self.read_attribute_value(t, |this, value| {
+ let name = this.data.take_attr_name().unwrap(); // unwrap() will always succeed here
+
+ // check that no attribute with such name is already present
+ // if there is one, XML is not well-formed
+ if this.data.attributes.iter().find(|a| a.name == name).is_some() { // TODO: looks bad
+ // TODO: ideally this error should point to the beginning of the attribute,
+ // TODO: not the end of its value
+ Some(self_error!(this; "Attribute '{}' is redefined", name))
+ } else {
+ match name.prefix_ref() {
+ // declaring a new prefix; it is sufficient to check prefix only
+ // because "xmlns" prefix is reserved
+ Some(namespace::NS_XMLNS_PREFIX) => {
+ let ln = &name.local_name[..];
+ if ln == namespace::NS_XMLNS_PREFIX {
+ Some(self_error!(this; "Cannot redefine prefix '{}'", namespace::NS_XMLNS_PREFIX))
+ } else if ln == namespace::NS_XML_PREFIX && &value[..] != namespace::NS_XML_URI {
+ Some(self_error!(this; "Prefix '{}' cannot be rebound to another value", namespace::NS_XML_PREFIX))
+ } else if value.is_empty() {
+ Some(self_error!(this; "Cannot undefine prefix '{}'", ln))
+ } else {
+ this.nst.put(name.local_name.clone(), value);
+ this.into_state_continue(State::InsideOpeningTag(OpeningTagSubstate::InsideTag))
+ }
+ }
+
+ // declaring default namespace
+ None if &name.local_name[..] == namespace::NS_XMLNS_PREFIX =>
+ match &value[..] {
+ namespace::NS_XMLNS_PREFIX | namespace::NS_XML_PREFIX =>
+ Some(self_error!(this; "Namespace '{}' cannot be default", value)),
+ _ => {
+ this.nst.put(namespace::NS_NO_PREFIX, value.clone());
+ this.into_state_continue(State::InsideOpeningTag(OpeningTagSubstate::InsideTag))
+ }
+ },
+
+ // regular attribute
+ _ => {
+ this.data.attributes.push(OwnedAttribute {
+ name: name.clone(),
+ value: value
+ });
+ this.into_state_continue(State::InsideOpeningTag(OpeningTagSubstate::InsideTag))
+ }
+ }
+ }
+ })
+ }
+ }
+
+}
diff --git a/src/reader/parser/inside_processing_instruction.rs b/src/reader/parser/inside_processing_instruction.rs
new file mode 100644
index 0000000..8ddf6b8
--- /dev/null
+++ b/src/reader/parser/inside_processing_instruction.rs
@@ -0,0 +1,96 @@
+use common::{
+ is_name_start_char, is_name_char,
+};
+
+use reader::events::XmlEvent;
+use reader::lexer::Token;
+
+use super::{Result, PullParser, State, ProcessingInstructionSubstate, DeclarationSubstate};
+
+impl PullParser {
+ pub fn inside_processing_instruction(&mut self, t: Token, s: ProcessingInstructionSubstate) -> Option<Result> {
+ match s {
+ ProcessingInstructionSubstate::PIInsideName => match t {
+ Token::Character(c) if !self.buf_has_data() && is_name_start_char(c) ||
+ self.buf_has_data() && is_name_char(c) => self.append_char_continue(c),
+
+ Token::ProcessingInstructionEnd => {
+ // self.buf contains PI name
+ let name = self.take_buf();
+
+ // Don't need to check for declaration because it has mandatory attributes
+ // but there is none
+ match &name[..] {
+ // Name is empty, it is an error
+ "" => Some(self_error!(self; "Encountered processing instruction without name")),
+
+ // Found <?xml-like PI not at the beginning of a document,
+ // it is an error - see section 2.6 of XML 1.1 spec
+ "xml"|"xmL"|"xMl"|"xML"|"Xml"|"XmL"|"XMl"|"XML" =>
+ Some(self_error!(self; "Invalid processing instruction: <?{}", name)),
+
+ // All is ok, emitting event
+ _ => {
+ self.into_state_emit(
+ State::OutsideTag,
+ Ok(XmlEvent::ProcessingInstruction {
+ name: name,
+ data: None
+ })
+ )
+ }
+ }
+ }
+
+ Token::Whitespace(_) => {
+ // self.buf contains PI name
+ let name = self.take_buf();
+
+ match &name[..] {
+ // We have not ever encountered an element and have not parsed XML declaration
+ "xml" if !self.encountered_element && !self.parsed_declaration =>
+ self.into_state_continue(State::InsideDeclaration(DeclarationSubstate::BeforeVersion)),
+
+ // Found <?xml-like PI after the beginning of a document,
+ // it is an error - see section 2.6 of XML 1.1 spec
+ "xml"|"xmL"|"xMl"|"xML"|"Xml"|"XmL"|"XMl"|"XML"
+ if self.encountered_element || self.parsed_declaration =>
+ Some(self_error!(self; "Invalid processing instruction: <?{}", name)),
+
+ // All is ok, starting parsing PI data
+ _ => {
+ self.lexer.disable_errors(); // data is arbitrary, so disable errors
+ self.data.name = name;
+ self.into_state_continue(State::InsideProcessingInstruction(ProcessingInstructionSubstate::PIInsideData))
+ }
+
+ }
+ }
+
+ _ => Some(self_error!(self; "Unexpected token: <?{}{}", self.buf, t))
+ },
+
+ ProcessingInstructionSubstate::PIInsideData => match t {
+ Token::ProcessingInstructionEnd => {
+ self.lexer.enable_errors();
+ let name = self.data.take_name();
+ let data = self.take_buf();
+ self.into_state_emit(
+ State::OutsideTag,
+ Ok(XmlEvent::ProcessingInstruction {
+ name: name,
+ data: Some(data)
+ })
+ )
+ },
+
+ // Any other token should be treated as plain characters
+ _ => {
+ t.push_to_string(&mut self.buf);
+ None
+ }
+ },
+ }
+ }
+
+}
diff --git a/src/reader/parser/inside_reference.rs b/src/reader/parser/inside_reference.rs
new file mode 100644
index 0000000..60026d5
--- /dev/null
+++ b/src/reader/parser/inside_reference.rs
@@ -0,0 +1,89 @@
+use std::char;
+
+use common::{is_name_start_char, is_name_char, is_whitespace_str};
+
+use reader::lexer::Token;
+
+use super::{Result, PullParser, State};
+
+impl PullParser {
+ pub fn inside_reference(&mut self, t: Token, prev_st: State) -> Option<Result> {
+ match t {
+ Token::Character(c) if !self.data.ref_data.is_empty() && is_name_char(c) ||
+ self.data.ref_data.is_empty() && (is_name_start_char(c) || c == '#') => {
+ self.data.ref_data.push(c);
+ None
+ }
+
+ Token::ReferenceEnd => {
+ // TODO: check for unicode correctness
+ let name = self.data.take_ref_data();
+ let name_len = name.len(); // compute once
+ let c = match &name[..] {
+ "lt" => Ok('<'.to_string()),
+ "gt" => Ok('>'.to_string()),
+ "amp" => Ok('&'.to_string()),
+ "apos" => Ok('\''.to_string()),
+ "quot" => Ok('"'.to_string()),
+ "" => Err(self_error!(self; "Encountered empty entity")),
+ _ if name_len > 2 && name.starts_with("#x") => {
+ let num_str = &name[2..name_len];
+ if num_str == "0" {
+ Err(self_error!(self; "Null character entity is not allowed"))
+ } else {
+ if self.config.replace_unknown_entity_references {
+ match u32::from_str_radix(num_str, 16).ok().map(|i| char::from_u32(i).unwrap_or('\u{fffd}')) {
+ Some(c) => Ok(c.to_string()),
+ None => Err(self_error!(self; "Invalid hexadecimal character number in an entity: {}", name))
+ }
+ } else {
+ match u32::from_str_radix(num_str, 16).ok().and_then(char::from_u32) {
+ Some(c) => Ok(c.to_string()),
+ None => Err(self_error!(self; "Invalid hexadecimal character number in an entity: {}", name))
+ }
+ }
+ }
+ }
+ _ if name_len > 1 && name.starts_with('#') => {
+ let num_str = &name[1..name_len];
+ if num_str == "0" {
+ Err(self_error!(self; "Null character entity is not allowed"))
+ } else {
+ if self.config.replace_unknown_entity_references {
+ match u32::from_str_radix(num_str, 10).ok().map(|i| char::from_u32(i).unwrap_or('\u{fffd}')) {
+ Some(c) => Ok(c.to_string()),
+ None => Err(self_error!(self; "Invalid decimal character number in an entity: {}", name))
+ }
+ }
+ else {
+ match u32::from_str_radix(num_str, 10).ok().and_then(char::from_u32) {
+ Some(c) => Ok(c.to_string()),
+ None => Err(self_error!(self; "Invalid decimal character number in an entity: {}", name))
+ }
+ }
+ }
+ },
+ _ => {
+ if let Some(v) = self.config.extra_entities.get(&name) {
+ Ok(v.clone())
+ } else {
+ Err(self_error!(self; "Unexpected entity: {}", name))
+ }
+ }
+ };
+ match c {
+ Ok(c) => {
+ self.buf.push_str(&c);
+ if prev_st == State::OutsideTag && !is_whitespace_str(&c) {
+ self.inside_whitespace = false;
+ }
+ self.into_state_continue(prev_st)
+ }
+ Err(e) => Some(e)
+ }
+ }
+
+ _ => Some(self_error!(self; "Unexpected token inside an entity: {}", t))
+ }
+ }
+}
diff --git a/src/reader/parser/mod.rs b/src/reader/parser/mod.rs
new file mode 100644
index 0000000..58ca3a6
--- /dev/null
+++ b/src/reader/parser/mod.rs
@@ -0,0 +1,622 @@
+//! Contains an implementation of pull-based XML parser.
+
+use std::mem;
+use std::borrow::Cow;
+use std::io::prelude::*;
+
+use common::{
+ self,
+ XmlVersion, Position, TextPosition,
+ is_name_start_char, is_name_char,
+};
+use name::OwnedName;
+use attribute::OwnedAttribute;
+use namespace::NamespaceStack;
+
+use reader::events::XmlEvent;
+use reader::config::ParserConfig;
+use reader::lexer::{Lexer, Token};
+
+macro_rules! gen_takes(
+ ($($field:ident -> $method:ident, $t:ty, $def:expr);+) => (
+ $(
+ impl MarkupData {
+ #[inline]
+ fn $method(&mut self) -> $t {
+ mem::replace(&mut self.$field, $def)
+ }
+ }
+ )+
+ )
+);
+
+gen_takes!(
+ name -> take_name, String, String::new();
+ ref_data -> take_ref_data, String, String::new();
+
+ version -> take_version, Option<common::XmlVersion>, None;
+ encoding -> take_encoding, Option<String>, None;
+ standalone -> take_standalone, Option<bool>, None;
+
+ element_name -> take_element_name, Option<OwnedName>, None;
+
+ attr_name -> take_attr_name, Option<OwnedName>, None;
+ attributes -> take_attributes, Vec<OwnedAttribute>, vec!()
+);
+
+macro_rules! self_error(
+ ($this:ident; $msg:expr) => ($this.error($msg));
+ ($this:ident; $fmt:expr, $($arg:expr),+) => ($this.error(format!($fmt, $($arg),+)))
+);
+
+mod outside_tag;
+mod inside_processing_instruction;
+mod inside_declaration;
+mod inside_doctype;
+mod inside_opening_tag;
+mod inside_closing_tag_name;
+mod inside_comment;
+mod inside_cdata;
+mod inside_reference;
+
+static DEFAULT_VERSION: XmlVersion = XmlVersion::Version10;
+static DEFAULT_ENCODING: &'static str = "UTF-8";
+static DEFAULT_STANDALONE: Option<bool> = None;
+
+type ElementStack = Vec<OwnedName>;
+pub type Result = super::Result<XmlEvent>;
+
+/// Pull-based XML parser.
+pub struct PullParser {
+ config: ParserConfig,
+ lexer: Lexer,
+ st: State,
+ buf: String,
+ nst: NamespaceStack,
+
+ data: MarkupData,
+ final_result: Option<Result>,
+ next_event: Option<Result>,
+ est: ElementStack,
+ pos: Vec<TextPosition>,
+
+ encountered_element: bool,
+ parsed_declaration: bool,
+ inside_whitespace: bool,
+ read_prefix_separator: bool,
+ pop_namespace: bool
+}
+
+impl PullParser {
+ /// Returns a new parser using the given config.
+ pub fn new(config: ParserConfig) -> PullParser {
+ PullParser {
+ config: config,
+ lexer: Lexer::new(),
+ st: State::OutsideTag,
+ buf: String::new(),
+ nst: NamespaceStack::default(),
+
+ data: MarkupData {
+ name: String::new(),
+ version: None,
+ encoding: None,
+ standalone: None,
+ ref_data: String::new(),
+ element_name: None,
+ quote: None,
+ attr_name: None,
+ attributes: Vec::new()
+ },
+ final_result: None,
+ next_event: None,
+ est: Vec::new(),
+ pos: vec![TextPosition::new()],
+
+ encountered_element: false,
+ parsed_declaration: false,
+ inside_whitespace: true,
+ read_prefix_separator: false,
+ pop_namespace: false
+ }
+ }
+
+ /// Checks if this parser ignores the end of stream errors.
+ pub fn is_ignoring_end_of_stream(&self) -> bool { self.config.ignore_end_of_stream }
+}
+
+impl Position for PullParser {
+ /// Returns the position of the last event produced by the parser
+ #[inline]
+ fn position(&self) -> TextPosition {
+ self.pos[0]
+ }
+}
+
+#[derive(Clone, PartialEq)]
+pub enum State {
+ OutsideTag,
+ InsideOpeningTag(OpeningTagSubstate),
+ InsideClosingTag(ClosingTagSubstate),
+ InsideProcessingInstruction(ProcessingInstructionSubstate),
+ InsideComment,
+ InsideCData,
+ InsideDeclaration(DeclarationSubstate),
+ InsideDoctype,
+ InsideReference(Box<State>)
+}
+
+#[derive(Clone, PartialEq)]
+pub enum OpeningTagSubstate {
+ InsideName,
+
+ InsideTag,
+
+ InsideAttributeName,
+ AfterAttributeName,
+
+ InsideAttributeValue,
+}
+
+#[derive(Clone, PartialEq)]
+pub enum ClosingTagSubstate {
+ CTInsideName,
+ CTAfterName
+}
+
+#[derive(Clone, PartialEq)]
+pub enum ProcessingInstructionSubstate {
+ PIInsideName,
+ PIInsideData
+}
+
+#[derive(Clone, PartialEq)]
+pub enum DeclarationSubstate {
+ BeforeVersion,
+ InsideVersion,
+ AfterVersion,
+
+ InsideVersionValue,
+ AfterVersionValue,
+
+ InsideEncoding,
+ AfterEncoding,
+
+ InsideEncodingValue,
+
+ BeforeStandaloneDecl,
+ InsideStandaloneDecl,
+ AfterStandaloneDecl,
+
+ InsideStandaloneDeclValue,
+ AfterStandaloneDeclValue
+}
+
+#[derive(PartialEq)]
+enum QualifiedNameTarget {
+ AttributeNameTarget,
+ OpeningTagNameTarget,
+ ClosingTagNameTarget
+}
+
+#[derive(Copy, Clone, PartialEq, Eq)]
+enum QuoteToken {
+ SingleQuoteToken,
+ DoubleQuoteToken
+}
+
+impl QuoteToken {
+ fn from_token(t: &Token) -> QuoteToken {
+ match *t {
+ Token::SingleQuote => QuoteToken::SingleQuoteToken,
+ Token::DoubleQuote => QuoteToken::DoubleQuoteToken,
+ _ => panic!("Unexpected token: {}", t)
+ }
+ }
+
+ fn as_token(self) -> Token {
+ match self {
+ QuoteToken::SingleQuoteToken => Token::SingleQuote,
+ QuoteToken::DoubleQuoteToken => Token::DoubleQuote
+ }
+ }
+}
+
+struct MarkupData {
+ name: String, // used for processing instruction name
+ ref_data: String, // used for reference content
+
+ version: Option<common::XmlVersion>, // used for XML declaration version
+ encoding: Option<String>, // used for XML declaration encoding
+ standalone: Option<bool>, // used for XML declaration standalone parameter
+
+ element_name: Option<OwnedName>, // used for element name
+
+ quote: Option<QuoteToken>, // used to hold opening quote for attribute value
+ attr_name: Option<OwnedName>, // used to hold attribute name
+ attributes: Vec<OwnedAttribute> // used to hold all accumulated attributes
+}
+
+impl PullParser {
+ /// Returns next event read from the given buffer.
+ ///
+ /// This method should be always called with the same buffer. If you call it
+ /// providing different buffers each time, the result will be undefined.
+ pub fn next<R: Read>(&mut self, r: &mut R) -> Result {
+ if let Some(ref ev) = self.final_result {
+ return ev.clone();
+ }
+
+ if let Some(ev) = self.next_event.take() {
+ return ev;
+ }
+
+ if self.pop_namespace {
+ self.pop_namespace = false;
+ self.nst.pop();
+ }
+
+ loop {
+ // While lexer gives us Ok(maybe_token) -- we loop.
+ // Upon having a complete XML-event -- we return from the whole function.
+ match self.lexer.next_token(r) {
+ Ok(maybe_token) =>
+ match maybe_token {
+ None => break,
+ Some(token) =>
+ match self.dispatch_token(token) {
+ None => {} // continue
+ Some(Ok(XmlEvent::EndDocument)) =>
+ return {
+ self.next_pos();
+ self.set_final_result(Ok(XmlEvent::EndDocument))
+ },
+ Some(Ok(xml_event)) =>
+ return {
+ self.next_pos();
+ Ok(xml_event)
+ },
+ Some(Err(xml_error)) =>
+ return {
+ self.next_pos();
+ self.set_final_result(Err(xml_error))
+ },
+ }
+ },
+ Err(lexer_error) =>
+ return self.set_final_result(Err(lexer_error)),
+ }
+ }
+
+ // Handle end of stream
+ // Forward pos to the lexer head
+ self.next_pos();
+ let ev = if self.depth() == 0 {
+ if self.encountered_element && self.st == State::OutsideTag { // all is ok
+ Ok(XmlEvent::EndDocument)
+ } else if !self.encountered_element {
+ self_error!(self; "Unexpected end of stream: no root element found")
+ } else { // self.st != State::OutsideTag
+ self_error!(self; "Unexpected end of stream") // TODO: add expected hint?
+ }
+ } else {
+ if self.config.ignore_end_of_stream {
+ self.final_result = None;
+ self.lexer.reset_eof_handled();
+ return self_error!(self; "Unexpected end of stream: still inside the root element");
+ } else {
+ self_error!(self; "Unexpected end of stream: still inside the root element")
+ }
+ };
+ self.set_final_result(ev)
+ }
+
+ // This function is to be called when a terminal event is reached.
+ // The function sets up the `self.final_result` into `Some(result)` and return `result`.
+ fn set_final_result(&mut self, result: Result) -> Result {
+ self.final_result = Some(result.clone());
+ result
+ }
+
+ #[inline]
+ fn error<M: Into<Cow<'static, str>>>(&self, msg: M) -> Result {
+ Err((&self.lexer, msg).into())
+ }
+
+ #[inline]
+ fn next_pos(&mut self) {
+ if self.pos.len() > 1 {
+ self.pos.remove(0);
+ } else {
+ self.pos[0] = self.lexer.position();
+ }
+ }
+
+ #[inline]
+ fn push_pos(&mut self) {
+ self.pos.push(self.lexer.position());
+ }
+
+ fn dispatch_token(&mut self, t: Token) -> Option<Result> {
+ match self.st.clone() {
+ State::OutsideTag => self.outside_tag(t),
+ State::InsideProcessingInstruction(s) => self.inside_processing_instruction(t, s),
+ State::InsideDeclaration(s) => self.inside_declaration(t, s),
+ State::InsideDoctype => self.inside_doctype(t),
+ State::InsideOpeningTag(s) => self.inside_opening_tag(t, s),
+ State::InsideClosingTag(s) => self.inside_closing_tag_name(t, s),
+ State::InsideComment => self.inside_comment(t),
+ State::InsideCData => self.inside_cdata(t),
+ State::InsideReference(s) => self.inside_reference(t, *s)
+ }
+ }
+
+ #[inline]
+ fn depth(&self) -> usize {
+ self.est.len()
+ }
+
+ #[inline]
+ fn buf_has_data(&self) -> bool {
+ self.buf.len() > 0
+ }
+
+ #[inline]
+ fn take_buf(&mut self) -> String {
+ mem::replace(&mut self.buf, String::new())
+ }
+
+ #[inline]
+ fn append_char_continue(&mut self, c: char) -> Option<Result> {
+ self.buf.push(c);
+ None
+ }
+
+ #[inline]
+ fn into_state(&mut self, st: State, ev: Option<Result>) -> Option<Result> {
+ self.st = st;
+ ev
+ }
+
+ #[inline]
+ fn into_state_continue(&mut self, st: State) -> Option<Result> {
+ self.into_state(st, None)
+ }
+
+ #[inline]
+ fn into_state_emit(&mut self, st: State, ev: Result) -> Option<Result> {
+ self.into_state(st, Some(ev))
+ }
+
+ /// Dispatches tokens in order to process qualified name. If qualified name cannot be parsed,
+ /// an error is returned.
+ ///
+ /// # Parameters
+ /// * `t` --- next token;
+ /// * `on_name` --- a callback which is executed when whitespace is encountered.
+ fn read_qualified_name<F>(&mut self, t: Token, target: QualifiedNameTarget, on_name: F) -> Option<Result>
+ where F: Fn(&mut PullParser, Token, OwnedName) -> Option<Result> {
+ // We can get here for the first time only when self.data.name contains zero or one character,
+ // but first character cannot be a colon anyway
+ if self.buf.len() <= 1 {
+ self.read_prefix_separator = false;
+ }
+
+ let invoke_callback = |this: &mut PullParser, t| {
+ let name = this.take_buf();
+ match name.parse() {
+ Ok(name) => on_name(this, t, name),
+ Err(_) => Some(self_error!(this; "Qualified name is invalid: {}", name))
+ }
+ };
+
+ match t {
+ // There can be only one colon, and not as the first character
+ Token::Character(':') if self.buf_has_data() && !self.read_prefix_separator => {
+ self.buf.push(':');
+ self.read_prefix_separator = true;
+ None
+ }
+
+ Token::Character(c) if c != ':' && (!self.buf_has_data() && is_name_start_char(c) ||
+ self.buf_has_data() && is_name_char(c)) =>
+ self.append_char_continue(c),
+
+ Token::EqualsSign if target == QualifiedNameTarget::AttributeNameTarget => invoke_callback(self, t),
+
+ Token::EmptyTagEnd if target == QualifiedNameTarget::OpeningTagNameTarget => invoke_callback(self, t),
+
+ Token::TagEnd if target == QualifiedNameTarget::OpeningTagNameTarget ||
+ target == QualifiedNameTarget::ClosingTagNameTarget => invoke_callback(self, t),
+
+ Token::Whitespace(_) => invoke_callback(self, t),
+
+ _ => Some(self_error!(self; "Unexpected token inside qualified name: {}", t))
+ }
+ }
+
+ /// Dispatches tokens in order to process attribute value.
+ ///
+ /// # Parameters
+ /// * `t` --- next token;
+ /// * `on_value` --- a callback which is called when terminating quote is encountered.
+ fn read_attribute_value<F>(&mut self, t: Token, on_value: F) -> Option<Result>
+ where F: Fn(&mut PullParser, String) -> Option<Result> {
+ match t {
+ Token::Whitespace(_) if self.data.quote.is_none() => None, // skip leading whitespace
+
+ Token::DoubleQuote | Token::SingleQuote => match self.data.quote {
+ None => { // Entered attribute value
+ self.data.quote = Some(QuoteToken::from_token(&t));
+ None
+ }
+ Some(q) if q.as_token() == t => {
+ self.data.quote = None;
+ let value = self.take_buf();
+ on_value(self, value)
+ }
+ _ => {
+ t.push_to_string(&mut self.buf);
+ None
+ }
+ },
+
+ Token::ReferenceStart => {
+ let st = Box::new(self.st.clone());
+ self.into_state_continue(State::InsideReference(st))
+ }
+
+ Token::OpeningTagStart =>
+ Some(self_error!(self; "Unexpected token inside attribute value: <")),
+
+ // Every character except " and ' and < is okay
+ _ => {
+ t.push_to_string(&mut self.buf);
+ None
+ }
+ }
+ }
+
+ fn emit_start_element(&mut self, emit_end_element: bool) -> Option<Result> {
+ let mut name = self.data.take_element_name().unwrap();
+ let mut attributes = self.data.take_attributes();
+
+ // check whether the name prefix is bound and fix its namespace
+ match self.nst.get(name.borrow().prefix_repr()) {
+ Some("") => name.namespace = None, // default namespace
+ Some(ns) => name.namespace = Some(ns.into()),
+ None => return Some(self_error!(self; "Element {} prefix is unbound", name))
+ }
+
+ // check and fix accumulated attributes prefixes
+ for attr in attributes.iter_mut() {
+ if let Some(ref pfx) = attr.name.prefix {
+ let new_ns = match self.nst.get(pfx) {
+ Some("") => None, // default namespace
+ Some(ns) => Some(ns.into()),
+ None => return Some(self_error!(self; "Attribute {} prefix is unbound", attr.name))
+ };
+ attr.name.namespace = new_ns;
+ }
+ }
+
+ if emit_end_element {
+ self.pop_namespace = true;
+ self.next_event = Some(Ok(XmlEvent::EndElement {
+ name: name.clone()
+ }));
+ } else {
+ self.est.push(name.clone());
+ }
+ let namespace = self.nst.squash();
+ self.into_state_emit(State::OutsideTag, Ok(XmlEvent::StartElement {
+ name: name,
+ attributes: attributes,
+ namespace: namespace
+ }))
+ }
+
+ fn emit_end_element(&mut self) -> Option<Result> {
+ let mut name = self.data.take_element_name().unwrap();
+
+ // check whether the name prefix is bound and fix its namespace
+ match self.nst.get(name.borrow().prefix_repr()) {
+ Some("") => name.namespace = None, // default namespace
+ Some(ns) => name.namespace = Some(ns.into()),
+ None => return Some(self_error!(self; "Element {} prefix is unbound", name))
+ }
+
+ let op_name = self.est.pop().unwrap();
+
+ if name == op_name {
+ self.pop_namespace = true;
+ self.into_state_emit(State::OutsideTag, Ok(XmlEvent::EndElement { name: name }))
+ } else {
+ Some(self_error!(self; "Unexpected closing tag: {}, expected {}", name, op_name))
+ }
+ }
+
+}
+
+#[cfg(test)]
+mod tests {
+ use std::io::BufReader;
+
+ use common::{Position, TextPosition};
+ use name::OwnedName;
+ use attribute::OwnedAttribute;
+ use reader::parser::PullParser;
+ use reader::ParserConfig;
+ use reader::events::XmlEvent;
+
+ fn new_parser() -> PullParser {
+ PullParser::new(ParserConfig::new())
+ }
+
+ macro_rules! expect_event(
+ ($r:expr, $p:expr, $t:pat) => (
+ match $p.next(&mut $r) {
+ $t => {}
+ e => panic!("Unexpected event: {:?}", e)
+ }
+ );
+ ($r:expr, $p:expr, $t:pat => $c:expr ) => (
+ match $p.next(&mut $r) {
+ $t if $c => {}
+ e => panic!("Unexpected event: {:?}", e)
+ }
+ )
+ );
+
+ macro_rules! test_data(
+ ($d:expr) => ({
+ static DATA: &'static str = $d;
+ let r = BufReader::new(DATA.as_bytes());
+ let p = new_parser();
+ (r, p)
+ })
+ );
+
+ #[test]
+ fn issue_3_semicolon_in_attribute_value() {
+ let (mut r, mut p) = test_data!(r#"
+ <a attr="zzz;zzz" />
+ "#);
+
+ expect_event!(r, p, Ok(XmlEvent::StartDocument { .. }));
+ expect_event!(r, p, Ok(XmlEvent::StartElement { ref name, ref attributes, ref namespace }) =>
+ *name == OwnedName::local("a") &&
+ attributes.len() == 1 &&
+ attributes[0] == OwnedAttribute::new(OwnedName::local("attr"), "zzz;zzz") &&
+ namespace.is_essentially_empty()
+ );
+ expect_event!(r, p, Ok(XmlEvent::EndElement { ref name }) => *name == OwnedName::local("a"));
+ expect_event!(r, p, Ok(XmlEvent::EndDocument));
+ }
+
+ #[test]
+ fn issue_140_entity_reference_inside_tag() {
+ let (mut r, mut p) = test_data!(r#"
+ <bla>♫</bla>
+ "#);
+
+ expect_event!(r, p, Ok(XmlEvent::StartDocument { .. }));
+ expect_event!(r, p, Ok(XmlEvent::StartElement { ref name, .. }) => *name == OwnedName::local("bla"));
+ expect_event!(r, p, Ok(XmlEvent::Characters(ref s)) => s == "\u{266b}");
+ expect_event!(r, p, Ok(XmlEvent::EndElement { ref name, .. }) => *name == OwnedName::local("bla"));
+ expect_event!(r, p, Ok(XmlEvent::EndDocument));
+ }
+
+ #[test]
+ fn opening_tag_in_attribute_value() {
+ let (mut r, mut p) = test_data!(r#"
+ <a attr="zzz<zzz" />
+ "#);
+
+ expect_event!(r, p, Ok(XmlEvent::StartDocument { .. }));
+ expect_event!(r, p, Err(ref e) =>
+ e.msg() == "Unexpected token inside attribute value: <" &&
+ e.position() == TextPosition { row: 1, column: 24 }
+ );
+ }
+}
diff --git a/src/reader/parser/outside_tag.rs b/src/reader/parser/outside_tag.rs
new file mode 100644
index 0000000..d3f7598
--- /dev/null
+++ b/src/reader/parser/outside_tag.rs
@@ -0,0 +1,130 @@
+use common::is_whitespace_char;
+
+use reader::events::XmlEvent;
+use reader::lexer::Token;
+
+use super::{
+ Result, PullParser, State, ClosingTagSubstate, OpeningTagSubstate,
+ ProcessingInstructionSubstate, DEFAULT_VERSION, DEFAULT_ENCODING, DEFAULT_STANDALONE
+};
+
+impl PullParser {
+ pub fn outside_tag(&mut self, t: Token) -> Option<Result> {
+ match t {
+ Token::ReferenceStart =>
+ self.into_state_continue(State::InsideReference(Box::new(State::OutsideTag))),
+
+ Token::Whitespace(_) if self.depth() == 0 && self.config.ignore_root_level_whitespace => None, // skip whitespace outside of the root element
+
+ Token::Whitespace(_) if self.config.trim_whitespace && !self.buf_has_data() => None,
+
+ Token::Whitespace(c) => {
+ if !self.buf_has_data() {
+ self.push_pos();
+ }
+ self.append_char_continue(c)
+ }
+
+ _ if t.contains_char_data() && self.depth() == 0 =>
+ Some(self_error!(self; "Unexpected characters outside the root element: {}", t)),
+
+ _ if t.contains_char_data() => { // Non-whitespace char data
+ if !self.buf_has_data() {
+ self.push_pos();
+ }
+ self.inside_whitespace = false;
+ t.push_to_string(&mut self.buf);
+ None
+ }
+
+ Token::ReferenceEnd => { // Semi-colon in a text outside an entity
+ self.inside_whitespace = false;
+ Token::ReferenceEnd.push_to_string(&mut self.buf);
+ None
+ }
+
+ Token::CommentStart if self.config.coalesce_characters && self.config.ignore_comments => {
+ // We need to switch the lexer into a comment mode inside comments
+ self.lexer.inside_comment();
+ self.into_state_continue(State::InsideComment)
+ }
+
+ Token::CDataStart if self.config.coalesce_characters && self.config.cdata_to_characters => {
+ if !self.buf_has_data() {
+ self.push_pos();
+ }
+ // We need to disable lexing errors inside CDATA
+ self.lexer.disable_errors();
+ self.into_state_continue(State::InsideCData)
+ }
+
+ _ => {
+ // Encountered some markup event, flush the buffer as characters
+ // or a whitespace
+ let mut next_event = if self.buf_has_data() {
+ let buf = self.take_buf();
+ if self.inside_whitespace && self.config.trim_whitespace {
+ None
+ } else if self.inside_whitespace && !self.config.whitespace_to_characters {
+ Some(Ok(XmlEvent::Whitespace(buf)))
+ } else if self.config.trim_whitespace {
+ Some(Ok(XmlEvent::Characters(buf.trim_matches(is_whitespace_char).into())))
+ } else {
+ Some(Ok(XmlEvent::Characters(buf)))
+ }
+ } else { None };
+ self.inside_whitespace = true; // Reset inside_whitespace flag
+ self.push_pos();
+ match t {
+ Token::ProcessingInstructionStart =>
+ self.into_state(State::InsideProcessingInstruction(ProcessingInstructionSubstate::PIInsideName), next_event),
+
+ Token::DoctypeStart if !self.encountered_element => {
+ // We don't have a doctype event so skip this position
+ // FIXME: update when we have a doctype event
+ self.next_pos();
+ self.lexer.disable_errors();
+ self.into_state(State::InsideDoctype, next_event)
+ }
+
+ Token::OpeningTagStart => {
+ // If declaration was not parsed and we have encountered an element,
+ // emit this declaration as the next event.
+ if !self.parsed_declaration {
+ self.parsed_declaration = true;
+ let sd_event = XmlEvent::StartDocument {
+ version: DEFAULT_VERSION,
+ encoding: DEFAULT_ENCODING.into(),
+ standalone: DEFAULT_STANDALONE
+ };
+ // next_event is always none here because we're outside of
+ // the root element
+ next_event = Some(Ok(sd_event));
+ self.push_pos();
+ }
+ self.encountered_element = true;
+ self.nst.push_empty();
+ self.into_state(State::InsideOpeningTag(OpeningTagSubstate::InsideName), next_event)
+ }
+
+ Token::ClosingTagStart if self.depth() > 0 =>
+ self.into_state(State::InsideClosingTag(ClosingTagSubstate::CTInsideName), next_event),
+
+ Token::CommentStart => {
+ // We need to switch the lexer into a comment mode inside comments
+ self.lexer.inside_comment();
+ self.into_state(State::InsideComment, next_event)
+ }
+
+ Token::CDataStart => {
+ // We need to disable lexing errors inside CDATA
+ self.lexer.disable_errors();
+ self.into_state(State::InsideCData, next_event)
+ }
+
+ _ => Some(self_error!(self; "Unexpected token: {}", t))
+ }
+ }
+ }
+ }
+}