Switch tokenizer to use a pointer rather than index for position tracking.

Fixes #210. Posted for posterity since it seems a regression at least in
some of our benchmarks, and would need more work.

Author: emilio

src/parser.rs

@@ -16,8 +16,8 @@ use std::ops::Range;
 /// Should only be used with the `Parser` instance it came from.
 #[derive(Debug, Clone)]
 pub struct ParserState {
-    pub(crate) position: usize,
-    pub(crate) current_line_start_position: usize,
+    pub(crate) position: SourcePosition,
+    pub(crate) current_line_start_position: SourcePosition,
     pub(crate) current_line_number: u32,
     pub(crate) at_start_of: Option<BlockType>,
 }
@@ -26,15 +26,15 @@ impl ParserState {
     /// The position from the start of the input, counted in UTF-8 bytes.
     #[inline]
     pub fn position(&self) -> SourcePosition {
-        SourcePosition(self.position)
+        self.position
     }
 
     /// The line number and column number
     #[inline]
     pub fn source_location(&self) -> SourceLocation {
         SourceLocation {
             line: self.current_line_number,
-            column: (self.position - self.current_line_start_position + 1) as u32,
+            column: (self.position.0 - self.current_line_start_position.0 + 1) as u32,
         }
     }
 }

src/size_of_tests.rs

@@ -42,8 +42,8 @@ size_of_test!(token, Token, 32);
 size_of_test!(std_cow_str, std::borrow::Cow<'static, str>, 24, 32);
 size_of_test!(cow_rc_str, CowRcStr, 16);
 
-size_of_test!(tokenizer, crate::tokenizer::Tokenizer, 72);
-size_of_test!(parser_input, crate::parser::ParserInput, 136);
+size_of_test!(tokenizer, crate::tokenizer::Tokenizer, 80);
+size_of_test!(parser_input, crate::parser::ParserInput, 144);
 size_of_test!(parser, crate::parser::Parser, 16);
 size_of_test!(source_position, crate::SourcePosition, 8);
 size_of_test!(parser_state, crate::ParserState, 24);