From 3263b73718258316d08c932a6272da306c49c582 Mon Sep 17 00:00:00 2001
From: Dylan Owen <dylan.owen@workday.com>
Date: Fri, 28 Jun 2019 14:57:20 -0700
Subject: [PATCH] Fixed a tokenizer edge case for strings

---
 src/tokenizer.rs | 14 +++++++++++---
 1 file changed, 11 insertions(+), 3 deletions(-)

diff --git a/src/tokenizer.rs b/src/tokenizer.rs
index e70dea1..eb5d9c2 100644
--- a/src/tokenizer.rs
+++ b/src/tokenizer.rs
@@ -250,12 +250,12 @@ impl<'a> TokenStream<'a> {
                         )
                     )
                 } else {
-                    let mut prev_char = cur_char;
                     let mut nchars = 1;
+                    let mut escaped = false;
                     for (idx, cur_char) in iter {
                         nchars += 1;
                         match cur_char {
-                            '"' if prev_char == '\\' => {}
+                            '"' if escaped => {}
                             '"' => {
                                 self.position.column += nchars;
                                 self.off += idx+1;
@@ -268,11 +268,14 @@ impl<'a> TokenStream<'a> {
                                     )
                                 );
                             }
+
                             _ => {
 
                             }
                         }
-                        prev_char = cur_char;
+
+                        // if we aren't escaped and the current char is a \, we are now escaped
+                        escaped = !escaped && cur_char == '\\';
                     }
                     Err(
                         Error::unexpected_message(
@@ -482,12 +485,17 @@ mod test {
     #[test] #[should_panic] fn letters_float2() { tok_str("0.bbc"); }
     #[test] #[should_panic] fn letters_float3() { tok_str("0.bbce0"); }
     #[test] #[should_panic] fn no_exp_sign_float() { tok_str("0e0"); }
+    #[test] #[should_panic] fn unterminated_string() { tok_str(r#""hello\""#); }
+    #[test] #[should_panic] fn extra_unterminated_string() { tok_str(r#""hello\\\""#); }
 
     #[test]
     fn string() {
         assert_eq!(tok_str(r#""""#), [r#""""#]);
         assert_eq!(tok_typ(r#""""#), [StringValue]);
         assert_eq!(tok_str(r#""hello""#), [r#""hello""#]);
+        assert_eq!(tok_str(r#""hello\\""#), [r#""hello\\""#]);
+        assert_eq!(tok_str(r#""hello\\\\""#), [r#""hello\\\\""#]);
+        assert_eq!(tok_str(r#""he\\llo""#), [r#""he\\llo""#]);
         assert_eq!(tok_typ(r#""hello""#), [StringValue]);
         assert_eq!(tok_str(r#""my\"quote""#), [r#""my\"quote""#]);
         assert_eq!(tok_typ(r#""my\"quote""#), [StringValue]);