From b4714cdf6e9caa0fc312b93917fe721f9c549426 Mon Sep 17 00:00:00 2001
From: Vadim Petrochenkov <vadim.petrochenkov@gmail.com>
Date: Sun, 20 May 2018 22:31:27 +0300
Subject: [PATCH] lexer: Fix span override for the first token in a string

---
 src/librustdoc/html/highlight.rs      |  4 ++--
 src/libsyntax/parse/lexer/comments.rs |  2 +-
 src/libsyntax/parse/lexer/mod.rs      | 19 +++++++++++--------
 src/libsyntax/parse/mod.rs            |  3 +--
 4 files changed, 15 insertions(+), 13 deletions(-)

diff --git a/src/librustdoc/html/highlight.rs b/src/librustdoc/html/highlight.rs
index cff89b03e3d..3d3d46555c3 100644
--- a/src/librustdoc/html/highlight.rs
+++ b/src/librustdoc/html/highlight.rs
@@ -48,7 +48,7 @@ pub fn render_with_highlighting(src: &str, class: Option<&str>, id: Option<&str>
     }
     write_header(class, id, &mut out).unwrap();
 
-    let mut classifier = Classifier::new(lexer::StringReader::new(&sess, fm), sess.codemap());
+    let mut classifier = Classifier::new(lexer::StringReader::new(&sess, fm, None), sess.codemap());
     if let Err(_) = classifier.write_source(&mut out) {
         return format!("<pre>{}</pre>", src);
     }
@@ -68,7 +68,7 @@ pub fn render_inner_with_highlighting(src: &str) -> io::Result<String> {
     let fm = sess.codemap().new_filemap(FileName::Custom("stdin".to_string()), src.to_string());
 
     let mut out = Vec::new();
-    let mut classifier = Classifier::new(lexer::StringReader::new(&sess, fm), sess.codemap());
+    let mut classifier = Classifier::new(lexer::StringReader::new(&sess, fm, None), sess.codemap());
     classifier.write_source(&mut out)?;
 
     Ok(String::from_utf8_lossy(&out).into_owned())
diff --git a/src/libsyntax/parse/lexer/comments.rs b/src/libsyntax/parse/lexer/comments.rs
index 672b0b9bbd1..7ead1ceeed0 100644
--- a/src/libsyntax/parse/lexer/comments.rs
+++ b/src/libsyntax/parse/lexer/comments.rs
@@ -362,7 +362,7 @@ pub fn gather_comments_and_literals(sess: &ParseSess, path: FileName, srdr: &mut
     let src = String::from_utf8(src).unwrap();
     let cm = CodeMap::new(sess.codemap().path_mapping().clone());
     let filemap = cm.new_filemap(path, src);
-    let mut rdr = lexer::StringReader::new_raw(sess, filemap);
+    let mut rdr = lexer::StringReader::new_raw(sess, filemap, None);
 
     let mut comments: Vec<Comment> = Vec::new();
     let mut literals: Vec<Literal> = Vec::new();
diff --git a/src/libsyntax/parse/lexer/mod.rs b/src/libsyntax/parse/lexer/mod.rs
index bbece1ee5e3..1d7f20e69e7 100644
--- a/src/libsyntax/parse/lexer/mod.rs
+++ b/src/libsyntax/parse/lexer/mod.rs
@@ -155,13 +155,15 @@ pub fn peek(&self) -> TokenAndSpan {
 
 impl<'a> StringReader<'a> {
     /// For comments.rs, which hackily pokes into next_pos and ch
-    pub fn new_raw(sess: &'a ParseSess, filemap: Lrc<syntax_pos::FileMap>) -> Self {
-        let mut sr = StringReader::new_raw_internal(sess, filemap);
+    pub fn new_raw(sess: &'a ParseSess, filemap: Lrc<syntax_pos::FileMap>,
+                   override_span: Option<Span>) -> Self {
+        let mut sr = StringReader::new_raw_internal(sess, filemap, override_span);
         sr.bump();
         sr
     }
 
-    fn new_raw_internal(sess: &'a ParseSess, filemap: Lrc<syntax_pos::FileMap>) -> Self {
+    fn new_raw_internal(sess: &'a ParseSess, filemap: Lrc<syntax_pos::FileMap>,
+                        override_span: Option<Span>) -> Self {
         if filemap.src.is_none() {
             sess.span_diagnostic.bug(&format!("Cannot lex filemap without source: {}",
                                               filemap.name));
@@ -185,12 +187,13 @@ fn new_raw_internal(sess: &'a ParseSess, filemap: Lrc<syntax_pos::FileMap>) -> S
             token: token::Eof,
             span: syntax_pos::DUMMY_SP,
             open_braces: Vec::new(),
-            override_span: None,
+            override_span,
         }
     }
 
-    pub fn new(sess: &'a ParseSess, filemap: Lrc<syntax_pos::FileMap>) -> Self {
-        let mut sr = StringReader::new_raw(sess, filemap);
+    pub fn new(sess: &'a ParseSess, filemap: Lrc<syntax_pos::FileMap>, override_span: Option<Span>)
+               -> Self {
+        let mut sr = StringReader::new_raw(sess, filemap, override_span);
         if sr.advance_token().is_err() {
             sr.emit_fatal_errors();
             FatalError.raise();
@@ -207,7 +210,7 @@ pub fn retokenize(sess: &'a ParseSess, mut span: Span) -> Self {
             span = span.shrink_to_lo();
         }
 
-        let mut sr = StringReader::new_raw_internal(sess, begin.fm);
+        let mut sr = StringReader::new_raw_internal(sess, begin.fm, None);
 
         // Seek the lexer to the right byte range.
         sr.save_new_lines_and_multibyte = false;
@@ -1795,7 +1798,7 @@ fn setup<'a>(cm: &CodeMap,
                  teststr: String)
                  -> StringReader<'a> {
         let fm = cm.new_filemap(PathBuf::from("zebra.rs").into(), teststr);
-        StringReader::new(sess, fm)
+        StringReader::new(sess, fm, None)
     }
 
     #[test]
diff --git a/src/libsyntax/parse/mod.rs b/src/libsyntax/parse/mod.rs
index 0abedb99bd0..25490829f9e 100644
--- a/src/libsyntax/parse/mod.rs
+++ b/src/libsyntax/parse/mod.rs
@@ -229,8 +229,7 @@ fn file_to_filemap(sess: &ParseSess, path: &Path, spanopt: Option<Span>)
 /// Given a filemap, produce a sequence of token-trees
 pub fn filemap_to_stream(sess: &ParseSess, filemap: Lrc<FileMap>, override_span: Option<Span>)
                          -> TokenStream {
-    let mut srdr = lexer::StringReader::new(sess, filemap);
-    srdr.override_span = override_span;
+    let mut srdr = lexer::StringReader::new(sess, filemap, override_span);
     srdr.real_token();
     panictry!(srdr.parse_all_token_trees())
 }
-- 
GitLab