lexer tests: makefile/configure

Makefile.in

@@ -252,6 +252,19 @@ ifneq ($(findstring clean,$(MAKECMDGOALS)),)
   include $(CFG_SRC_DIR)mk/clean.mk
 endif
 
+# Grammar tests
+
+ifneq ($(findstring lexer,$(MAKECMDGOALS)),)
+  ifdef CFG_JAVAC
+	ifdef CFG_ANTLR4
+	  ifdef CFG_GRUN
+	    CFG_INFO := $(info cfg: including grammar tests)
+	    include $(CFG_SRC_DIR)mk/grammar.mk
+	  endif
+	endif
+  endif
+endif
+
 # CTAGS building
 ifneq ($(strip $(findstring TAGS.emacs,$(MAKECMDGOALS)) \
                $(findstring TAGS.vi,$(MAKECMDGOALS))),)

configure

@@ -493,6 +493,9 @@ probe CFG_VALGRIND         valgrind
 probe CFG_PERF             perf
 probe CFG_ISCC             iscc
 probe CFG_LLNEXTGEN        LLnextgen
+probe CFG_JAVAC            javac
+probe CFG_ANTLR4           antlr4
+probe CFG_GRUN             grun
 probe CFG_PANDOC           pandoc
 probe CFG_PDFLATEX         pdflatex
 probe CFG_XELATEX          xelatex

mk/grammar.mk

+# Copyright 2014 The Rust Project Developers. See the COPYRIGHT
+# file at the top-level directory of this distribution and at
+# http://rust-lang.org/COPYRIGHT.
+#
+# Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+# http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+# <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+# option. This file may not be copied, modified, or distributed
+# except according to those terms.
+
+BG = $(CFG_BUILD_DIR)/grammar/
+SG = $(S)src/grammar/
+B = $(CFG_BUILD_DIR)/$(CFG_BUILD)/stage2/
+L = $(B)lib/rustlib/$(CFG_BUILD)/lib
+LD = $(CFG_BUILD)/stage2/lib/rustlib/$(CFG_BUILD)/lib/
+RUSTC = $(B)bin/rustc
+
+# Run the reference lexer against libsyntax and compare the tokens and spans.
+# If "// ignore-lexer-test" is present in the file, it will be ignored.
+#
+# $(1) is the file to test.
+define LEXER_TEST
+grep "// ignore-lexer-test" $(1) ; \
+  if [ $$? -eq 1 ]; then \
+   CLASSPATH=$(B)grammar $(CFG_GRUN) RustLexer tokens -tokens < $(1) \
+   | $(B)grammar/verify $(1) ; \
+  fi
+endef
+
+$(BG):
+	$(Q)mkdir -p $(BG)
+
+$(BG)RustLexer.class: $(SG)RustLexer.g4
+	$(Q)$(CFG_ANTLR4) -o $(B)grammar $(SG)RustLexer.g4
+	$(Q)$(CFG_JAVAC) -d $(BG) $(BG)RustLexer.java
+
+$(BG)verify: $(SG)verify.rs rustc-stage2-H-$(CFG_BUILD) $(LD)stamp.regex_macros $(LD)stamp.rustc
+	$(Q)$(RUSTC) -O --out-dir $(BG) -L $(L) $(SG)verify.rs
+
+check-lexer: $(BG) $(BG)RustLexer.class $(BG)verify
+	$(info Verifying libsyntax against the reference lexer ...)
+	$(Q)find $(S) -iname '*.rs' -exec "$(SG)check.sh" {} "$(BG)" \
+      "$(CFG_GRUN)" "$(BG)verify" "$(BG)RustLexer.tokens" "$(VERBOSE)" \;

src/grammar/check.sh

+#!/bin/sh
+
+# Run the reference lexer against libsyntax and compare the tokens and spans.
+# If "// ignore-lexer-test" is present in the file, it will be ignored.
+#
+# Argument $1 is the file to check, $2 is the classpath to use, $3 is the path
+# to the grun binary, $4 is the path to the verify binary, $5 is the path to
+# RustLexer.tokens
+
+if [ "${VERBOSE}" == "1" ]; then
+    set -x
+fi
+
+grep -q "// ignore lexer-test" $1;
+
+if [ $? -eq 1 ]; then
+    cd $2 # This `cd` is so java will pick up RustLexer.class. I couldn't
+    # figure out how to wrangle the CLASSPATH, just adding build/grammr didn't
+    # seem to have anny effect.
+    $3 RustLexer tokens -tokens < $1 | $4 $1 $5
+fi

src/grammar/verify.rs

@@ -223,11 +223,13 @@ fn next(r: &mut lexer::StringReader) -> TokenAndSpan {
         r.next_token()
     }
 
-    let token_map = parse_token_list(File::open(&Path::new("RustLexer.tokens")).unwrap().read_to_string().unwrap().as_slice());
+    let args = std::os::args();
+
+    let token_map = parse_token_list(File::open(&Path::new(args.get(2).as_slice())).unwrap().read_to_string().unwrap().as_slice());
     let mut stdin = std::io::stdin();
     let mut antlr_tokens = stdin.lines().map(|l| parse_antlr_token(l.unwrap().as_slice().trim(), &token_map));
 
-    let code = File::open(&Path::new(std::os::args().get(1).as_slice())).unwrap().read_to_string().unwrap();
+    let code = File::open(&Path::new(args.get(1).as_slice())).unwrap().read_to_string().unwrap();
     let options = config::basic_options();
     let session = session::build_session(options, None,
                                          syntax::diagnostics::registry::Registry::new([]));