Use latest ruby_parser

fixes most parsing issues, but will use parser
based on Ruby version used to run Brakeman

lib/brakeman/processors/controller_processor.rb

-require 'ruby_parser'
 require 'brakeman/processors/base_processor'
 
 #Processes controller. Results are put in tracker.controllers

lib/brakeman/scanner.rb

 require 'rubygems'
 begin
-  require 'ruby_parser'
+  #Load our own version of ruby_parser :'(
+  require 'ruby_parser/ruby_parser.rb'
+
   require 'haml'
   require 'sass'
   require 'erb'
   require 'erubis'
   require 'brakeman/processor'
-
-  #Load our own version of ruby_parser :(
-  original_verbosity = $VERBOSE
-  $VERBOSE = nil
-  require 'ruby_parser/ruby_parser.rb'
-  $VERBOSE = original_verbosity
-
 rescue LoadError => e
   $stderr.puts e.message
   $stderr.puts "Please install the appropriate dependency."
@@ -40,6 +35,12 @@ class Brakeman::Scanner
     @path = options[:app_path]
     @app_path = File.join(@path, "app")
     @processor = Brakeman::Processor.new options
+
+    if RUBY_1_9
+      @ruby_parser = ::Ruby19Parser
+    else
+      @ruby_parser = ::Ruby18Parser
+    end
   end
 
   #Returns the Tracker generated from the scan
@@ -75,13 +76,13 @@ class Brakeman::Scanner
   #Stores parsed information in tracker.config
   def process_config
     if options[:rails3]
-      @processor.process_config(RubyParser.new.parse(File.read("#@path/config/application.rb")))
-      @processor.process_config(RubyParser.new.parse(File.read("#@path/config/environments/production.rb")))
+      @processor.process_config(parse_ruby(File.read("#@path/config/application.rb")))
+      @processor.process_config(parse_ruby(File.read("#@path/config/environments/production.rb")))
     else
-      @processor.process_config(RubyParser.new.parse(File.read("#@path/config/environment.rb")))
+      @processor.process_config(parse_ruby(File.read("#@path/config/environment.rb")))
 
       if File.exists? "#@path/config/gems.rb"
-        @processor.process_config(RubyParser.new.parse(File.read("#@path/config/gems.rb")))
+        @processor.process_config(parse_ruby(File.read("#@path/config/gems.rb")))
       end
 
     end
@@ -99,9 +100,9 @@ class Brakeman::Scanner
   def process_gems
     if File.exists? "#@path/Gemfile"
       if File.exists? "#@path/Gemfile.lock"
-        @processor.process_gems(RubyParser.new.parse(File.read("#@path/Gemfile")), File.read("#@path/Gemfile.lock"))
+        @processor.process_gems(parse_ruby(File.read("#@path/Gemfile")), File.read("#@path/Gemfile.lock"))
       else
-        @processor.process_gems(RubyParser.new.parse(File.read("#@path/Gemfile")))
+        @processor.process_gems(parse_ruby(File.read("#@path/Gemfile")))
       end
     end
   end
@@ -112,7 +113,7 @@ class Brakeman::Scanner
   def process_initializers
     Dir.glob(@path + "/config/initializers/**/*.rb").sort.each do |f|
       begin
-        @processor.process_initializer(f, RubyParser.new.parse(File.read(f)))
+        @processor.process_initializer(f, parse_ruby(File.read(f)))
       rescue Racc::ParseError => e
         tracker.error e, "could not parse #{f}. There is probably a typo in the file. Test it with 'ruby_parse #{f}'"
       rescue Exception => e
@@ -132,7 +133,7 @@ class Brakeman::Scanner
 
     Dir.glob(@path + "/lib/**/*.rb").sort.each do |f|
       begin
-        @processor.process_lib RubyParser.new.parse(File.read(f)), f
+        @processor.process_lib parse_ruby(File.read(f)), f
       rescue Racc::ParseError => e
         tracker.error e, "could not parse #{f}. There is probably a typo in the file. Test it with 'ruby_parse #{f}'"
       rescue Exception => e
@@ -147,7 +148,7 @@ class Brakeman::Scanner
   def process_routes
     if File.exists? "#@path/config/routes.rb"
       begin
-        @processor.process_routes RubyParser.new.parse(File.read("#@path/config/routes.rb"))
+        @processor.process_routes parse_ruby(File.read("#@path/config/routes.rb"))
       rescue Exception => e
         tracker.error e.exception(e.message + "\nWhile processing routes.rb"), e.backtrace
         warn "[Notice] Error while processing routes - assuming all public controller methods are actions."
@@ -164,7 +165,7 @@ class Brakeman::Scanner
   def process_controllers
     Dir.glob(@app_path + "/controllers/**/*.rb").sort.each do |f|
       begin
-        @processor.process_controller(RubyParser.new.parse(File.read(f)), f)
+        @processor.process_controller(parse_ruby(File.read(f)), f)
       rescue Racc::ParseError => e
         tracker.error e, "could not parse #{f}. There is probably a typo in the file. Test it with 'ruby_parse #{f}'"
       rescue Exception => e
@@ -210,11 +211,11 @@ class Brakeman::Scanner
             src.sub!(/^#.*\n/, '') if RUBY_1_9
           end
 
-          parsed = RubyParser.new.parse src
+          parsed = parse_ruby src
         elsif type == :haml
           src = Haml::Engine.new(text,
                                  :escape_html => !!tracker.config[:escape_html]).precompiled
-          parsed = RubyParser.new.parse src
+          parsed = parse_ruby src
         else
           tracker.error "Unkown template type in #{f}"
         end
@@ -251,7 +252,7 @@ class Brakeman::Scanner
   def process_models
     Dir.glob(@app_path + "/models/*.rb").sort.each do |f|
       begin
-        @processor.process_model(RubyParser.new.parse(File.read(f)), f)
+        @processor.process_model(parse_ruby(File.read(f)), f)
       rescue Racc::ParseError => e
         tracker.error e, "could not parse #{f}"
       rescue Exception => e
@@ -263,6 +264,14 @@ class Brakeman::Scanner
   def index_call_sites
     tracker.index_call_sites
   end
+
+  def parse_ruby input
+    if RUBY_1_9
+      Ruby19Parser.new.parse input
+    else
+      Ruby18Parser.new.parse input
+    end
+  end
 end
 
 #This is from Rails 3 version of the Erubis handler

lib/ruby_parser/ruby_lexer.rb

@@ -6,6 +6,12 @@ class RubyLexer
 
   ESC_RE = /\\([0-7]{1,3}|x[0-9a-fA-F]{1,2}|M-[^\\]|(C-|c)[^\\]|[^0-7xMCc])/
 
+  ##
+  # What version of ruby to parse. 18 and 19 are the only valid values
+  # currently supported.
+
+  attr_accessor :version
+
   # Additional context surrounding tokens that both the lexer and
   # grammar use.
   attr_reader :lex_state
@@ -58,6 +64,7 @@ class RubyLexer
     "===" => :tEQQ,
     "=>"  => :tASSOC,
     "=~"  => :tMATCH,
+    "->"  => :tLAMBDA,
   }
 
   # How the parser advances to the next token.
@@ -130,7 +137,7 @@ class RubyLexer
         string_buffer << '#'
       end
 
-      until src.scan(eos_re) do
+      until src.check(eos_re) do
         c = tokadd_string func, "\n", nil
 
         rb_compile_error err_msg if
@@ -146,9 +153,6 @@ class RubyLexer
         rb_compile_error err_msg if
           src.eos?
       end
-
-      # tack on a NL after the heredoc token - FIX NL should not be needed
-      src.unread_many(eos + "\n") # TODO: remove this... stupid stupid stupid
     else
       until src.check(eos_re) do
         string_buffer << src.scan(/.*(\n|\z)/)
@@ -195,12 +199,10 @@ class RubyLexer
       return nil
     end
 
-    if src.check(/.*\n/) then
+    if src.scan(/.*\n/) then
       # TODO: think about storing off the char range instead
-      line = src.string[src.pos, src.matched_size]
-      src.string[src.pos, src.matched_size] = "\n"
+      line = src.matched
       src.extra_lines_added += 1
-      src.pos += 1
     else
       line = nil
     end
@@ -216,7 +218,8 @@ class RubyLexer
     end
   end
 
-  def initialize
+  def initialize v = 18
+    self.version = v
     self.cond = RubyParser::StackState.new(:cond)
     self.cmdarg = RubyParser::StackState.new(:cmdarg)
     self.nest = 0
@@ -702,7 +705,6 @@ class RubyLexer
           end
         elsif src.scan(/\(/) then
           result = :tLPAREN2
-          self.command_start = true
 
           if lex_state == :expr_beg || lex_state == :expr_mid then
             result = :tLPAREN
@@ -838,6 +840,12 @@ class RubyLexer
             return :tPIPE
           end
         elsif src.scan(/\{/) then
+          if defined?(@hack_expects_lambda) && @hack_expects_lambda
+            @hack_expects_lambda = false
+            self.lex_state = :expr_beg
+            return :tLAMBEG
+          end
+
           result = if lex_state.is_argument || lex_state == :expr_end then
                      :tLCURLY      #  block (primary)
                    elsif lex_state == :expr_endarg then
@@ -850,6 +858,10 @@ class RubyLexer
           self.command_start = true unless result == :tLBRACE
 
           return result
+        elsif src.scan(/->/) then
+          @hack_expects_lambda = true
+          self.lex_state = :expr_arg
+          return :tLAMBDA
         elsif src.scan(/[+-]/) then
           sign = src.matched
           utype, type = if sign == "+" then
@@ -1033,8 +1045,14 @@ class RubyLexer
                 src.getch
               end
           self.lex_state = :expr_end
-          self.yacc_value = c[0].ord & 0xff
-          return :tINTEGER
+
+          if version == 18 then
+            self.yacc_value = c[0].ord & 0xff
+            return :tINTEGER
+          else
+            self.yacc_value = c
+            return :tSTRING
+          end
         elsif src.check(/\&/) then
           if src.scan(/\&\&\=/) then
             self.yacc_value = "&&"
@@ -1236,13 +1254,6 @@ class RubyLexer
           end
         end
 
-        if src.scan(/:(?!:)/)
-            result = :tHASHKEY
-            token << src.matched
-            self.yacc_value = token
-            return result
-        end
-
         result ||= if token =~ /^[A-Z]/ then
                      :tCONSTANT
                    else
@@ -1250,6 +1261,20 @@ class RubyLexer
                    end
       end
 
+      if (lex_state == :expr_beg && !command_state) || lex_state == :expr_arg || lex_state == :expr_cmdarg
+        colon = src.scan(/:/)
+
+        if colon && src.peek(1) != ":"
+          src.unscan
+          self.lex_state = :expr_beg
+          src.scan(/:/)
+          self.yacc_value = [token, src.lineno]
+          return :tLABEL
+        end
+
+        src.unscan if colon
+      end
+
       unless lex_state == :expr_dot then
         # See if it is a reserved word.
         keyword = RubyParser::Keyword.keyword token
@@ -1269,6 +1294,10 @@ class RubyLexer
             return :kDO_COND  if cond.is_in_state
             return :kDO_BLOCK if cmdarg.is_in_state && state != :expr_cmdarg
             return :kDO_BLOCK if state == :expr_endarg
+            if defined?(@hack_expects_lambda) && @hack_expects_lambda
+              @hack_expects_lambda = false
+              return :kDO_LAMBDA
+            end
             return :kDO
           end
 

test/tests/test_brakeman.rb

@@ -4,10 +4,10 @@ class UtilTests < Test::Unit::TestCase
   end
 
   def test_cookies?
-    assert util.cookies?(RubyParser.new.parse 'cookies[:x][:y][:z]')
+    assert util.cookies?(Ruby18Parser.new.parse 'cookies[:x][:y][:z]')
   end
 
   def test_params?
-    assert util.params?(RubyParser.new.parse 'params[:x][:y][:z]')
+    assert util.params?(Ruby18Parser.new.parse 'params[:x][:y][:z]')
   end
 end