Fix a number of errors in the config.gem mechanism.

* Rails::GemDependency was missing definitions for hash and eql?, causing Array#uniq to not work.

* If several versions of a gem are unpacked in vendor, now chooses the highest if no version is specified.

* streamlined add_load_path. Now sets up Rubygems correctly to allow 'gem' to find frozen gems, with
  gems frozen to vendor/gems and specifications in vendor/gems/<gem-name>/.specification

* Rails::GemDependency#specification would return a spec for the highest installed version, even for
  frozen gems and/or previously loaded lower versions. See in part ticket #1123.

* removed vendor from default_load_paths - it was causing autoloading to append Gems::Gems::<gem-dir> to
  constant names

* added additional tests for loading frozen gems.

* incorporates the fix from #1107 for vendor rails

* defers to freeze:gems for handling the Rails framework. gems:unpack WILL NOT place a copy of Rails
  in vendor/gems. Should close #1123 completely.

* incorporates, via using the gem loader for frozen gems, fixes corresponding to #227, #324, #362, #527, and #742.

* gem plugins now work the same whether frozen or not. Correctness of the behavior is a matter for another ticket...
Signed-off-by: Nrick <technoweenie@gmail.com>

railties/lib/initializer.rb

@@ -212,6 +212,7 @@ def install_gem_spec_stubs
           Gem.loaded_specs[stub] = Gem::Specification.new do |s|
             s.name = stub
             s.version = Rails::VERSION::STRING
+            s.loaded_from = ""
           end
         end
       end
@@ -878,7 +879,6 @@ def default_load_paths
           components
           config
           lib
-          vendor
         ).map { |dir| "#{root_path}/#{dir}" }.select { |dir| File.directory?(dir) }
 
         paths.concat builtin_directories

railties/lib/rails/gem_builder.rb

@@ -12,10 +12,10 @@ def initialize(spec, gem_dir)
       @spec    = spec
       @gem_dir = gem_dir
     end
-    
+
     # silence the underlying builder
     def say(message)
     end
-    
+
   end
-end
\ No newline at end of file
+end

railties/lib/rails/gem_dependency.rb

+require 'rails/vendor_gem_source_index'
+
+module Gem
+  def self.source_index=(index)
+    @@source_index = index
+  end
+end
+
 module Rails
   class GemDependency
-    attr_accessor :name, :requirement, :version, :lib, :source
+    attr_accessor :lib, :source
 
     def self.unpacked_path
       @unpacked_path ||= File.join(RAILS_ROOT, 'vendor', 'gems')
     end
 
+    @@framework_gems = {}
+
+    def self.add_frozen_gem_path
+      @@paths_loaded ||= begin
+        Gem.source_index = Rails::VendorGemSourceIndex.new(Gem.source_index)
+        # loaded before us - we can't change them, so mark them
+        Gem.loaded_specs.each do |name, spec|
+          @@framework_gems[name] = spec
+        end
+      end
+    end
+
+    def framework_gem?
+      @@framework_gems.has_key?(name)
+    end
+
+    def vendor_rails?
+      Gem.loaded_specs.has_key?(name) && Gem.loaded_specs[name].loaded_from.empty?
+    end
+
+    def vendor_gem?
+      Gem.loaded_specs.has_key?(name) && Gem.loaded_specs[name].loaded_from.include?(self.class.unpacked_path)
+    end
+
     def initialize(name, options = {})
       require 'rubygems' unless Object.const_defined?(:Gem)
 
       if options[:requirement]
-        @requirement = options[:requirement]
+        req = options[:requirement]
       elsif options[:version]
-        @requirement = Gem::Requirement.create(options[:version])
+        req = Gem::Requirement.create(options[:version])
+      else
+        req = Gem::Requirement.default
       end
 
-      @version  = @requirement.instance_variable_get("@requirements").first.last if @requirement
-      @name     = name.to_s
+      @dep = Gem::Dependency.new(name, req)
       @lib      = options[:lib]
       @source   = options[:source]
       @loaded   = @frozen = @load_paths_added = false
-      @unpack_directory = nil
-    end
-
-    def unpacked_paths
-      Dir[File.join(self.class.unpacked_path, "#{@name}-#{@version || "*"}")]
     end
 
     def add_load_paths
+      self.class.add_frozen_gem_path
       return if @loaded || @load_paths_added
-      unpacked_paths = self.unpacked_paths
-      if unpacked_paths.empty?
-        args = [@name]
-        args << @requirement.to_s if @requirement
-        gem *args
-      else
-        $LOAD_PATH.unshift File.join(unpacked_paths.first, 'lib')
-        ext = File.join(unpacked_paths.first, 'ext')
-        $LOAD_PATH.unshift(ext) if File.exist?(ext)
-        @frozen = true
+      if framework_gem?
+        @load_paths_added = @loaded = @frozen = true
+        return
       end
+      gem @dep
+      @spec = Gem.loaded_specs[name]
+      @frozen = @spec.loaded_from.include?(self.class.unpacked_path) if @spec
       @load_paths_added = true
     rescue Gem::LoadError
     end
 
     def dependencies
+      return [] if framework_gem?
       all_dependencies = specification.dependencies.map do |dependency|
         GemDependency.new(dependency.name, :requirement => dependency.version_requirements)
       end
@@ -58,22 +84,51 @@ def gem_dir(base_directory)
 
     def load
       return if @loaded || @load_paths_added == false
-      require(@lib || @name) unless @lib == false
+      require(@lib || name) unless @lib == false
       @loaded = true
     rescue LoadError
       puts $!.to_s
       $!.backtrace.each { |b| puts b }
     end
 
+    def name
+      @dep.name.to_s
+    end
+
+    def requirement
+      r = @dep.version_requirements
+      (r == Gem::Requirement.default) ? nil : r
+    end
+
     def frozen?
-      @frozen
+      @frozen ||= vendor_rails? || vendor_gem?
     end
 
     def loaded?
-      @loaded
+      @loaded ||= begin
+        if vendor_rails?
+          true
+        else
+          # check if the gem is loaded by inspecting $"
+          # specification.files lists all the files contained in the gem
+          gem_files = specification.files
+          # select only the files contained in require_paths - typically in bin and lib
+          require_paths_regexp = Regexp.new("^(#{specification.require_paths*'|'})/")
+          gem_lib_files = gem_files.select { |f| require_paths_regexp.match(f) }
+          # chop the leading directory off - a typical file might be in
+          # lib/gem_name/file_name.rb, but it will be 'require'd as gem_name/file_name.rb
+          gem_lib_files.map! { |f| f.split('/', 2)[1] }
+          # if any of the files from the above list appear in $", the gem is assumed to
+          # have been loaded
+          !(gem_lib_files & $").empty?
+        end
+      end
     end
 
     def load_paths_added?
+      # always try to add load paths - even if a gem is loaded, it may not
+      # be a compatible version (ie random_gem 0.4 is loaded and a later spec
+      # needs >= 0.5 - gem 'random_gem' will catch this and error out)
       @load_paths_added
     end
 
@@ -93,17 +148,44 @@ def unpack_to(directory)
       # we can access information about the gem on deployment systems
       # without having the gem installed
       spec_filename = File.join(gem_dir(directory), '.specification')
+      # Gem.activate changes the spec - get the original
+      spec = Gem::Specification.load(specification.loaded_from)
       File.open(spec_filename, 'w') do |file|
-        file.puts specification.to_yaml
+        file.puts spec.to_yaml
       end
     end
 
     def ==(other)
       self.name == other.name && self.requirement == other.requirement
     end
+    alias_method :"eql?", :"=="
+
+    def hash
+      @dep.hash
+    end
 
     def specification
-      @spec ||= Gem.source_index.search(Gem::Dependency.new(@name, @requirement)).sort_by { |s| s.version }.last
+      # code repeated from Gem.activate. Find a matching spec, or the currently loaded version.
+      # error out if loaded version and requested version are incompatible.
+      @spec ||= begin
+        matches = Gem.source_index.search(@dep)
+        matches << @@framework_gems[name] if framework_gem?
+        if Gem.loaded_specs[name] then
+          # This gem is already loaded.  If the currently loaded gem is not in the
+          # list of candidate gems, then we have a version conflict.
+          existing_spec = Gem.loaded_specs[name]
+          unless matches.any? { |spec| spec.version == existing_spec.version } then
+            raise Gem::Exception,
+                  "can't activate #{@dep}, already activated #{existing_spec.full_name}"
+          end
+          # we're stuck with it, so change to match
+          @dep.version_requirements = Gem::Requirement.create("=#{existing_spec.version}")
+          existing_spec
+        else
+          # new load
+          matches.last
+        end
+      end
     end
 
     private
@@ -112,17 +194,15 @@ def gem_command
       end
 
       def install_command
-        cmd = %w(install) << @name
-        cmd << "--version" << %("#{@requirement.to_s}") if @requirement
+        cmd = %w(install) << name
+        cmd << "--version" << %("#{requirement.to_s}") if requirement
         cmd << "--source"  << @source  if @source
         cmd
       end
 
       def unpack_command
-        cmd = %w(unpack) << @name
-        # We don't quote this requirement as it's run through GemRunner instead
-        # of shelling out to gem
-        cmd << "--version" << @requirement.to_s if @requirement
+        cmd = %w(unpack) << name
+        cmd << "--version" << "= "+specification.version.to_s if requirement
         cmd
       end
   end

railties/lib/rails/plugin.rb

@@ -112,7 +112,7 @@ def evaluate_init_rb(initializer)
   class GemPlugin < Plugin
     # Initialize this plugin from a Gem::Specification.
     def initialize(spec, gem)
-      directory = (gem.frozen? && gem.unpacked_paths.first) || File.join(spec.full_gem_path)
+      directory = spec.full_gem_path
       super(directory)
       @name = spec.name
     end

railties/lib/rails/vendor_gem_source_index.rb

+require 'rubygems'
+require 'yaml'
+
+module Rails
+
+  class VendorGemSourceIndex
+    # VendorGemSourceIndex acts as a proxy for the Gem source index, allowing
+    # gems to be loaded from vendor/gems. Rather than the standard gem repository format,
+    # vendor/gems contains unpacked gems, with YAML specifications in .specification in
+    # each gem directory.
+    include Enumerable
+
+    attr_reader :installed_source_index
+    attr_reader :vendor_source_index
+
+    def initialize(installed_index, vendor_dir=Rails::GemDependency.unpacked_path)
+      @installed_source_index = installed_index
+      @vendor_dir = vendor_dir
+      refresh!
+    end
+
+    def refresh!
+      # reload the installed gems
+      @installed_source_index.refresh!
+      vendor_gems = {}
+
+      # handle vendor Rails gems - they are identified by having loaded_from set to ""
+      # we add them manually to the list, so that other gems can find them via dependencies
+      Gem.loaded_specs.each do |n, s|
+        next unless s.loaded_from.empty?
+        vendor_gems[s.full_name] = s
+      end
+
+      # load specifications from vendor/gems
+      Dir[File.join(Rails::GemDependency.unpacked_path, '*')].each do |d|
+        spec = load_specification(d)
+        next unless spec
+        # NOTE: this is a bit of a hack - the gem system expects a different structure
+        # than we have.
+        # It's looking for:
+        # repository
+        #   -> specifications
+        #      - gem_name.spec      <= loaded_from points to this
+        #   -> gems
+        #      - gem_name           <= gem files here
+        # and therefore goes up one directory from loaded_from, then adds gems/gem_name
+        # to the path.
+        # But we have:
+        # vendor
+        #   -> gems
+        #      -> gem_name          <= gem files here
+        #         - .specification
+        # so we set loaded_from to vendor/gems/.specification (not a real file) to
+        # get the correct behavior.
+        spec.loaded_from = File.join(Rails::GemDependency.unpacked_path, '.specification')
+        vendor_gems[File.basename(d)] = spec
+      end
+      @vendor_source_index = Gem::SourceIndex.new(vendor_gems)
+    end
+
+    def load_specification(gem_dir)
+      spec_file = File.join(gem_dir, '.specification')
+      YAML.load_file(spec_file) if File.exist?(spec_file)
+    end
+
+    def find_name(gem_name, version_requirement = Gem::Requirement.default)
+      search(/^#{gem_name}$/, version_requirement)
+    end
+
+    def search(*args)
+      # look for vendor gems, and then installed gems - later elements take priority
+      @installed_source_index.search(*args) + @vendor_source_index.search(*args)
+    end
+
+    def each(&block)
+      @vendor_source_index.each(&block)
+      @installed_source_index.each(&block)
+    end
+
+    def add_spec(spec)
+      @vendor_source_index.add_spec spec
+    end
+
+    def remove_spec(spec)
+      @vendor_source_index.remove_spec spec
+    end
+
+    def size
+      @vendor_source_index.size + @installed_source_index.size
+    end
+
+  end
+end
\ No newline at end of file

railties/lib/tasks/gems.rake

 desc "List the gems that this rails application depends on"
 task :gems => 'gems:base' do
   Rails.configuration.gems.each do |gem|
-    code = gem.loaded? ? (gem.frozen? ? "F" : "I") : " "
-    puts "[#{code}] #{gem.name} #{gem.requirement.to_s}"
+    print_gem_status(gem)
   end
   puts
   puts "I = Installed"
   puts "F = Frozen"
 end
 
+def print_gem_status(gem, indent=1)
+  code = gem.loaded? ? (gem.frozen? ? "F" : "I") : " "
+  puts "   "*(indent-1)+" - [#{code}] #{gem.name} #{gem.requirement.to_s}"
+  gem.dependencies.each { |g| print_gem_status(g, indent+1)}
+end
+
 namespace :gems do
   task :base do
     $rails_gem_installer = true
@@ -19,7 +24,7 @@ namespace :gems do
   task :build do
     $rails_gem_installer = true
     require 'rails/gem_builder'
-    Dir[File.join(RAILS_ROOT, 'vendor', 'gems', '*')].each do |gem_dir|
+    Dir[File.join(Rails::GemDependency.unpacked_path, '*')].each do |gem_dir|
       spec_file = File.join(gem_dir, '.specification')
       next unless File.exists?(spec_file)
       specification = YAML::load_file(spec_file)
@@ -28,7 +33,7 @@ namespace :gems do
       puts "Built gem: '#{gem_dir}'"
     end
   end
-  
+
   desc "Installs all required gems for this application."
   task :install => :base do
     require 'rubygems'
@@ -42,10 +47,10 @@ namespace :gems do
     require 'rubygems/gem_runner'
     Rails.configuration.gems.each do |gem|
       next unless !gem.frozen? && (ENV['GEM'].blank? || ENV['GEM'] == gem.name)
-      gem.unpack_to(File.join(RAILS_ROOT, 'vendor', 'gems')) if gem.loaded?
+      gem.unpack_to(Rails::GemDependency.unpacked_path) if gem.loaded?
     end
   end
-  
+
   namespace :unpack do
     desc "Unpacks the specified gems and its dependencies into vendor/gems"
     task :dependencies => :unpack do
@@ -54,9 +59,8 @@ namespace :gems do
       Rails.configuration.gems.each do |gem|
         next unless ENV['GEM'].blank? || ENV['GEM'] == gem.name
         gem.dependencies.each do |dependency|
-          dependency.add_load_paths # double check that we have not already unpacked
           next if dependency.frozen?
-          dependency.unpack_to(File.join(RAILS_ROOT, 'vendor', 'gems'))
+          dependency.unpack_to(Rails::GemDependency.unpacked_path)
         end
       end
     end

railties/test/gem_dependency_test.rb

@@ -37,39 +37,78 @@ def test_gem_creates_unpack_command
     end
 
     def test_gem_with_version_unpack_install_command
+      # stub out specification method, or else test will fail if hpricot 0.6 isn't installed
+      mock_spec = mock()
+      mock_spec.stubs(:version).returns('0.6')
+      @gem_with_version.stubs(:specification).returns(mock_spec)
       assert_equal ["unpack", "hpricot", "--version", '= 0.6'], @gem_with_version.unpack_command
     end
 
     def test_gem_adds_load_paths
-      @gem.expects(:gem).with(@gem.name)
+      @gem.expects(:gem).with(Gem::Dependency.new(@gem.name, nil))
       @gem.add_load_paths
     end
 
     def test_gem_with_version_adds_load_paths
-      @gem_with_version.expects(:gem).with(@gem_with_version.name, @gem_with_version.requirement.to_s)
+      @gem_with_version.expects(:gem).with(Gem::Dependency.new(@gem_with_version.name, @gem_with_version.requirement.to_s))
       @gem_with_version.add_load_paths
     end
 
     def test_gem_loading
-      @gem.expects(:gem).with(@gem.name)
+      @gem.expects(:gem).with(Gem::Dependency.new(@gem.name, nil))
       @gem.expects(:require).with(@gem.name)
       @gem.add_load_paths
       @gem.load
     end
 
     def test_gem_with_lib_loading
-      @gem_with_lib.expects(:gem).with(@gem_with_lib.name)
+      @gem_with_lib.expects(:gem).with(Gem::Dependency.new(@gem_with_lib.name, nil))
       @gem_with_lib.expects(:require).with(@gem_with_lib.lib)
       @gem_with_lib.add_load_paths
       @gem_with_lib.load
     end
 
     def test_gem_without_lib_loading
-      @gem_without_load.expects(:gem).with(@gem_without_load.name)
+      @gem_without_load.expects(:gem).with(Gem::Dependency.new(@gem_without_load.name, nil))
       @gem_without_load.expects(:require).with(@gem_without_load.lib).never
       @gem_without_load.add_load_paths
       @gem_without_load.load
     end
 
+    def test_gem_dependencies_compare_for_uniq
+      gem1 = Rails::GemDependency.new "gem1"
+      gem1a = Rails::GemDependency.new "gem1"
+      gem2 = Rails::GemDependency.new "gem2"
+      gem2a = Rails::GemDependency.new "gem2"
+      gem3 = Rails::GemDependency.new "gem2", :version => ">=0.1"
+      gem3a = Rails::GemDependency.new "gem2", :version => ">=0.1"
+      gem4 = Rails::GemDependency.new "gem3"
+      gems = [gem1, gem2, gem1a, gem3, gem2a, gem4, gem3a, gem2, gem4]
+      assert_equal 4, gems.uniq.size
+    end
+
+    def test_gem_load_frozen
+      dummy_gem = Rails::GemDependency.new "dummy-gem-a"
+      dummy_gem.add_load_paths
+      dummy_gem.load
+      assert_not_nil DUMMY_GEM_A_VERSION
+    end
+
+    def test_gem_load_frozen_specific_version
+      dummy_gem = Rails::GemDependency.new "dummy-gem-b", :version => '0.4.0'
+      dummy_gem.add_load_paths
+      dummy_gem.load
+      assert_not_nil DUMMY_GEM_B_VERSION
+      assert_equal '0.4.0', DUMMY_GEM_B_VERSION
+    end
+
+    def test_gem_load_frozen_minimum_version
+      dummy_gem = Rails::GemDependency.new "dummy-gem-c", :version => '>=0.5.0'
+      dummy_gem.add_load_paths
+      dummy_gem.load
+      assert_not_nil DUMMY_GEM_C_VERSION
+      assert_equal '0.6.0', DUMMY_GEM_C_VERSION
+    end
+
   end
 end

railties/test/vendor/gems/dummy-gem-a-0.4.0/.specification

+--- !ruby/object:Gem::Specification
+name: dummy-gem-a
+version: !ruby/object:Gem::Version
+  version: 0.4.0
+platform: ruby
+authors:
+- "Nobody"
+date: 2008-10-03 00:00:00 -04:00
+files:
+- lib
+- lib/dummy-gem-a.rb
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: "0"
+  version:
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: "0"
+  version:
+requirements: []
+specification_version: 2
+summary: Dummy Gem A

railties/test/vendor/gems/dummy-gem-a-0.4.0/lib/dummy-gem-a.rb

+DUMMY_GEM_A_VERSION="0.4.0"

railties/test/vendor/gems/dummy-gem-b-0.4.0/.specification

+--- !ruby/object:Gem::Specification
+name: dummy-gem-b
+version: !ruby/object:Gem::Version
+  version: 0.4.0
+platform: ruby
+authors:
+- "Nobody"
+date: 2008-10-03 00:00:00 -04:00
+files:
+- lib
+- lib/dummy-gem-b.rb
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: "0"
+  version:
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: "0"
+  version:
+requirements: []
+specification_version: 2
+summary: Dummy Gem B

railties/test/vendor/gems/dummy-gem-b-0.4.0/lib/dummy-gem-b.rb

+DUMMY_GEM_B_VERSION="0.4.0"

railties/test/vendor/gems/dummy-gem-b-0.6.0/.specification

+--- !ruby/object:Gem::Specification
+name: dummy-gem-b
+version: !ruby/object:Gem::Version
+  version: 0.6.0
+platform: ruby
+authors:
+- "Nobody"
+date: 2008-10-03 00:00:00 -04:00
+files:
+- lib
+- lib/dummy-gem-b.rb
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: "0"
+  version:
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: "0"
+  version:
+requirements: []
+specification_version: 2
+summary: Dummy Gem B

railties/test/vendor/gems/dummy-gem-b-0.6.0/lib/dummy-gem-b.rb

+DUMMY_GEM_B_VERSION="0.6.0"

railties/test/vendor/gems/dummy-gem-c-0.4.0/.specification

+--- !ruby/object:Gem::Specification
+name: dummy-gem-c
+version: !ruby/object:Gem::Version
+  version: 0.4.0
+platform: ruby
+authors:
+- "Nobody"
+date: 2008-10-03 00:00:00 -04:00
+files:
+- lib
+- lib/dummy-gem-c.rb
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: "0"
+  version:
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: "0"
+  version:
+requirements: []
+specification_version: 2
+summary: Dummy Gem C

railties/test/vendor/gems/dummy-gem-c-0.4.0/lib/dummy-gem-c.rb

+DUMMY_GEM_C_VERSION="0.4.0"

railties/test/vendor/gems/dummy-gem-c-0.6.0/.specification

+--- !ruby/object:Gem::Specification
+name: dummy-gem-c
+version: !ruby/object:Gem::Version
+  version: 0.6.0
+platform: ruby
+authors:
+- "Nobody"
+date: 2008-10-03 00:00:00 -04:00
+files:
+- lib
+- lib/dummy-gem-c.rb
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: "0"
+  version:
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: "0"
+  version:
+requirements: []
+specification_version: 2
+summary: Dummy Gem C

railties/test/vendor/gems/dummy-gem-c-0.6.0/lib/dummy-gem-c.rb

+DUMMY_GEM_C_VERSION="0.6.0"