* lib/rubygems: Update to RubyGems 2.5.0+ HEAD(db78980).

this version includes #1367 , #1373 , #1375
* test/rubygems: ditto.

git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@52546 b2dd03c8-39d4-4d8f-98ff-823fe69b080e

ChangeLog

@@ -1,3 +1,9 @@
+Thu Nov 12 13:49:50 2015  SHIBATA Hiroshi  <hsbt@ruby-lang.org>
+
+	* lib/rubygems: Update to RubyGems 2.5.0+ HEAD(db78980).
+	  this version includes #1367 , #1373 , #1375
+	* test/rubygems: ditto.
+
 Thu Nov 12 10:53:41 2015  Eric Wong  <e@80x24.org>
 
 	* benchmark/bm_io_nonblock_noex2.rb: new benchmark based

lib/rubygems.rb

@@ -307,11 +307,10 @@ module Gem
   # package is not available as a gem, return nil.
 
   def self.datadir(gem_name)
-# TODO: deprecate and move to Gem::Specification
+# TODO: deprecate
-#       and drop the extra ", gem_name" which is uselessly redundant
     spec = @loaded_specs[gem_name]
     return nil if spec.nil?
-    File.join spec.full_gem_path, "data", gem_name
+    spec.datadir
   end
 
   ##

lib/rubygems/basic_specification.rb

@@ -37,6 +37,14 @@ class Gem::BasicSpecification
     File.join(Gem.default_dir, "specifications", "default")
   end
 
+  ##
+  # The path to the gem.build_complete file within the extension install
+  # directory.
+
+  def gem_build_complete_path # :nodoc:
+    File.join extension_dir, 'gem.build_complete'
+  end
+
   ##
   # True when the gem has been activated
 
@@ -50,12 +58,7 @@ class Gem::BasicSpecification
   # eg: /usr/local/lib/ruby/gems/1.8
 
   def base_dir
-    return Gem.dir unless loaded_from
+    raise NotImplementedError
-    @base_dir ||= if default_gem? then
-                    File.dirname File.dirname File.dirname loaded_from
-                  else
-                    File.dirname File.dirname loaded_from
-                  end
   end
 
   ##
@@ -65,7 +68,7 @@ class Gem::BasicSpecification
     @contains_requirable_file ||= {}
     @contains_requirable_file[file] ||=
     begin
-      if instance_variable_defined?(:@ignored) then
+      if @ignored then
         return false
       elsif missing_extensions? then
         @ignored = true
@@ -75,12 +78,7 @@ class Gem::BasicSpecification
         return false
       end
 
-      suffixes = Gem.suffixes
+      have_file? file, Gem.suffixes
-
-      full_require_paths.any? do |dir|
-        base = "#{dir}/#{file}"
-        suffixes.any? { |suf| File.file? "#{base}#{suf}" }
-      end
     end ? :yes : :no
     @contains_requirable_file[file] == :yes
   end
@@ -94,7 +92,7 @@ class Gem::BasicSpecification
   # Returns full path to the directory where gem's extensions are installed.
 
   def extension_dir
-    @extension_dir ||= File.expand_path File.join(extensions_dir, full_name)
+    @extension_dir ||= File.expand_path(File.join(extensions_dir, full_name)).untaint
   end
 
   ##
@@ -110,7 +108,7 @@ class Gem::BasicSpecification
     # TODO: also, shouldn't it default to full_name if it hasn't been written?
     path = File.expand_path File.join(gems_dir, full_name)
     path.untaint
-    path if File.directory? path
+    path
   end
 
   private :find_full_gem_path
@@ -148,12 +146,20 @@ class Gem::BasicSpecification
         File.join full_gem_path, path.untaint
       end
 
-      full_paths << extension_dir unless @extensions.nil? || @extensions.empty?
+      full_paths << extension_dir if have_extensions?
 
       full_paths
     end
   end
 
+  ##
+  # The path to the data directory for this gem.
+
+  def datadir
+# TODO: drop the extra ", gem_name" which is uselessly redundant
+    File.expand_path(File.join(gems_dir, full_name, "data", name)).untaint
+  end
+
   ##
   # Full path of the target library file.
   # If the file is not in this gem, return nil.
@@ -189,8 +195,7 @@ class Gem::BasicSpecification
   # gem directory. eg: /usr/local/lib/ruby/1.8/gems
 
   def gems_dir
-    # TODO: this logic seems terribly broken, but tests fail if just base_dir
+    raise NotImplementedError
-    @gems_dir ||= File.join(loaded_from && base_dir || Gem.dir, "gems")
   end
 
   def internal_init # :nodoc:
@@ -198,8 +203,7 @@ class Gem::BasicSpecification
     @extensions_dir = nil
     @full_gem_path         = nil
     @gem_dir               = nil
-    @gems_dir              = nil
+    @ignored = nil
-    @base_dir              = nil
   end
 
   ##
@@ -217,7 +221,7 @@ class Gem::BasicSpecification
   end
 
   def raw_require_paths # :nodoc:
-    Array(@require_paths)
+    raise NotImplementedError
   end
 
   ##
@@ -238,7 +242,7 @@ class Gem::BasicSpecification
   #   spec.require_path = '.'
 
   def require_paths
-    return raw_require_paths if @extensions.nil? || @extensions.empty?
+    return raw_require_paths unless have_extensions?
 
     [extension_dir].concat raw_require_paths
   end
@@ -250,8 +254,8 @@ class Gem::BasicSpecification
   def source_paths
     paths = raw_require_paths.dup
 
-    if @extensions then
+    if have_extensions? then
-      ext_dirs = @extensions.map do |extension|
+      ext_dirs = extensions.map do |extension|
         extension.split(File::SEPARATOR, 2).first
       end.uniq
 
@@ -307,5 +311,23 @@ class Gem::BasicSpecification
     raise NotImplementedError
   end
 
+  private
+
+  def have_extensions?; !extensions.empty?; end
+
+  def have_file? file, suffixes
+    return true if raw_require_paths.any? do |path|
+      base = File.join(gems_dir, full_name, path.untaint, file).untaint
+      suffixes.any? { |suf| File.file? base + suf }
+    end
+
+    if have_extensions?
+      base = File.join extension_dir, file
+      suffixes.any? { |suf| File.file? base + suf }
+    else
+      false
+    end
+  end
+
 end
 

lib/rubygems/config_file.rb

@@ -321,12 +321,12 @@ if you believe they were disclosed to a third party.
     @rubygems_api_key = api_key
   end
 
-  YAMLErrors = [ArgumentError]
-  YAMLErrors << Psych::SyntaxError if defined?(Psych::SyntaxError)
-
   def load_file(filename)
     Gem.load_yaml
 
+    yaml_errors = [ArgumentError]
+    yaml_errors << Psych::SyntaxError if defined?(Psych::SyntaxError)
+
     return {} unless filename and File.exist? filename
 
     begin
@@ -336,7 +336,7 @@ if you believe they were disclosed to a third party.
         return {}
       end
       return content
-    rescue *YAMLErrors => e
+    rescue *yaml_errors => e
       warn "Failed to load #{filename}, #{e}"
     rescue Errno::EACCES
       warn "Failed to load #{filename} due to permissions problem."

lib/rubygems/resolver.rb

@@ -232,7 +232,7 @@ class Gem::Resolver
       exc.errors = @set.errors
       raise exc
     end
-    possibles.sort_by { |s| [s.source, s.version, s.platform == Gem::Platform.local.to_s ? 1 : 0] }.
+    possibles.sort_by { |s| [s.source, s.version, s.platform.to_s == Gem::Platform.local.to_s ? 1 : 0] }.
       map { |s| ActivationRequest.new s, dependency, [] }
   end
 

lib/rubygems/specification.rb

@@ -740,23 +740,41 @@ class Gem::Specification < Gem::BasicSpecification
   end
 
   def self.gemspec_stubs_in dir, pattern
-    Dir[File.join(dir, pattern)].map { |path|
+    Dir[File.join(dir, pattern)].map { |path| yield path }.select(&:valid?)
-      if dir == default_specifications_dir
-        Gem::StubSpecification.default_gemspec_stub(path)
-      else
-        Gem::StubSpecification.gemspec_stub(path)
-      end
-    }.select(&:valid?)
   end
   private_class_method :gemspec_stubs_in
 
+  def self.default_stubs pattern
+    base_dir = Gem.default_dir
+    gems_dir = File.join base_dir, "gems"
+    gemspec_stubs_in(default_specifications_dir, pattern) do |path|
+      Gem::StubSpecification.default_gemspec_stub(path, base_dir, gems_dir)
+    end
+  end
+  private_class_method :default_stubs
+
+  def self.installed_stubs dirs, pattern
+    map_stubs(dirs, pattern) do |path, base_dir, gems_dir|
+      Gem::StubSpecification.gemspec_stub(path, base_dir, gems_dir)
+    end
+  end
+  private_class_method :installed_stubs
+
   if [].respond_to? :flat_map
     def self.map_stubs(dirs, pattern) # :nodoc:
-      dirs.flat_map { |dir| gemspec_stubs_in(dir, pattern) }
+      dirs.flat_map { |dir|
+        base_dir = File.dirname dir
+        gems_dir = File.join base_dir, "gems"
+        gemspec_stubs_in(dir, pattern) { |path| yield path, base_dir, gems_dir }
+      }
     end
   else # FIXME: remove when 1.8 is dropped
     def self.map_stubs(dirs, pattern) # :nodoc:
-      dirs.map { |dir| gemspec_stubs_in(dir, pattern) }.flatten 1
+      dirs.map { |dir|
+        base_dir = File.dirname dir
+        gems_dir = File.join base_dir, "gems"
+        gemspec_stubs_in(dir, pattern) { |path| yield path, base_dir, gems_dir }
+      }.flatten 1
     end
   end
   private_class_method :map_stubs
@@ -803,7 +821,8 @@ class Gem::Specification < Gem::BasicSpecification
 
   def self.stubs
     @@stubs ||= begin
-      stubs = map_stubs([default_specifications_dir] + dirs, "*.gemspec")
+      pattern = "*.gemspec"
+      stubs = default_stubs(pattern).concat installed_stubs(dirs, pattern)
       stubs = uniq_by(stubs) { |stub| stub.full_name }
 
       _resort!(stubs)
@@ -818,10 +837,11 @@ class Gem::Specification < Gem::BasicSpecification
   # Returns a Gem::StubSpecification for installed gem named +name+
 
   def self.stubs_for name
-    if @@stubs || @@stubs_by_name[name]
+    if @@stubs
       @@stubs_by_name[name] || []
     else
-      stubs = map_stubs([default_specifications_dir] + dirs, "#{name}-*.gemspec")
+      pattern = "#{name}-*.gemspec"
+      stubs = default_stubs(pattern) + installed_stubs(dirs, pattern)
       stubs = uniq_by(stubs) { |stub| stub.full_name }.group_by(&:name)
       stubs.each_value { |v| sort_by!(v) { |i| i.version } }
 
@@ -1006,8 +1026,9 @@ class Gem::Specification < Gem::BasicSpecification
   # Return the best specification that contains the file matching +path+.
 
   def self.find_by_path path
+    path = path.dup.freeze
     stub = stubs.find { |spec|
-      spec.contains_requirable_file? path if spec
+      spec.contains_requirable_file? path
     }
     stub && stub.to_spec
   end
@@ -1018,7 +1039,7 @@ class Gem::Specification < Gem::BasicSpecification
 
   def self.find_inactive_by_path path
     stub = stubs.find { |s|
-      s.contains_requirable_file? path unless s.nil? || s.activated?
+      s.contains_requirable_file? path unless s.activated?
     }
     stub && stub.to_spec
   end
@@ -1030,7 +1051,7 @@ class Gem::Specification < Gem::BasicSpecification
     # TODO: do we need these?? Kill it
     specs = unresolved_deps.values.map { |dep| dep.to_specs }.flatten
 
-    specs.find_all { |spec| spec.contains_requirable_file? path if spec }
+    specs.find_all { |spec| spec.contains_requirable_file? path }
   end
 
   ##
@@ -1924,23 +1945,10 @@ class Gem::Specification < Gem::BasicSpecification
     spec
   end
 
-  def find_full_gem_path # :nodoc:
-    super || File.expand_path(File.join(gems_dir, original_name))
-  end
-  private :find_full_gem_path
-
   def full_name
     @full_name ||= super
   end
 
-  ##
-  # The path to the gem.build_complete file within the extension install
-  # directory.
-
-  def gem_build_complete_path # :nodoc:
-    File.join extension_dir, 'gem.build_complete'
-  end
-
   ##
   # Work around bundler removing my methods
 
@@ -1948,6 +1956,11 @@ class Gem::Specification < Gem::BasicSpecification
     super
   end
 
+  def gems_dir
+    # TODO: this logic seems terribly broken, but tests fail if just base_dir
+    @gems_dir ||= File.join(loaded_from && base_dir || Gem.dir, "gems")
+  end
+
   ##
   # Deprecated and ignored, defaults to true.
   #
@@ -1995,6 +2008,8 @@ class Gem::Specification < Gem::BasicSpecification
 
   def initialize name = nil, version = nil
     super()
+    @gems_dir              = nil
+    @base_dir              = nil
     @loaded = false
     @activated = false
     @loaded_from = nil
@@ -2044,6 +2059,15 @@ class Gem::Specification < Gem::BasicSpecification
     end
   end
 
+  def base_dir
+    return Gem.dir unless loaded_from
+    @base_dir ||= if default_gem? then
+                    File.dirname File.dirname File.dirname loaded_from
+                  else
+                    File.dirname File.dirname loaded_from
+                  end
+  end
+
   ##
   # Expire memoized instance variables that can incorrectly generate, replace
   # or miss files due changes in certain attributes used to compute them.
@@ -2954,6 +2978,10 @@ open-ended dependency on #{dep} is not recommended
     alert_warning statement
   end
 
+  def raw_require_paths # :nodoc:
+    @require_paths
+  end
+
   extend Gem::Deprecate
 
   # TODO:

lib/rubygems/stub_specification.rb

@@ -15,33 +15,65 @@ class Gem::StubSpecification < Gem::BasicSpecification
     end
 
   class StubLine # :nodoc: all
-    attr_reader :name, :version, :platform, :require_paths
+    attr_reader :name, :version, :platform, :require_paths, :extensions,
+                :full_name
 
-    def initialize(data)
+    NO_EXTENSIONS = [].freeze
-      parts          = data[PREFIX.length..-1].split(" ")
+
+    # These are common require paths.
+    REQUIRE_PATHS = { # :nodoc:
+      'lib'  => 'lib'.freeze,
+      'test' => 'test'.freeze,
+      'ext'  => 'ext'.freeze,
+    }
+
+    # These are common require path lists.  This hash is used to optimize
+    # and consolidate require_path objects.  Most specs just specify "lib"
+    # in their require paths, so lets take advantage of that by pre-allocating
+    # a require path list for that case.
+    REQUIRE_PATH_LIST = { # :nodoc:
+      'lib' => ['lib'].freeze
+    }
+
+    def initialize data, extensions
+      parts          = data[PREFIX.length..-1].split(" ".freeze, 4)
       @name          = parts[0].freeze
       @version       = Gem::Version.new parts[1]
       @platform      = Gem::Platform.new parts[2]
-      @require_paths = parts.drop(3).join(" ").split("\0")
+      @extensions    = extensions
+      @full_name     = if platform == Gem::Platform::RUBY
+                         "#{name}-#{version}"
+                       else
+                         "#{name}-#{version}-#{platform}"
+                       end
+
+      path_list = parts.last
+      @require_paths = REQUIRE_PATH_LIST[path_list] || path_list.split("\0".freeze).map! { |x|
+        REQUIRE_PATHS[x] || x
+      }
     end
   end
 
-  def self.default_gemspec_stub filename
+  def self.default_gemspec_stub filename, base_dir, gems_dir
-    new filename, true
+    new filename, base_dir, gems_dir, true
   end
 
-  def self.gemspec_stub filename
+  def self.gemspec_stub filename, base_dir, gems_dir
-    new filename, false
+    new filename, base_dir, gems_dir, false
   end
 
-  def initialize filename, default_gem
+  attr_reader :base_dir, :gems_dir
+
+  def initialize filename, base_dir, gems_dir, default_gem
+    super()
     filename.untaint
 
     self.loaded_from = filename
     @data            = nil
-    @extensions      = nil
     @name            = nil
     @spec            = nil
+    @base_dir        = base_dir
+    @gems_dir        = gems_dir
     @default_gem     = default_gem
   end
 
@@ -73,8 +105,6 @@ class Gem::StubSpecification < Gem::BasicSpecification
 
   def data
     unless @data
-      @extensions = []
-
       begin
         saved_lineno = $.
         open loaded_from, OPEN_MODE do |file|
@@ -82,10 +112,13 @@ class Gem::StubSpecification < Gem::BasicSpecification
             file.readline # discard encoding line
             stubline = file.readline.chomp
             if stubline.start_with?(PREFIX) then
-              @data = StubLine.new stubline
+              extensions = if /\A#{PREFIX}/ =~ file.readline.chomp
+                             $'.split "\0"
+                           else
+                             StubLine::NO_EXTENSIONS
+                           end
 
-              @extensions = $'.split "\0" if
+              @data = StubLine.new stubline, extensions
-                /\A#{PREFIX}/ =~ file.readline.chomp
             end
           rescue EOFError
           end
@@ -100,41 +133,14 @@ class Gem::StubSpecification < Gem::BasicSpecification
 
   private :data
 
-  ##
+  def raw_require_paths # :nodoc:
-  # Extensions for this gem
+    data.require_paths
-
-  def extensions
-    return @extensions if @extensions
-
-    data # load
-
-    @extensions
-  end
-
-  ##
-  # If a gem has a stub specification it doesn't need to bother with
-  # compatibility with original_name gems.  It was installed with the
-  # normalized name.
-
-  def find_full_gem_path # :nodoc:
-    path = File.expand_path File.join gems_dir, full_name
-    path.untaint
-    path
-  end
-
-  ##
-  # Full paths in the gem to add to <code>$LOAD_PATH</code> when this gem is
-  # activated.
-
-  def full_require_paths
-    @require_paths ||= data.require_paths
-
-    super
   end
 
   def missing_extensions?
     return false if default_gem?
     return false if extensions.empty?
+    return false if File.exist? gem_build_complete_path
 
     to_spec.missing_extensions?
   end
@@ -154,12 +160,21 @@ class Gem::StubSpecification < Gem::BasicSpecification
   end
 
   ##
-  # Require paths of the gem
+  # Extensions for this gem
 
-  def require_paths
+  def extensions
-    @require_paths ||= data.require_paths
+    data.extensions
+  end
 
-    super
+  ##
+  # Version of the gem
+
+  def version
+    data.version
+  end
+
+  def full_name
+    data.full_name
   end
 
   ##
@@ -173,7 +188,7 @@ class Gem::StubSpecification < Gem::BasicSpecification
               end
 
     @spec ||= Gem::Specification.load(loaded_from)
-    @spec.ignored = @ignored if instance_variable_defined? :@ignored
+    @spec.ignored = @ignored if @spec
 
     @spec
   end
@@ -186,13 +201,6 @@ class Gem::StubSpecification < Gem::BasicSpecification
     data
   end
 
-  ##
-  # Version of the gem
-
-  def version
-    @version ||= data.version
-  end
-
   ##
   # Is there a stub line present for this StubSpecification?
 

lib/rubygems/test_case.rb

@@ -37,9 +37,6 @@ require 'tmpdir'
 require 'uri'
 require 'zlib'
 require 'benchmark' # stdlib
-
-Gem.load_yaml
-
 require 'rubygems/mock_gem_ui'
 
 module Gem

test/rubygems/test_gem_config_file.rb

@@ -416,7 +416,7 @@ if you believe they were disclosed to a third party.
 
   def test_ignore_invalid_config_file
     File.open @temp_conf, 'w' do |fp|
-      fp.puts "some-non-yaml-hash-string"
+      fp.puts "invalid: yaml:"
     end
 
     begin

test/rubygems/test_gem_stub_specification.rb

@@ -9,7 +9,9 @@ class TestStubSpecification < Gem::TestCase
   def setup
     super
 
-    @foo = Gem::StubSpecification.gemspec_stub FOO
+    @base_dir = File.dirname(SPECIFICATIONS)
+    @gems_dir = File.join File.dirname(SPECIFICATIONS), 'gem'
+    @foo = Gem::StubSpecification.gemspec_stub FOO, @base_dir, @gems_dir
   end
 
   def test_initialize
@@ -31,7 +33,7 @@ class TestStubSpecification < Gem::TestCase
   end
 
   def test_initialize_missing_stubline
-    stub = Gem::StubSpecification.gemspec_stub(BAR)
+    stub = Gem::StubSpecification.gemspec_stub(BAR, @base_dir, @gems_dir)
     assert_equal "bar", stub.name
     assert_equal Gem::Version.new("0.0.2"), stub.version
     assert_equal Gem::Platform.new("ruby"), stub.platform
@@ -118,7 +120,7 @@ class TestStubSpecification < Gem::TestCase
       io.write spec.to_ruby_for_cache
     end
 
-    default_spec = Gem::StubSpecification.gemspec_stub spec.loaded_from
+    default_spec = Gem::StubSpecification.gemspec_stub spec.loaded_from, spec.base_dir, spec.gems_dir
 
     refute default_spec.missing_extensions?
   end
@@ -140,7 +142,7 @@ class TestStubSpecification < Gem::TestCase
   def test_to_spec_with_other_specs_loaded_does_not_warn
     real_foo = util_spec @foo.name, @foo.version
     real_foo.activate
-    bar = Gem::StubSpecification.gemspec_stub BAR
+    bar = Gem::StubSpecification.gemspec_stub BAR, real_foo.base_dir, real_foo.gems_dir
     refute_predicate Gem.loaded_specs, :empty?
     assert bar.to_spec
   end
@@ -148,7 +150,7 @@ class TestStubSpecification < Gem::TestCase
   def test_to_spec_activated
     assert @foo.to_spec.is_a?(Gem::Specification)
     assert_equal "foo", @foo.to_spec.name
-    refute @foo.to_spec.instance_variable_defined? :@ignored
+    refute @foo.to_spec.instance_variable_get :@ignored
   end
 
   def test_to_spec_missing_extensions
@@ -179,7 +181,7 @@ end
 
       io.flush
 
-      stub = Gem::StubSpecification.gemspec_stub io.path
+      stub = Gem::StubSpecification.gemspec_stub io.path, @gemhome, File.join(@gemhome, 'gems')
 
       yield stub if block_given?
 
@@ -202,7 +204,7 @@ end
 
       io.flush
 
-      stub = Gem::StubSpecification.gemspec_stub io.path
+      stub = Gem::StubSpecification.gemspec_stub io.path, @gemhome, File.join(@gemhome, 'gems')
 
       yield stub if block_given?