[rubygems/rubygems] Extract generate_index command to rubygems-generate_index gem

So generate_index can be implemented with dependencies, such as the compact index

Took this approach from feedback in https://github.com/rubygems/rubygems/pull/6853

Running `gem generate_index` by default will use an installed rubygems-generate_index, or install and then use the command from the gem

Apply suggestions from code review

https://github.com/rubygems/rubygems/commit/fc1cb9bc9e

Co-authored-by: Hiroshi SHIBATA <hsbt@ruby-lang.org>
This commit is contained in:
Samuel Giddins 2023-10-20 17:56:22 -07:00 committed by git
parent 0166d56f2b
commit 4817166e54
10 changed files with 59 additions and 978 deletions

View File

@ -249,6 +249,7 @@ class Gem::CommandManager
def invoke_command(args, build_args) def invoke_command(args, build_args)
cmd_name = args.shift.downcase cmd_name = args.shift.downcase
cmd = find_command cmd_name cmd = find_command cmd_name
terminate_interaction 1 unless cmd
cmd.deprecation_warning if cmd.deprecated? cmd.deprecation_warning if cmd.deprecated?
cmd.invoke_with_build_args args, build_args cmd.invoke_with_build_args args, build_args
end end

View File

@ -1,86 +1,51 @@
# frozen_string_literal: true # frozen_string_literal: true
require_relative "../command" require_relative "../command"
require_relative "../indexer"
## unless defined? Gem::Commands::GenerateIndexCommand
# Generates a index files for use as a gem server. class Gem::Commands::GenerateIndexCommand < Gem::Command
# module RubygemsTrampoline
# See `gem help generate_index` def description # :nodoc:
<<~EOF
The generate_index command has been moved to the rubygems-generate_index gem.
EOF
end
class Gem::Commands::GenerateIndexCommand < Gem::Command def execute
def initialize alert_error "Install the rubygems-generate_index gem for the generate_index command"
super "generate_index", end
"Generates the index files for a gem server directory",
directory: ".", build_modern: true
add_option "-d", "--directory=DIRNAME", def invoke_with_build_args(args, build_args)
"repository base dir containing gems subdir" do |dir, options| name = "rubygems-generate_index"
options[:directory] = File.expand_path dir spec = begin
end Gem::Specification.find_by_name(name)
rescue Gem::LoadError
require "rubygems/dependency_installer"
Gem.install(name, Gem::Requirement.default, Gem::DependencyInstaller::DEFAULT_OPTIONS).find {|s| s.name == name }
end
add_option "--[no-]modern", # remove the methods defined in this file so that the methods defined in the gem are used instead,
"Generate indexes for RubyGems", # and without a method redefinition warning
"(always true)" do |value, options| %w[description execute invoke_with_build_args].each do |method|
options[:build_modern] = value RubygemsTrampoline.remove_method(method)
end end
self.class.singleton_class.remove_method(:new)
deprecate_option("--modern", version: "4.0", extra_msg: "Modern indexes (specs, latest_specs, and prerelease_specs) are always generated, so this option is not needed.") spec.activate
deprecate_option("--no-modern", version: "4.0", extra_msg: "The `--no-modern` option is currently ignored. Modern indexes (specs, latest_specs, and prerelease_specs) are always generated.") Gem.load_plugin_files spec.matches_for_glob("rubygems_plugin#{Gem.suffix_pattern}")
add_option "--update", self.class.new.invoke_with_build_args(args, build_args)
"Update modern indexes with gems added",
"since the last update" do |value, options|
options[:update] = value
end
end
def defaults_str # :nodoc:
"--directory . --modern"
end
def description # :nodoc:
<<-EOF
The generate_index command creates a set of indexes for serving gems
statically. The command expects a 'gems' directory under the path given to
the --directory option. The given directory will be the directory you serve
as the gem repository.
For `gem generate_index --directory /path/to/repo`, expose /path/to/repo via
your HTTP server configuration (not /path/to/repo/gems).
When done, it will generate a set of files like this:
gems/*.gem # .gem files you want to
# index
specs.<version>.gz # specs index
latest_specs.<version>.gz # latest specs index
prerelease_specs.<version>.gz # prerelease specs index
quick/Marshal.<version>/<gemname>.gemspec.rz # Marshal quick index file
The .rz extension files are compressed with the inflate algorithm.
The Marshal version number comes from ruby's Marshal::MAJOR_VERSION and
Marshal::MINOR_VERSION constants. It is used to ensure compatibility.
EOF
end
def execute
# This is always true because it's the only way now.
options[:build_modern] = true
if !File.exist?(options[:directory]) ||
!File.directory?(options[:directory])
alert_error "unknown directory name #{options[:directory]}."
terminate_interaction 1
else
indexer = Gem::Indexer.new options.delete(:directory), options
if options[:update]
indexer.update_index
else
indexer.generate_index
end end
end end
private_constant :RubygemsTrampoline
# remove_method(:initialize) warns, but removing new does not warn
def self.new
command = allocate
command.send(:initialize, "generate_index", "Generates the index files for a gem server directory (requires rubygems-generate_index)")
command
end
prepend(RubygemsTrampoline)
end end
end end

View File

@ -333,7 +333,7 @@ platform.
@command_manager.command_names.each do |cmd_name| @command_manager.command_names.each do |cmd_name|
command = @command_manager[cmd_name] command = @command_manager[cmd_name]
next if command.deprecated? next if command&.deprecated?
summary = summary =
if command if command

View File

@ -1,429 +0,0 @@
# frozen_string_literal: true
require_relative "../rubygems"
require_relative "package"
require "tmpdir"
##
# Top level class for building the gem repository index.
class Gem::Indexer
include Gem::UserInteraction
##
# Build indexes for RubyGems 1.2.0 and newer when true
attr_accessor :build_modern
##
# Index install location
attr_reader :dest_directory
##
# Specs index install location
attr_reader :dest_specs_index
##
# Latest specs index install location
attr_reader :dest_latest_specs_index
##
# Prerelease specs index install location
attr_reader :dest_prerelease_specs_index
##
# Index build directory
attr_reader :directory
##
# Create an indexer that will index the gems in +directory+.
def initialize(directory, options = {})
require "fileutils"
require "tmpdir"
require "zlib"
options = { build_modern: true }.merge options
@build_modern = options[:build_modern]
@dest_directory = directory
@directory = Dir.mktmpdir "gem_generate_index"
marshal_name = "Marshal.#{Gem.marshal_version}"
@master_index = File.join @directory, "yaml"
@marshal_index = File.join @directory, marshal_name
@quick_dir = File.join @directory, "quick"
@quick_marshal_dir = File.join @quick_dir, marshal_name
@quick_marshal_dir_base = File.join "quick", marshal_name # FIX: UGH
@quick_index = File.join @quick_dir, "index"
@latest_index = File.join @quick_dir, "latest_index"
@specs_index = File.join @directory, "specs.#{Gem.marshal_version}"
@latest_specs_index =
File.join(@directory, "latest_specs.#{Gem.marshal_version}")
@prerelease_specs_index =
File.join(@directory, "prerelease_specs.#{Gem.marshal_version}")
@dest_specs_index =
File.join(@dest_directory, "specs.#{Gem.marshal_version}")
@dest_latest_specs_index =
File.join(@dest_directory, "latest_specs.#{Gem.marshal_version}")
@dest_prerelease_specs_index =
File.join(@dest_directory, "prerelease_specs.#{Gem.marshal_version}")
@files = []
end
##
# Build various indices
def build_indices
specs = map_gems_to_specs gem_file_list
Gem::Specification._resort! specs
build_marshal_gemspecs specs
build_modern_indices specs if @build_modern
compress_indices
end
##
# Builds Marshal quick index gemspecs.
def build_marshal_gemspecs(specs)
count = specs.count
progress = ui.progress_reporter count,
"Generating Marshal quick index gemspecs for #{count} gems",
"Complete"
files = []
Gem.time "Generated Marshal quick index gemspecs" do
specs.each do |spec|
next if spec.default_gem?
spec_file_name = "#{spec.original_name}.gemspec.rz"
marshal_name = File.join @quick_marshal_dir, spec_file_name
marshal_zipped = Gem.deflate Marshal.dump(spec)
File.open marshal_name, "wb" do |io|
io.write marshal_zipped
end
files << marshal_name
progress.updated spec.original_name
end
progress.done
end
@files << @quick_marshal_dir
files
end
##
# Build a single index for RubyGems 1.2 and newer
def build_modern_index(index, file, name)
say "Generating #{name} index"
Gem.time "Generated #{name} index" do
File.open(file, "wb") do |io|
specs = index.map do |*spec|
# We have to splat here because latest_specs is an array, while the
# others are hashes.
spec = spec.flatten.last
platform = spec.original_platform
# win32-api-1.0.4-x86-mswin32-60
unless String === platform
alert_warning "Skipping invalid platform in gem: #{spec.full_name}"
next
end
platform = Gem::Platform::RUBY if platform.nil? || platform.empty?
[spec.name, spec.version, platform]
end
specs = compact_specs(specs)
Marshal.dump(specs, io)
end
end
end
##
# Builds indices for RubyGems 1.2 and newer. Handles full, latest, prerelease
def build_modern_indices(specs)
prerelease, released = specs.partition do |s|
s.version.prerelease?
end
latest_specs =
Gem::Specification._latest_specs specs
build_modern_index(released.sort, @specs_index, "specs")
build_modern_index(latest_specs.sort, @latest_specs_index, "latest specs")
build_modern_index(prerelease.sort, @prerelease_specs_index,
"prerelease specs")
@files += [@specs_index,
"#{@specs_index}.gz",
@latest_specs_index,
"#{@latest_specs_index}.gz",
@prerelease_specs_index,
"#{@prerelease_specs_index}.gz"]
end
def map_gems_to_specs(gems)
gems.map do |gemfile|
if File.size(gemfile) == 0
alert_warning "Skipping zero-length gem: #{gemfile}"
next
end
begin
spec = Gem::Package.new(gemfile).spec
spec.loaded_from = gemfile
spec.abbreviate
spec.sanitize
spec
rescue SignalException
alert_error "Received signal, exiting"
raise
rescue StandardError => e
msg = ["Unable to process #{gemfile}",
"#{e.message} (#{e.class})",
"\t#{e.backtrace.join "\n\t"}"].join("\n")
alert_error msg
end
end.compact
end
##
# Compresses indices on disk
#--
# All future files should be compressed using gzip, not deflate
def compress_indices
say "Compressing indices"
Gem.time "Compressed indices" do
if @build_modern
gzip @specs_index
gzip @latest_specs_index
gzip @prerelease_specs_index
end
end
end
##
# Compacts Marshal output for the specs index data source by using identical
# objects as much as possible.
def compact_specs(specs)
names = {}
versions = {}
platforms = {}
specs.map do |(name, version, platform)|
names[name] = name unless names.include? name
versions[version] = version unless versions.include? version
platforms[platform] = platform unless platforms.include? platform
[names[name], versions[version], platforms[platform]]
end
end
##
# Compress +filename+ with +extension+.
def compress(filename, extension)
data = Gem.read_binary filename
zipped = Gem.deflate data
File.open "#{filename}.#{extension}", "wb" do |io|
io.write zipped
end
end
##
# List of gem file names to index.
def gem_file_list
Gem::Util.glob_files_in_dir("*.gem", File.join(@dest_directory, "gems"))
end
##
# Builds and installs indices.
def generate_index
make_temp_directories
build_indices
install_indices
rescue SignalException
ensure
FileUtils.rm_rf @directory
end
##
# Zlib::GzipWriter wrapper that gzips +filename+ on disk.
def gzip(filename)
Zlib::GzipWriter.open "#{filename}.gz" do |io|
io.write Gem.read_binary(filename)
end
end
##
# Install generated indices into the destination directory.
def install_indices
verbose = Gem.configuration.really_verbose
say "Moving index into production dir #{@dest_directory}" if verbose
files = @files
files.delete @quick_marshal_dir if files.include? @quick_dir
if files.include?(@quick_marshal_dir) && !files.include?(@quick_dir)
files.delete @quick_marshal_dir
dst_name = File.join(@dest_directory, @quick_marshal_dir_base)
FileUtils.mkdir_p File.dirname(dst_name), verbose: verbose
FileUtils.rm_rf dst_name, verbose: verbose
FileUtils.mv(@quick_marshal_dir, dst_name,
verbose: verbose, force: true)
end
files = files.map do |path|
path.sub(%r{^#{Regexp.escape @directory}/?}, "") # HACK?
end
files.each do |file|
src_name = File.join @directory, file
dst_name = File.join @dest_directory, file
FileUtils.rm_rf dst_name, verbose: verbose
FileUtils.mv(src_name, @dest_directory,
verbose: verbose, force: true)
end
end
##
# Make directories for index generation
def make_temp_directories
FileUtils.rm_rf @directory
FileUtils.mkdir_p @directory, mode: 0o700
FileUtils.mkdir_p @quick_marshal_dir
end
##
# Ensure +path+ and path with +extension+ are identical.
def paranoid(path, extension)
data = Gem.read_binary path
compressed_data = Gem.read_binary "#{path}.#{extension}"
unless data == Gem::Util.inflate(compressed_data)
raise "Compressed file #{compressed_path} does not match uncompressed file #{path}"
end
end
##
# Perform an in-place update of the repository from newly added gems.
def update_index
make_temp_directories
specs_mtime = File.stat(@dest_specs_index).mtime
newest_mtime = Time.at 0
updated_gems = gem_file_list.select do |gem|
gem_mtime = File.stat(gem).mtime
newest_mtime = gem_mtime if gem_mtime > newest_mtime
gem_mtime >= specs_mtime
end
if updated_gems.empty?
say "No new gems"
terminate_interaction 0
end
specs = map_gems_to_specs updated_gems
prerelease, released = specs.partition {|s| s.version.prerelease? }
files = build_marshal_gemspecs specs
Gem.time "Updated indexes" do
update_specs_index released, @dest_specs_index, @specs_index
update_specs_index released, @dest_latest_specs_index, @latest_specs_index
update_specs_index(prerelease,
@dest_prerelease_specs_index,
@prerelease_specs_index)
end
compress_indices
verbose = Gem.configuration.really_verbose
say "Updating production dir #{@dest_directory}" if verbose
files << @specs_index
files << "#{@specs_index}.gz"
files << @latest_specs_index
files << "#{@latest_specs_index}.gz"
files << @prerelease_specs_index
files << "#{@prerelease_specs_index}.gz"
files = files.map do |path|
path.sub(%r{^#{Regexp.escape @directory}/?}, "") # HACK?
end
files.each do |file|
src_name = File.join @directory, file
dst_name = File.join @dest_directory, file # REFACTOR: duped above
FileUtils.mv src_name, dst_name, verbose: verbose,
force: true
File.utime newest_mtime, newest_mtime, dst_name
end
ensure
FileUtils.rm_rf @directory
end
##
# Combines specs in +index+ and +source+ then writes out a new copy to
# +dest+. For a latest index, does not ensure the new file is minimal.
def update_specs_index(index, source, dest)
Gem.load_safe_marshal
specs_index = Gem::SafeMarshal.safe_load Gem.read_binary(source)
index.each do |spec|
platform = spec.original_platform
platform = Gem::Platform::RUBY if platform.nil? || platform.empty?
specs_index << [spec.name, spec.version, platform]
end
specs_index = compact_specs specs_index.uniq.sort
File.open dest, "wb" do |io|
Marshal.dump specs_index, io
end
end
end

View File

@ -556,7 +556,7 @@ RSpec.describe "bundle install with gem sources" do
end end
it "fails gracefully when downloading an invalid specification from the full index" do it "fails gracefully when downloading an invalid specification from the full index" do
build_repo2 do build_repo2(build_compact_index: false) do
build_gem "ajp-rails", "0.0.0", gemspec: false, skip_validation: true do |s| build_gem "ajp-rails", "0.0.0", gemspec: false, skip_validation: true do |s|
bad_deps = [["ruby-ajp", ">= 0.2.0"], ["rails", ">= 0.14"]] bad_deps = [["ruby-ajp", ">= 0.2.0"], ["rails", ">= 0.14"]]
s. s.

View File

@ -44,7 +44,7 @@ RSpec.describe "compact index api" do
end end
it "should handle case sensitivity conflicts" do it "should handle case sensitivity conflicts" do
build_repo4 do build_repo4(build_compact_index: false) do
build_gem "rack", "1.0" do |s| build_gem "rack", "1.0" do |s|
s.add_runtime_dependency("Rack", "0.1") s.add_runtime_dependency("Rack", "0.1")
end end

View File

@ -191,25 +191,25 @@ module Spec
end end
end end
def build_repo2(&blk) def build_repo2(**kwargs, &blk)
FileUtils.rm_rf gem_repo2 FileUtils.rm_rf gem_repo2
FileUtils.cp_r gem_repo1, gem_repo2 FileUtils.cp_r gem_repo1, gem_repo2
update_repo2(&blk) if block_given? update_repo2(**kwargs, &blk) if block_given?
end end
# A repo that has no pre-installed gems included. (The caller completely # A repo that has no pre-installed gems included. (The caller completely
# determines the contents with the block.) # determines the contents with the block.)
def build_repo4(&blk) def build_repo4(**kwargs, &blk)
FileUtils.rm_rf gem_repo4 FileUtils.rm_rf gem_repo4
build_repo(gem_repo4, &blk) build_repo(gem_repo4, **kwargs, &blk)
end end
def update_repo4(&blk) def update_repo4(&blk)
update_repo(gem_repo4, &blk) update_repo(gem_repo4, &blk)
end end
def update_repo2(&blk) def update_repo2(**kwargs, &blk)
update_repo(gem_repo2, &blk) update_repo(gem_repo2, **kwargs, &blk)
end end
def build_security_repo def build_security_repo
@ -227,12 +227,12 @@ module Spec
end end
end end
def build_repo(path, &blk) def build_repo(path, **kwargs, &blk)
return if File.directory?(path) return if File.directory?(path)
FileUtils.mkdir_p("#{path}/gems") FileUtils.mkdir_p("#{path}/gems")
update_repo(path, &blk) update_repo(path,**kwargs, &blk)
end end
def check_test_gems! def check_test_gems!
@ -249,7 +249,7 @@ module Spec
end end
end end
def update_repo(path) def update_repo(path, build_compact_index: true)
if path == gem_repo1 && caller.first.split(" ").last == "`build_repo`" if path == gem_repo1 && caller.first.split(" ").last == "`build_repo`"
raise "Updating gem_repo1 is unsupported -- use gem_repo2 instead" raise "Updating gem_repo1 is unsupported -- use gem_repo2 instead"
end end
@ -258,7 +258,12 @@ module Spec
@_build_repo = File.basename(path) @_build_repo = File.basename(path)
yield yield
with_gem_path_as Path.base_system_gem_path do with_gem_path_as Path.base_system_gem_path do
gem_command :generate_index, dir: path Dir[Spec::Path.base_system_gem_path.join("gems/rubygems-generate_index*/lib")].first ||
raise("Could not find rubygems-generate_index lib directory in #{Spec::Path.base_system_gem_path}")
command = "generate_index"
command += " --no-compact" if !build_compact_index && gem_command(command + " --help").include?("--[no-]compact")
gem_command command, dir: path
end end
ensure ensure
@_build_path = nil @_build_path = nil

View File

@ -1,81 +0,0 @@
# frozen_string_literal: true
require_relative "helper"
require "rubygems/indexer"
require "rubygems/commands/generate_index_command"
class TestGemCommandsGenerateIndexCommand < Gem::TestCase
def setup
super
@cmd = Gem::Commands::GenerateIndexCommand.new
@cmd.options[:directory] = @gemhome
end
def test_execute
use_ui @ui do
@cmd.execute
end
specs = File.join @gemhome, "specs.4.8.gz"
assert File.exist?(specs), specs
end
def test_execute_no_modern
@cmd.options[:modern] = false
use_ui @ui do
@cmd.execute
end
specs = File.join @gemhome, "specs.4.8.gz"
assert File.exist?(specs), specs
end
def test_handle_options_directory
return if Gem.win_platform?
refute_equal "/nonexistent", @cmd.options[:directory]
@cmd.handle_options %w[--directory /nonexistent]
assert_equal "/nonexistent", @cmd.options[:directory]
end
def test_handle_options_directory_windows
return unless Gem.win_platform?
refute_equal "/nonexistent", @cmd.options[:directory]
@cmd.handle_options %w[--directory C:/nonexistent]
assert_equal "C:/nonexistent", @cmd.options[:directory]
end
def test_handle_options_update
@cmd.handle_options %w[--update]
assert @cmd.options[:update]
end
def test_handle_options_modern
use_ui @ui do
@cmd.handle_options %w[--modern]
end
assert_equal \
"WARNING: The \"--modern\" option has been deprecated and will be removed in Rubygems 4.0. Modern indexes (specs, latest_specs, and prerelease_specs) are always generated, so this option is not needed.\n",
@ui.error
end
def test_handle_options_no_modern
use_ui @ui do
@cmd.handle_options %w[--no-modern]
end
assert_equal \
"WARNING: The \"--no-modern\" option has been deprecated and will be removed in Rubygems 4.0. The `--no-modern` option is currently ignored. Modern indexes (specs, latest_specs, and prerelease_specs) are always generated.\n",
@ui.error
end
end

View File

@ -1,380 +0,0 @@
# frozen_string_literal: true
require_relative "helper"
require "rubygems/indexer"
class TestGemIndexer < Gem::TestCase
def setup
super
util_make_gems
@d2_0 = util_spec "d", "2.0" do |s|
s.date = Gem::Specification::TODAY - 86_400 * 3
end
util_build_gem @d2_0
@d2_0_a = util_spec "d", "2.0.a"
util_build_gem @d2_0_a
@d2_0_b = util_spec "d", "2.0.b"
util_build_gem @d2_0_b
@default = new_default_spec "default", 2
install_default_gems @default
@indexerdir = File.join(@tempdir, "indexer")
gems = File.join(@indexerdir, "gems")
FileUtils.mkdir_p gems
FileUtils.mv Dir[File.join(@gemhome, "cache", "*.gem")], gems
@indexer = Gem::Indexer.new(@indexerdir)
end
def teardown
FileUtils.rm_rf(@indexer.directory)
ensure
super
end
def with_indexer(dir, **opts)
indexer = Gem::Indexer.new(dir, **opts)
build_directory = indexer.directory
yield indexer
ensure
FileUtils.rm_rf(build_directory) if build_directory
end
def test_initialize
assert_equal @indexerdir, @indexer.dest_directory
Dir.mktmpdir("gem_generate_index") do |tmpdir|
assert_match(%r{#{tmpdir.match(/.*-/)}}, @indexer.directory) # rubocop:disable Style/RegexpLiteral
end
with_indexer(@indexerdir) do |indexer|
assert_predicate indexer, :build_modern
end
with_indexer(@indexerdir, build_modern: true) do |indexer|
assert_predicate indexer, :build_modern
end
end
def test_build_indices
@indexer.make_temp_directories
use_ui @ui do
@indexer.build_indices
end
specs_path = File.join @indexer.directory, "specs.#{@marshal_version}"
specs_dump = Gem.read_binary specs_path
specs = Marshal.load specs_dump
expected = [["a", Gem::Version.new("1"), "ruby"],
["a", Gem::Version.new("2"), "ruby"],
["a_evil", Gem::Version.new("9"), "ruby"],
["b", Gem::Version.new("2"), "ruby"],
["c", Gem::Version.new("1.2"), "ruby"],
["d", Gem::Version.new("2.0"), "ruby"],
["dep_x", Gem::Version.new("1"), "ruby"],
["pl", Gem::Version.new("1"), "i386-linux"],
["x", Gem::Version.new("1"), "ruby"]]
assert_equal expected, specs
latest_specs_path = File.join(@indexer.directory,
"latest_specs.#{@marshal_version}")
latest_specs_dump = Gem.read_binary latest_specs_path
latest_specs = Marshal.load latest_specs_dump
expected = [["a", Gem::Version.new("2"), "ruby"],
["a_evil", Gem::Version.new("9"), "ruby"],
["b", Gem::Version.new("2"), "ruby"],
["c", Gem::Version.new("1.2"), "ruby"],
["d", Gem::Version.new("2.0"), "ruby"],
["dep_x", Gem::Version.new("1"), "ruby"],
["pl", Gem::Version.new("1"), "i386-linux"],
["x", Gem::Version.new("1"), "ruby"]]
assert_equal expected, latest_specs, "latest_specs"
end
def test_generate_index
use_ui @ui do
@indexer.generate_index
end
quickdir = File.join @indexerdir, "quick"
marshal_quickdir = File.join quickdir, "Marshal.#{@marshal_version}"
assert_directory_exists quickdir
assert_directory_exists marshal_quickdir
assert_indexed marshal_quickdir, "#{File.basename(@a1.spec_file)}.rz"
assert_indexed marshal_quickdir, "#{File.basename(@a2.spec_file)}.rz"
refute_indexed marshal_quickdir, File.basename(@c1_2.spec_file)
assert_indexed @indexerdir, "specs.#{@marshal_version}"
assert_indexed @indexerdir, "specs.#{@marshal_version}.gz"
assert_indexed @indexerdir, "latest_specs.#{@marshal_version}"
assert_indexed @indexerdir, "latest_specs.#{@marshal_version}.gz"
refute_directory_exists @indexer.directory
end
def test_generate_index_modern
@indexer.build_modern = true
use_ui @ui do
@indexer.generate_index
end
refute_indexed @indexerdir, "yaml"
refute_indexed @indexerdir, "yaml.Z"
refute_indexed @indexerdir, "Marshal.#{@marshal_version}"
refute_indexed @indexerdir, "Marshal.#{@marshal_version}.Z"
quickdir = File.join @indexerdir, "quick"
marshal_quickdir = File.join quickdir, "Marshal.#{@marshal_version}"
assert_directory_exists quickdir, "quickdir should be directory"
assert_directory_exists marshal_quickdir
refute_indexed quickdir, "index"
refute_indexed quickdir, "index.rz"
refute_indexed quickdir, "latest_index"
refute_indexed quickdir, "latest_index.rz"
refute_indexed quickdir, "#{File.basename(@a1.spec_file)}.rz"
refute_indexed quickdir, "#{File.basename(@a2.spec_file)}.rz"
refute_indexed quickdir, "#{File.basename(@b2.spec_file)}.rz"
refute_indexed quickdir, "#{File.basename(@c1_2.spec_file)}.rz"
refute_indexed quickdir, "#{@pl1.original_name}.gemspec.rz"
refute_indexed quickdir, "#{File.basename(@pl1.spec_file)}.rz"
assert_indexed marshal_quickdir, "#{File.basename(@a1.spec_file)}.rz"
assert_indexed marshal_quickdir, "#{File.basename(@a2.spec_file)}.rz"
refute_indexed quickdir, File.basename(@c1_2.spec_file).to_s
refute_indexed marshal_quickdir, File.basename(@c1_2.spec_file).to_s
assert_indexed @indexerdir, "specs.#{@marshal_version}"
assert_indexed @indexerdir, "specs.#{@marshal_version}.gz"
assert_indexed @indexerdir, "latest_specs.#{@marshal_version}"
assert_indexed @indexerdir, "latest_specs.#{@marshal_version}.gz"
end
def test_generate_index_modern_back_to_back
@indexer.build_modern = true
use_ui @ui do
@indexer.generate_index
end
with_indexer @indexerdir do |indexer|
indexer.build_modern = true
use_ui @ui do
indexer.generate_index
end
quickdir = File.join @indexerdir, "quick"
marshal_quickdir = File.join quickdir, "Marshal.#{@marshal_version}"
assert_directory_exists quickdir
assert_directory_exists marshal_quickdir
assert_indexed marshal_quickdir, "#{File.basename(@a1.spec_file)}.rz"
assert_indexed marshal_quickdir, "#{File.basename(@a2.spec_file)}.rz"
assert_indexed @indexerdir, "specs.#{@marshal_version}"
assert_indexed @indexerdir, "specs.#{@marshal_version}.gz"
assert_indexed @indexerdir, "latest_specs.#{@marshal_version}"
assert_indexed @indexerdir, "latest_specs.#{@marshal_version}.gz"
end
end
def test_generate_index_ui
use_ui @ui do
@indexer.generate_index
end
assert_match(/^\.\.\.\.\.\.\.\.\.\.\.\.$/, @ui.output)
assert_match(/^Generating Marshal quick index gemspecs for 12 gems$/, @ui.output)
assert_match(/^Complete$/, @ui.output)
assert_match(/^Generating specs index$/, @ui.output)
assert_match(/^Generating latest specs index$/, @ui.output)
assert_match(/^Generating prerelease specs index$/, @ui.output)
assert_match(/^Complete$/, @ui.output)
assert_match(/^Compressing indices$/, @ui.output)
assert_equal "", @ui.error
end
def test_generate_index_specs
use_ui @ui do
@indexer.generate_index
end
specs_path = File.join @indexerdir, "specs.#{@marshal_version}"
specs_dump = Gem.read_binary specs_path
specs = Marshal.load specs_dump
expected = [
["a", Gem::Version.new(1), "ruby"],
["a", Gem::Version.new(2), "ruby"],
["a_evil", Gem::Version.new(9), "ruby"],
["b", Gem::Version.new(2), "ruby"],
["c", Gem::Version.new("1.2"), "ruby"],
["d", Gem::Version.new("2.0"), "ruby"],
["dep_x", Gem::Version.new(1), "ruby"],
["pl", Gem::Version.new(1), "i386-linux"],
["x", Gem::Version.new(1), "ruby"],
]
assert_equal expected, specs
assert_same specs[0].first, specs[1].first,
"identical names not identical"
assert_same specs[0][1], specs[-1][1],
"identical versions not identical"
assert_same specs[0].last, specs[1].last,
"identical platforms not identical"
refute_same specs[1][1], specs[5][1],
"different versions not different"
end
def test_generate_index_latest_specs
use_ui @ui do
@indexer.generate_index
end
latest_specs_path = File.join @indexerdir, "latest_specs.#{@marshal_version}"
latest_specs_dump = Gem.read_binary latest_specs_path
latest_specs = Marshal.load latest_specs_dump
expected = [
["a", Gem::Version.new(2), "ruby"],
["a_evil", Gem::Version.new(9), "ruby"],
["b", Gem::Version.new(2), "ruby"],
["c", Gem::Version.new("1.2"), "ruby"],
["d", Gem::Version.new("2.0"), "ruby"],
["dep_x", Gem::Version.new(1), "ruby"],
["pl", Gem::Version.new(1), "i386-linux"],
["x", Gem::Version.new(1), "ruby"],
]
assert_equal expected, latest_specs
assert_same latest_specs[0][1], latest_specs[2][1],
"identical versions not identical"
assert_same latest_specs[0].last, latest_specs[1].last,
"identical platforms not identical"
end
def test_generate_index_prerelease_specs
use_ui @ui do
@indexer.generate_index
end
prerelease_specs_path = File.join @indexerdir, "prerelease_specs.#{@marshal_version}"
prerelease_specs_dump = Gem.read_binary prerelease_specs_path
prerelease_specs = Marshal.load prerelease_specs_dump
assert_equal [["a", Gem::Version.new("3.a"), "ruby"],
["d", Gem::Version.new("2.0.a"), "ruby"],
["d", Gem::Version.new("2.0.b"), "ruby"]],
prerelease_specs
end
##
# Emulate the starting state of Gem::Specification in a live environment,
# where it will carry the list of system gems
def with_system_gems
Gem::Specification.reset
sys_gem = util_spec "systemgem", "1.0"
util_build_gem sys_gem
install_default_gems sys_gem
yield
util_remove_gem sys_gem
end
def test_update_index
use_ui @ui do
@indexer.generate_index
end
quickdir = File.join @indexerdir, "quick"
marshal_quickdir = File.join quickdir, "Marshal.#{@marshal_version}"
assert_directory_exists quickdir
assert_directory_exists marshal_quickdir
@d2_1 = util_spec "d", "2.1"
util_build_gem @d2_1
@d2_1_tuple = [@d2_1.name, @d2_1.version, @d2_1.original_platform]
@d2_1_a = util_spec "d", "2.2.a"
util_build_gem @d2_1_a
@d2_1_a_tuple = [@d2_1_a.name, @d2_1_a.version, @d2_1_a.original_platform]
gems = File.join @indexerdir, "gems"
FileUtils.mv @d2_1.cache_file, gems
FileUtils.mv @d2_1_a.cache_file, gems
with_system_gems do
use_ui @ui do
@indexer.update_index
end
assert_indexed marshal_quickdir, "#{File.basename(@d2_1.spec_file)}.rz"
specs_index = Marshal.load Gem.read_binary(@indexer.dest_specs_index)
assert_includes specs_index, @d2_1_tuple
refute_includes specs_index, @d2_1_a_tuple
latest_specs_index = Marshal.load \
Gem.read_binary(@indexer.dest_latest_specs_index)
assert_includes latest_specs_index, @d2_1_tuple
assert_includes latest_specs_index,
[@d2_0.name, @d2_0.version, @d2_0.original_platform]
refute_includes latest_specs_index, @d2_1_a_tuple
pre_specs_index = Marshal.load \
Gem.read_binary(@indexer.dest_prerelease_specs_index)
assert_includes pre_specs_index, @d2_1_a_tuple
refute_includes pre_specs_index, @d2_1_tuple
refute_directory_exists @indexer.directory
end
end
def assert_indexed(dir, name)
file = File.join dir, name
assert File.exist?(file), "#{file} does not exist"
end
def refute_indexed(dir, name)
file = File.join dir, name
refute File.exist?(file), "#{file} exists"
end
end

View File

@ -2,7 +2,6 @@
require_relative "helper" require_relative "helper"
require "rubygems/source" require "rubygems/source"
require "rubygems/indexer"
class TestGemSource < Gem::TestCase class TestGemSource < Gem::TestCase
def tuple(*args) def tuple(*args)
@ -55,7 +54,8 @@ class TestGemSource < Gem::TestCase
end end
def test_dependency_resolver_set_file_uri def test_dependency_resolver_set_file_uri
Gem::Indexer.new(@tempdir).generate_index File.write(File.join(@tempdir, "prerelease_specs.4.8.gz"), Gem::Util.gzip("\x04\x08[\x05".b))
File.write(File.join(@tempdir, "specs.4.8.gz"), Gem::Util.gzip("\x04\x08[\x05".b))
source = Gem::Source.new "file://#{@tempdir}/" source = Gem::Source.new "file://#{@tempdir}/"