1berry/ruby
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

[rubygems/rubygems] Improve errors and register checksums reliably

Browse Source 
Improve error reporting for checksums, raises a new error class.

Solve for multi-source checksum errors.

Add CHECKSUMS to tool/bundler/(dev|standard|rubocop)26_gems.rb

https://github.com/rubygems/rubygems/commit/26ceee0e76

Co-authored-by: Samuel Giddins <segiddins@segiddins.me>
This commit is contained in:
Martin Emde 2023-09-01 15:15:49 -07:00 committed by Hiroshi SHIBATA
parent 6362bfdc33
commit c667de72ff
No known key found for this signature in database
GPG Key ID: F9CF13417264FAC2
24 changed files with 543 additions and 273 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

248
lib/bundler/checksum.rb
View File

@ -2,37 +2,38 @@
module Bundler
class Checksum
DEFAULT_ALGORITHM = "sha256"
private_constant :DEFAULT_ALGORITHM
DEFAULT_BLOCK_SIZE = 16_384
private_constant :DEFAULT_BLOCK_SIZE
class << self
def from_gem_source(source, digest_algorithms: %w[sha256])
raise ArgumentError, "not a valid gem source: #{source}" unless source.respond_to?(:with_read_io)
source.with_read_io do |io|
checksums = from_io(io, "#{source.path || source.inspect} gem source hexdigest", :digest_algorithms => digest_algorithms)
io.rewind
return checksums
end
def from_gem(io, pathname, algo = DEFAULT_ALGORITHM)
digest = Bundler::SharedHelpers.digest(algo.upcase).new
buf = String.new(:capacity => DEFAULT_BLOCK_SIZE)
digest << io.readpartial(DEFAULT_BLOCK_SIZE, buf) until io.eof?
Checksum.new(algo, digest.hexdigest!, Source.new(:gem, pathname))
end
def from_io(io, source, digest_algorithms: %w[sha256])
digests = digest_algorithms.to_h do |algo|
[algo.to_s, Bundler::SharedHelpers.digest(algo.upcase).new]
def from_api(digest, source_uri)
# transform the bytes from base64 to hex, switch to unpack1 when we drop older rubies
hexdigest = digest.length == 44 ? digest.unpack("m0").first.unpack("H*").first : digest
if hexdigest.length != 64
raise ArgumentError, "#{digest.inspect} is not a valid SHA256 hexdigest nor base64digest"
end
until io.eof?
ret = io.read DEFAULT_BLOCK_SIZE
digests.each_value {|digest| digest << ret }
end
Checksum.new(DEFAULT_ALGORITHM, hexdigest, Source.new(:api, source_uri))
end
digests.map do |algo, digest|
Checksum.new(algo, digest.hexdigest!, source)
end
def from_lock(lock_checksum, lockfile_location)
algo, digest = lock_checksum.strip.split("-", 2)
Checksum.new(algo, digest, Source.new(:lock, lockfile_location))
end
end
attr_reader :algo, :digest, :sources
def initialize(algo, digest, source)
@algo = algo
@digest = digest
@ -62,20 +63,81 @@ module Bundler
end
def merge!(other)
raise ArgumentError, "cannot merge checksums of different algorithms" unless algo == other.algo
unless digest == other.digest
raise SecurityError, <<~MESSAGE
#{other}
#{to_lock} from:
* #{sources.join("\n* ")}
MESSAGE
end
return nil unless match?(other)
@sources.concat(other.sources).uniq!
self
end
def formatted_sources
sources.join("\n and ").concat("\n")
end
def removable?
sources.all?(&:removable?)
end
def removal_instructions
msg = +""
i = 1
sources.each do |source|
msg << " #{i}. #{source.removal}\n"
i += 1
end
msg << " #{i}. run `bundle install`\n"
end
def inspect
abbr = "#{algo}-#{digest[0, 8]}"
from = "from #{sources.join(" and ")}"
"#<#{self.class}:#{object_id} #{abbr} #{from}>"
end
class Source
attr_reader :type, :location
def initialize(type, location)
@type = type
@location = location
end
def removable?
type == :lock || type == :gem
end
def ==(other)
other.is_a?(self.class) && other.type == type && other.location == location
end
# phrased so that the usual string format is grammatically correct
# rake (10.3.2) sha256-abc123 from #{to_s}
def to_s
case type
when :lock
"the lockfile CHECKSUMS at #{location}"
when :gem
"the gem at #{location}"
when :api
"the API at #{location}"
else
"#{location} (#{type})"
end
end
# A full sentence describing how to remove the checksum
def removal
case type
when :lock
"remove the matching checksum in #{location}"
when :gem
"remove the gem at #{location}"
when :api
"checksums from #{location} cannot be locally modified, you may need to update your sources"
else
"remove #{location} (#{type})"
end
end
end
class Store
attr_reader :store
protected :store
@ -86,89 +148,81 @@ module Bundler
def initialize_copy(other)
@store = {}
other.store.each do |full_name, checksums|
store[full_name] = checksums.dup
other.store.each do |name_tuple, checksums|
store[name_tuple] = checksums.dup
end
end
def checksums(full_name)
store[full_name]
def inspect
"#<#{self.class}:#{object_id} size=#{store.size}>"
end
def register_gem_package(spec, source)
new_checksums = Checksum.from_gem_source(source)
new_checksums.each do |checksum|
register spec.full_name, checksum
end
rescue SecurityError
expected = checksums(spec.full_name)
gem_lock_name = GemHelpers.lock_name(spec.name, spec.version, spec.platform)
raise SecurityError, <<~MESSAGE
Bundler cannot continue installing #{gem_lock_name}.
The checksum for the downloaded `#{spec.full_name}.gem` does not match \
the known checksum for the gem.
This means the contents of the downloaded \
gem is different from what was uploaded to the server \
or first used by your teammates, and could be a potential security issue.
To resolve this issue:
1. delete the downloaded gem located at: `#{source.path}`
2. run `bundle install`
If you are sure that the new checksum is correct, you can \
remove the `#{gem_lock_name}` entry under the lockfile `CHECKSUMS` \
section and rerun `bundle install`.
If you wish to continue installing the downloaded gem, and are certain it does not pose a \
security issue despite the mismatching checksum, do the following:
1. run `bundle config set --local disable_checksum_validation true` to turn off checksum verification
2. run `bundle install`
#{expected.map do |checksum|
next unless actual = new_checksums.find {|c| c.algo == checksum.algo }
next if actual.digest == checksum.digest
"(More info: The expected #{checksum.algo.upcase} checksum was #{checksum.digest.inspect}, but the " \
"checksum for the downloaded gem was #{actual.digest.inspect}. The expected checksum came from: #{checksum.sources.join(", ")})"
end.compact.join("\n")}
MESSAGE
def fetch(spec, algo = DEFAULT_ALGORITHM)
store[spec.name_tuple]&.fetch(algo, nil)
end
def register(full_name, checksum)
# Replace when the new checksum is from the same source.
# The primary purpose of this registering checksums from gems where there are
# duplicates of the same gem (according to full_name) in the index.
# In particular, this is when 2 gems have two similar platforms, e.g.
# "darwin20" and "darwin-20", both of which resolve to darwin-20.
# In the Index, the later gem replaces the former, so we do that here.
#
# However, if the new checksum is from a different source, we register like normal.
# This ensures a mismatch error where there are multiple top level sources
# that contain the same gem with different checksums.
def replace(spec, checksum)
return if Bundler.settings[:disable_checksum_validation]
return unless checksum
sums = (store[full_name] ||= [])
sums.find {|c| c.algo == checksum.algo }&.merge!(checksum) || sums << checksum
rescue SecurityError => e
raise e.exception(<<~MESSAGE)
Bundler found multiple different checksums for #{full_name}.
This means that there are multiple different `#{full_name}.gem` files.
This is a potential security issue, since Bundler could be attempting \
to install a different gem than what you expect.
name_tuple = spec.name_tuple
checksums = (store[name_tuple] ||= {})
existing = checksums[checksum.algo]
#{e.message}
To resolve this issue:
1. delete any downloaded gems referenced above
2. run `bundle install`
If you are sure that the new checksum is correct, you can \
remove the `#{full_name}` entry under the lockfile `CHECKSUMS` \
section and rerun `bundle install`.
If you wish to continue installing the downloaded gem, and are certain it does not pose a \
security issue despite the mismatching checksum, do the following:
1. run `bundle config set --local disable_checksum_validation true` to turn off checksum verification
2. run `bundle install`
MESSAGE
# we assume only one source because this is used while building the index
if !existing || existing.sources.first == checksum.sources.first
checksums[checksum.algo] = checksum
else
register_checksum(name_tuple, checksum)
end
end
def replace(full_name, checksum)
store[full_name] = checksum ? [checksum] : nil
def register(spec, checksum)
return if Bundler.settings[:disable_checksum_validation]
return unless checksum
register_checksum(spec.name_tuple, checksum)
end
def register_store(other)
other.store.each do |full_name, checksums|
checksums.each {|checksum| register(full_name, checksum) }
def merge!(other)
other.store.each do |name_tuple, checksums|
checksums.each do |_algo, checksum|
register_checksum(name_tuple, checksum)
end
end
end
def to_lock(spec)
name_tuple = spec.name_tuple
if checksums = store[name_tuple]
"#{name_tuple.lock_name} #{checksums.values.map(&:to_lock).sort.join(",")}"
else
name_tuple.lock_name
end
end
private
def register_checksum(name_tuple, checksum)
return unless checksum
checksums = (store[name_tuple] ||= {})
existing = checksums[checksum.algo]
if !existing
checksums[checksum.algo] = checksum
elsif existing.merge!(checksum)
checksum
else
raise ChecksumMismatchError.new(name_tuple, existing, checksum)
end
end
end

4
lib/bundler/definition.rb
View File

@ -751,8 +751,8 @@ module Bundler
sources.all_sources.each do |source|
# has to be done separately, because we want to keep the locked checksum
# store for a source, even when doing a full update
if @locked_gems && locked_source = @locked_gems.sources.find {|s| s == source }
source.checksum_store.register_store(locked_source.checksum_store)
if @locked_gems && locked_source = @locked_gems.sources.find {|s| s == source && !s.equal?(source) }
source.checksum_store.merge!(locked_source.checksum_store)
end
# If the source is unlockable and the current command allows an unlock of
# the source (for example, you are doing a `bundle update <foo>` of a git-pinned

13
lib/bundler/endpoint_specification.rb
View File

@ -126,16 +126,11 @@ module Bundler
case k.to_s
when "checksum"
next if Bundler.settings[:disable_checksum_validation]
digest = v.last
if digest.length == 64
# nothing to do, it's a hexdigest
elsif digest.length == 44
# transform the bytes from base64 to hex
digest = digest.unpack("m0").first.unpack("H*").first
else
raise ArgumentError, "The given checksum for #{full_name} (#{digest.inspect}) is not a valid SHA256 hexdigest nor base64digest"
begin
@checksum = Checksum.from_api(v.last, @spec_fetcher.uri)
rescue ArgumentError => e
raise ArgumentError, "Invalid checksum for #{full_name}: #{e.message}"
end
@checksum = Checksum.new("sha256", digest, "API response from #{@spec_fetcher.uri}")
when "rubygems"
@required_rubygems_version = Gem::Requirement.new(v)
when "ruby"

43
lib/bundler/errors.rb
View File

@ -52,6 +52,49 @@ module Bundler
class GemfileEvalError < GemfileError; end
class MarshalError < StandardError; end
class ChecksumMismatchError < SecurityError
def initialize(name_tuple, existing, checksum)
@name_tuple = name_tuple
@existing = existing
@checksum = checksum
end
def message
<<~MESSAGE
Bundler found mismatched checksums. This is a potential security risk.
#{@name_tuple.lock_name} #{@existing.to_lock}
from #{@existing.sources.join("\n and ")}
#{@name_tuple.lock_name} #{@checksum.to_lock}
from #{@checksum.sources.join("\n and ")}
#{mismatch_resolution_instructions}
To ignore checksum security warnings, disable checksum validation with
`bundle config set --local disable_checksum_validation true`
MESSAGE
end
def mismatch_resolution_instructions
removable, remote = [@existing, @checksum].partition(&:removable?)
case removable.size
when 0
msg = +"Mismatched checksums each have an authoritative source:\n"
msg << " 1. #{@existing.sources.reject(&:removable?).map(&:to_s).join(" and ")}\n"
msg << " 2. #{@checksum.sources.reject(&:removable?).map(&:to_s).join(" and ")}\n"
msg << "You may need to alter your Gemfile sources to resolve this issue.\n"
when 1
msg = +"If you trust #{remote.first.sources.first}, to resolve this issue you can:\n"
msg << removable.first.removal_instructions
when 2
msg = +"To resolve this issue you can either:\n"
msg << @checksum.removal_instructions
msg << "or if you are sure that the new checksum from #{@checksum.sources.first} is correct:\n"
msg << @existing.removal_instructions
end
end
status_code(37)
end
class PermissionError < BundlerError
def initialize(path, permission_type = :write)
@path = path

6
lib/bundler/fetcher.rb
View File

@ -140,11 +140,7 @@ module Bundler
fetch_specs(gem_names).each do |name, version, platform, dependencies, metadata|
spec = if dependencies
EndpointSpecification.new(name, version, platform, self, dependencies, metadata).tap do |es|
# Duplicate spec.full_names, different spec.original_names
# index#<< ensures that the last one added wins, so if we're overriding
# here, make sure to also override the checksum, otherwise downloading the
# specs (even if that version is completely unused) will cause a SecurityError
source.checksum_store.replace(es.full_name, es.checksum)
source.checksum_store.replace(es, es.checksum)
end
else
RemoteSpecification.new(name, version, platform, self)

18
lib/bundler/gem_helpers.rb
View File

@ -113,23 +113,5 @@ module Bundler
same_runtime_deps && same_metadata_deps
end
module_function :same_deps
def spec_full_name(name, version, platform)
if platform == Gem::Platform::RUBY
"#{name}-#{version}"
else
"#{name}-#{version}-#{platform}"
end
end
module_function :spec_full_name
def lock_name(name, version, platform)
if platform == Gem::Platform::RUBY
"#{name} (#{version})"
else
"#{name} (#{version}-#{platform})"
end
end
module_function :lock_name
end
end

18
lib/bundler/lazy_specification.rb
View File

@ -20,7 +20,19 @@ module Bundler
end
def full_name
@full_name ||= GemHelpers.spec_full_name(@name, @version, platform)
@full_name ||= if platform == Gem::Platform::RUBY
"#{@name}-#{@version}"
else
"#{@name}-#{@version}-#{platform}"
end
end
def lock_name
@lock_name ||= name_tuple.lock_name
end
def name_tuple
Gem::NameTuple.new(@name, @version, @platform)
end
def ==(other)
@ -57,7 +69,7 @@ module Bundler
def to_lock
out = String.new
out << " #{GemHelpers.lock_name(name, version, platform)}\n"
out << " #{lock_name}\n"
dependencies.sort_by(&:to_s).uniq.each do |dep|
next if dep.type == :development
@ -113,7 +125,7 @@ module Bundler
end
def to_s
@to_s ||= GemHelpers.lock_name(name, version, platform)
lock_name
end
def git_version

11
lib/bundler/lockfile_generator.rb
View File

@ -67,15 +67,10 @@ module Bundler
end
def add_checksums
out << "\nCHECKSUMS\n"
definition.resolve.sort_by(&:full_name).each do |spec|
lock_name = GemHelpers.lock_name(spec.name, spec.version, spec.platform)
out << " #{lock_name}"
checksums = spec.source.checksum_store.checksums(spec.full_name)
out << " #{checksums.map(&:to_lock).sort.join(",")}" if checksums
out << "\n"
checksums = definition.resolve.map do |spec|
spec.source.checksum_store.to_lock(spec)
end
add_section("CHECKSUMS", checksums)
end
def add_locked_ruby_version

20
lib/bundler/lockfile_parser.rb
View File

@ -101,7 +101,10 @@ module Bundler
"Run `git checkout HEAD -- #{@lockfile_path}` first to get a clean lock."
end
lockfile.split(/((?:\r?\n)+)/).each_slice(2) do |line, whitespace|
lockfile.split(/((?:\r?\n)+)/) do |line|
# split alternates between the line and the following whitespace
next @pos.advance!(line) if line.match?(/^\s*$/)
if SOURCE.include?(line)
@parse_method = :parse_source
parse_source(line)
@ -121,7 +124,6 @@ module Bundler
send(@parse_method, line)
end
@pos.advance!(line)
@pos.advance!(whitespace)
end
@specs = @specs.values.sort_by!(&:full_name)
rescue ArgumentError => e
@ -217,23 +219,23 @@ module Bundler
spaces = $1
return unless spaces.size == 2
checksums = $6
return unless checksums
name = $2
version = $3
platform = $4
checksums = $6
return unless checksums
version = Gem::Version.new(version)
platform = platform ? Gem::Platform.new(platform) : Gem::Platform::RUBY
full_name = GemHelpers.spec_full_name(name, version, platform)
full_name = Gem::NameTuple.new(name, version, platform).full_name
# Don't raise exception if there's a checksum for a gem that's not in the lockfile,
# we prefer to heal invalid lockfiles
return unless spec = @specs[full_name]
checksums.split(",").each do |c|
algo, digest = c.split("-", 2)
lock_name = GemHelpers.lock_name(spec.name, spec.version, spec.platform)
spec.source.checksum_store.register(full_name, Checksum.new(algo, digest, "#{@lockfile_path}:#{@pos} CHECKSUMS #{lock_name}"))
checksums.split(",") do |lock_checksum|
column = line.index(lock_checksum) + 1
checksum = Checksum.from_lock(lock_checksum, "#{@lockfile_path}:#{@pos.line}:#{column}")
spec.source.checksum_store.register(spec, checksum)
end
end

21
lib/bundler/rubygems_ext.rb
View File

@ -359,6 +359,27 @@ module Gem
end
end
require "rubygems/name_tuple"
class NameTuple
def self.new(name, version, platform="ruby")
if Gem::Platform === platform
super(name, version, platform.to_s)
else
super
end
end
def lock_name
@lock_name ||=
if platform == Gem::Platform::RUBY
"#{name} (#{version})"
else
"#{name} (#{version}-#{platform})"
end
end
end
require "rubygems/util"
Util.singleton_class.module_eval do

25
lib/bundler/rubygems_gem_installer.rb
View File

@ -60,10 +60,6 @@ module Bundler
end
end
def pre_install_checks
super && validate_bundler_checksum(options[:bundler_checksum_store])
end
def build_extensions
extension_cache_path = options[:bundler_extension_cache_path]
extension_dir = spec.extension_dir
@ -98,6 +94,18 @@ module Bundler
end
end
def gem_checksum
return nil if Bundler.settings[:disable_checksum_validation]
return nil unless source = @package.instance_variable_get(:@gem)
return nil unless source.respond_to?(:with_read_io)
source.with_read_io do |io|
Checksum.from_gem(io, source.path)
ensure
io.rewind
end
end
private
def prepare_extension_build(extension_dir)
@ -114,14 +122,5 @@ module Bundler
raise DirectoryRemovalError.new(e, "Could not delete previous installation of `#{dir}`")
end
def validate_bundler_checksum(checksum_store)
return true if Bundler.settings[:disable_checksum_validation]
return true unless source = @package.instance_variable_get(:@gem)
return true unless source.respond_to?(:with_read_io)
checksum_store.register_gem_package spec, source
true
end
end
end

3
lib/bundler/source/rubygems.rb
View File

@ -178,7 +178,6 @@ module Bundler
:wrappers => true,
:env_shebang => true,
:build_args => options[:build_args],
:bundler_checksum_store => spec.source.checksum_store,
:bundler_extension_cache_path => extension_cache_path(spec)
)
@ -197,6 +196,8 @@ module Bundler
spec.__swap__(s)
end
spec.source.checksum_store.register(spec, installer.gem_checksum)
message = "Installing #{version_message(spec, options[:previous_spec])}"
message += " with native extensions" if spec.extensions.any?
Bundler.ui.confirm message

47
spec/bundler/bundler/lockfile_parser_spec.rb
View File

@ -119,6 +119,12 @@ RSpec.describe Bundler::LockfileParser do
let(:bundler_version) { Gem::Version.new("1.12.0.rc.2") }
let(:ruby_version) { "ruby 2.1.3p242" }
let(:lockfile_path) { Bundler.default_lockfile.relative_path_from(Dir.pwd) }
let(:rake_checksum) do
Bundler::Checksum.from_lock(
"sha256-814828c34f1315d7e7b7e8295184577cc4e969bad6156ac069d02d63f58d82e8",
"#{lockfile_path}:??:1"
)
end
shared_examples_for "parsing" do
it "parses correctly" do
@ -129,11 +135,9 @@ RSpec.describe Bundler::LockfileParser do
expect(subject.platforms).to eq platforms
expect(subject.bundler_version).to eq bundler_version
expect(subject.ruby_version).to eq ruby_version
checksums = subject.sources.last.checksum_store.checksums("rake-10.3.2")
expect(checksums.size).to eq(1)
expected_checksum = Bundler::Checksum.new("sha256", "814828c34f1315d7e7b7e8295184577cc4e969bad6156ac069d02d63f58d82e8", "#{lockfile_path}:??:1")
expect(checksums.first).to be_match(expected_checksum)
expect(checksums.first.sources.first).to match(/#{Regexp.escape(lockfile_path.to_s)}:\d+:\d+/)
checksum = subject.sources.last.checksum_store.fetch(specs.last)
expect(checksum).to be_match(rake_checksum)
expect(checksum.sources.first.to_s).to match(/the lockfile CHECKSUMS at #{Regexp.escape(lockfile_path.to_s)}:\d+:\d+/)
end
end
@ -159,29 +163,28 @@ RSpec.describe Bundler::LockfileParser do
include_examples "parsing"
end
context "when CHECKSUMS has duplicate checksums that don't match" do
let(:lockfile_contents) { super().split(/(?<=CHECKSUMS\n)/m).insert(1, " rake (10.3.2) sha256-69b69b69b69b69b69b69b69b69b69b69b69b69b69b69b69b69b69b69b69b69b6\n").join }
context "when CHECKSUMS has duplicate checksums in the lockfile that don't match" do
let(:bad_checksum) { "sha256-c0ffee11c0ffee11c0ffee11c0ffee11c0ffee11c0ffee11c0ffee11c0ffee11" }
let(:lockfile_contents) { super().split(/(?<=CHECKSUMS\n)/m).insert(1, " rake (10.3.2) #{bad_checksum}\n").join }
it "raises a security error" do
expect { subject }.to raise_error(Bundler::SecurityError) do |e|
expect(e.message).to match <<~MESSAGE
Bundler found multiple different checksums for rake-10.3.2.
This means that there are multiple different `rake-10.3.2.gem` files.
This is a potential security issue, since Bundler could be attempting to install a different gem than what you expect.
Bundler found mismatched checksums. This is a potential security risk.
rake (10.3.2) #{bad_checksum}
from the lockfile CHECKSUMS at #{lockfile_path}:20:17
rake (10.3.2) #{rake_checksum.to_lock}
from the lockfile CHECKSUMS at #{lockfile_path}:21:17
sha256-814828c34f1315d7e7b7e8295184577cc4e969bad6156ac069d02d63f58d82e8 (from #{lockfile_path}:21:1 CHECKSUMS rake (10.3.2))
sha256-69b69b69b69b69b69b69b69b69b69b69b69b69b69b69b69b69b69b69b69b69b6 from:
* #{lockfile_path}:20:1 CHECKSUMS rake (10.3.2)
To resolve this issue you can either:
1. remove the matching checksum in #{lockfile_path}:21:17
2. run `bundle install`
or if you are sure that the new checksum from the lockfile CHECKSUMS at #{lockfile_path}:21:17 is correct:
1. remove the matching checksum in #{lockfile_path}:20:17
2. run `bundle install`
To resolve this issue:
1. delete any downloaded gems referenced above
2. run `bundle install`
If you are sure that the new checksum is correct, you can remove the `rake-10.3.2` entry under the lockfile `CHECKSUMS` section and rerun `bundle install`.
If you wish to continue installing the downloaded gem, and are certain it does not pose a security issue despite the mismatching checksum, do the following:
1. run `bundle config set --local disable_checksum_validation true` to turn off checksum verification
2. run `bundle install`
To ignore checksum security warnings, disable checksum validation with
`bundle config set --local disable_checksum_validation true`
MESSAGE
end
end

19
spec/bundler/cache/gems_spec.rb vendored
View File

@ -281,9 +281,26 @@ RSpec.describe "bundle cache" do
build_gem "rack", "1.0.0",
:path => bundled_app("vendor/cache"),
:rubygems_version => "1.3.2"
# This test is only really valid if the checksum isn't saved. It otherwise can't be the same gem. Tested below.
bundled_app_lock.write remove_checksums_from_lockfile(bundled_app_lock.read, "rack (1.0.0)")
simulate_new_machine
pending "Causes checksum mismatch exception"
bundle :install
expect(cached_gem("rack-1.0.0")).to exist
end
it "raises an error when the gem file is altered and produces a different checksum" do
cached_gem("rack-1.0.0").rmtree
build_gem "rack", "1.0.0", :path => bundled_app("vendor/cache")
simulate_new_machine
bundle :install, :raise_on_error => false
expect(exitstatus).to eq(37)
expect(err).to include("Bundler found mismatched checksums.")
expect(err).to include("1. remove the gem at #{cached_gem("rack-1.0.0")}")
expect(cached_gem("rack-1.0.0")).to exist
cached_gem("rack-1.0.0").rmtree
bundle :install
expect(cached_gem("rack-1.0.0")).to exist
end

6
spec/bundler/commands/lock_spec.rb
View File

@ -91,14 +91,16 @@ RSpec.describe "bundle lock" do
bundle "lock --update"
expect(read_lockfile).to eq(@lockfile)
expect(read_lockfile).to eq(remove_checksums_from_lockfile(@lockfile, "(2.3.2)"))
end
it "writes a lockfile when there is an outdated lockfile using a bundle is frozen" do
lockfile @lockfile.gsub("2.3.2", "2.3.1")
bundle "lock --update", :env => { "BUNDLE_FROZEN" => "true" }
# No checksums for the updated gems
expect(read_lockfile).to eq(remove_checksums_from_lockfile(@lockfile, " (2.3.2)"))
expect(read_lockfile).to eq(remove_checksums_from_lockfile(@lockfile, "(2.3.2)"))
end
it "does not fetch remote specs when using the --local option" do

2
spec/bundler/commands/update_spec.rb
View File

@ -551,7 +551,7 @@ RSpec.describe "bundle update" do
lockfile original_lockfile
bundle "lock --update"
expect(the_bundle).to include_gems("activesupport 6.0.4.1", "tzinfo 1.2.9")
expect(lockfile).to eq expected_lockfile
expect(lockfile).to eq(expected_lockfile)
end
end

42
spec/bundler/install/gemfile/gemspec_spec.rb
View File

@ -28,6 +28,16 @@ RSpec.describe "bundle install from an existing gemspec" do
x64_mingw_archs.join("\n ")
end
let(:x64_mingw_checksums) do
x64_mingw_archs.map do |arch|
if arch == "x64-mingw-ucrt"
gem_no_checksum "platform_specific", "1.0", arch
else
checksum_for_repo_gem gem_repo2, "platform_specific", "1.0", arch
end
end.join("\n ")
end
it "should install runtime and development dependencies" do
build_lib("foo", :path => tmp.join("foo")) do |s|
s.write("Gemfile", "source :rubygems\ngemspec")
@ -449,12 +459,6 @@ RSpec.describe "bundle install from an existing gemspec" do
it "keeps all platform dependencies in the lockfile" do
expect(the_bundle).to include_gems "foo 1.0", "platform_specific 1.0 RUBY"
expected_checksums = checksum_section do |c|
c.repo_gem gem_repo2, "platform_specific", "1.0"
c.repo_gem gem_repo2, "platform_specific", "1.0", "java"
c.repo_gem gem_repo2, "platform_specific", "1.0", x64_mingw32
end
expect(lockfile).to eq <<~L
PATH
remote: .
@ -479,7 +483,9 @@ RSpec.describe "bundle install from an existing gemspec" do
CHECKSUMS
foo (1.0)
#{expected_checksums}
#{checksum_for_repo_gem gem_repo2, "platform_specific", "1.0"}
#{checksum_for_repo_gem gem_repo2, "platform_specific", "1.0", "java"}
#{x64_mingw_checksums}
BUNDLED WITH
#{Bundler::VERSION}
@ -493,12 +499,6 @@ RSpec.describe "bundle install from an existing gemspec" do
it "keeps all platform dependencies in the lockfile" do
expect(the_bundle).to include_gems "foo 1.0", "platform_specific 1.0 RUBY"
expected_checksums = checksum_section do |c|
c.repo_gem gem_repo2, "platform_specific", "1.0"
c.repo_gem gem_repo2, "platform_specific", "1.0", "java"
c.repo_gem gem_repo2, "platform_specific", "1.0", x64_mingw32
end
expect(lockfile).to eq <<~L
PATH
remote: .
@ -523,7 +523,9 @@ RSpec.describe "bundle install from an existing gemspec" do
CHECKSUMS
foo (1.0)
#{expected_checksums}
#{checksum_for_repo_gem gem_repo2, "platform_specific", "1.0"}
#{checksum_for_repo_gem gem_repo2, "platform_specific", "1.0", "java"}
#{x64_mingw_checksums}
BUNDLED WITH
#{Bundler::VERSION}
@ -538,13 +540,6 @@ RSpec.describe "bundle install from an existing gemspec" do
it "keeps all platform dependencies in the lockfile" do
expect(the_bundle).to include_gems "foo 1.0", "indirect_platform_specific 1.0", "platform_specific 1.0 RUBY"
expected_checksums = checksum_section do |c|
c.repo_gem gem_repo2, "indirect_platform_specific", "1.0"
c.repo_gem gem_repo2, "platform_specific", "1.0"
c.repo_gem gem_repo2, "platform_specific", "1.0", "java"
c.repo_gem gem_repo2, "platform_specific", "1.0", x64_mingw32
end
expect(lockfile).to eq <<~L
PATH
remote: .
@ -571,7 +566,10 @@ RSpec.describe "bundle install from an existing gemspec" do
CHECKSUMS
foo (1.0)
#{expected_checksums}
#{checksum_for_repo_gem gem_repo2, "indirect_platform_specific", "1.0"}
#{checksum_for_repo_gem gem_repo2, "platform_specific", "1.0"}
#{checksum_for_repo_gem gem_repo2, "platform_specific", "1.0", "java"}
#{x64_mingw_checksums}
BUNDLED WITH
#{Bundler::VERSION}

195
spec/bundler/install/gemfile/sources_spec.rb
View File

@ -27,27 +27,55 @@ RSpec.describe "bundle install with gems on multiple sources" do
G
end
it "warns about ambiguous gems, but installs anyway, prioritizing sources last to first", :bundler => "< 3" do
bundle :install, :artifice => "compact_index"
expect(err).to include("Warning: the gem 'rack' was found in multiple sources.")
expect(err).to include("Installed from: https://gem.repo1")
expect(the_bundle).to include_gems("rack-obama 1.0.0", "rack 1.0.0", :source => "remote1")
end
it "does not use the full index unnecessarily", :bundler => "< 3" do
bundle :install, :artifice => "compact_index", :verbose => true
expect(out).to include("https://gem.repo1/versions")
expect(out).to include("https://gem.repo3/versions")
expect(out).not_to include("https://gem.repo1/quick/Marshal.4.8/")
expect(out).not_to include("https://gem.repo3/quick/Marshal.4.8/")
end
it "fails", :bundler => "3" do
it "refuses to install mismatched checksum because one gem has been tampered with", :bundler => "< 3" do
bundle :install, :artifice => "compact_index", :raise_on_error => false
expect(err).to include("Each source after the first must include a block")
expect(exitstatus).to eq(4)
expect(exitstatus).to eq(37)
expect(err).to eq <<~E.strip
[DEPRECATED] Your Gemfile contains multiple global sources. Using `source` more than once without a block is a security risk, and may result in installing unexpected gems. To resolve this warning, use a block to indicate which gems should come from the secondary source.
Bundler found mismatched checksums. This is a potential security risk.
#{checksum_for_repo_gem(gem_repo1, "rack", "1.0.0")}
from the API at https://gem.repo1/
#{checksum_for_repo_gem(gem_repo3, "rack", "1.0.0")}
from the API at https://gem.repo3/
Mismatched checksums each have an authoritative source:
1. the API at https://gem.repo1/
2. the API at https://gem.repo3/
You may need to alter your Gemfile sources to resolve this issue.
To ignore checksum security warnings, disable checksum validation with
`bundle config set --local disable_checksum_validation true`
E
end
context "when checksum validation is disabled" do
before do
bundle "config set --local disable_checksum_validation true"
end
it "warns about ambiguous gems, but installs anyway, prioritizing sources last to first", :bundler => "< 3" do
bundle :install, :artifice => "compact_index"
expect(err).to include("Warning: the gem 'rack' was found in multiple sources.")
expect(err).to include("Installed from: https://gem.repo1")
expect(the_bundle).to include_gems("rack-obama 1.0.0", "rack 1.0.0", :source => "remote1")
end
it "does not use the full index unnecessarily", :bundler => "< 3" do
bundle :install, :artifice => "compact_index", :verbose => true
expect(out).to include("https://gem.repo1/versions")
expect(out).to include("https://gem.repo3/versions")
expect(out).not_to include("https://gem.repo1/quick/Marshal.4.8/")
expect(out).not_to include("https://gem.repo3/quick/Marshal.4.8/")
end
it "fails", :bundler => "3" do
bundle :install, :artifice => "compact_index", :raise_on_error => false
expect(err).to include("Each source after the first must include a block")
expect(exitstatus).to eq(4)
end
end
end
@ -101,7 +129,8 @@ RSpec.describe "bundle install with gems on multiple sources" do
end
it "works in standalone mode", :bundler => "< 3" do
bundle "install --standalone", :artifice => "compact_index"
gem_checksum = checksum_for_repo_gem(gem_repo4, "foo", "1.0").split("-").last
bundle "install --standalone", :artifice => "compact_index", :env => { "BUNDLER_SPEC_FOO_CHECKSUM" => gem_checksum }
end
end
@ -279,8 +308,55 @@ RSpec.describe "bundle install with gems on multiple sources" do
G
end
it "installs from the other source and warns about ambiguous gems", :bundler => "< 3" do
bundle :install, :artifice => "compact_index"
it "fails when the two sources don't have the same checksum", :bundler => "< 3" do
bundle :install, :artifice => "compact_index", :raise_on_error => false
expect(err).to eq(<<~E.strip)
[DEPRECATED] Your Gemfile contains multiple global sources. Using `source` more than once without a block is a security risk, and may result in installing unexpected gems. To resolve this warning, use a block to indicate which gems should come from the secondary source.
Bundler found mismatched checksums. This is a potential security risk.
#{checksum_for_repo_gem(gem_repo2, "rack", "1.0.0")}
from the API at https://gem.repo2/
#{checksum_for_repo_gem(gem_repo1, "rack", "1.0.0")}
from the API at https://gem.repo1/
Mismatched checksums each have an authoritative source:
1. the API at https://gem.repo2/
2. the API at https://gem.repo1/
You may need to alter your Gemfile sources to resolve this issue.
To ignore checksum security warnings, disable checksum validation with
`bundle config set --local disable_checksum_validation true`
E
expect(exitstatus).to eq(37)
end
it "fails when the two sources agree, but the local gem calculates a different checksum", :bundler => "< 3" do
rack_checksum = "c0ffee11" * 8
bundle :install, :artifice => "compact_index", :env => { "BUNDLER_SPEC_RACK_CHECKSUM" => rack_checksum }, :raise_on_error => false
expect(err).to eq(<<~E.strip)
[DEPRECATED] Your Gemfile contains multiple global sources. Using `source` more than once without a block is a security risk, and may result in installing unexpected gems. To resolve this warning, use a block to indicate which gems should come from the secondary source.
Bundler found mismatched checksums. This is a potential security risk.
rack (1.0.0) sha256-#{rack_checksum}
from the API at https://gem.repo2/
and the API at https://gem.repo1/
#{checksum_for_repo_gem(gem_repo2, "rack", "1.0.0")}
from the gem at #{default_bundle_path("cache", "rack-1.0.0.gem")}
If you trust the API at https://gem.repo2/, to resolve this issue you can:
1. remove the gem at #{default_bundle_path("cache", "rack-1.0.0.gem")}
2. run `bundle install`
To ignore checksum security warnings, disable checksum validation with
`bundle config set --local disable_checksum_validation true`
E
expect(exitstatus).to eq(37)
end
it "installs from the other source and warns about ambiguous gems when the sources have the same checksum", :bundler => "< 3" do
gem_checksum = checksum_for_repo_gem(gem_repo2, "rack", "1.0.0").split("-").last
bundle :install, :artifice => "compact_index", :env => { "BUNDLER_SPEC_RACK_CHECKSUM" => gem_checksum, "DEBUG" => "1" }
expect(err).to include("Warning: the gem 'rack' was found in multiple sources.")
expect(err).to include("Installed from: https://gem.repo2")
@ -320,6 +396,49 @@ RSpec.describe "bundle install with gems on multiple sources" do
expect(lockfile).to eq(previous_lockfile)
end
it "installs from the other source and warns about ambiguous gems when checksum validation is disabled", :bundler => "< 3" do
bundle "config set --local disable_checksum_validation true"
bundle :install, :artifice => "compact_index"
expect(err).to include("Warning: the gem 'rack' was found in multiple sources.")
expect(err).to include("Installed from: https://gem.repo2")
expected_checksums = checksum_section do |c|
c.no_checksum "depends_on_rack", "1.0.1"
c.no_checksum "rack", "1.0.0"
end
expect(lockfile).to eq <<~L
GEM
remote: https://gem.repo1/
remote: https://gem.repo2/
specs:
rack (1.0.0)
GEM
remote: https://gem.repo3/
specs:
depends_on_rack (1.0.1)
rack
PLATFORMS
#{local_platform}
DEPENDENCIES
depends_on_rack!
CHECKSUMS
#{expected_checksums}
BUNDLED WITH
#{Bundler::VERSION}
L
previous_lockfile = lockfile
expect(the_bundle).to include_gems("depends_on_rack 1.0.1", "rack 1.0.0")
expect(lockfile).to eq(previous_lockfile)
end
it "fails", :bundler => "3" do
bundle :install, :artifice => "compact_index", :raise_on_error => false
expect(err).to include("Each source after the first must include a block")
@ -1167,8 +1286,9 @@ RSpec.describe "bundle install with gems on multiple sources" do
lockfile aggregate_gem_section_lockfile
end
it "installs the existing lockfile but prints a warning", :bundler => "< 3" do
it "installs the existing lockfile but prints a warning when checksum validation is disabled", :bundler => "< 3" do
bundle "config set --local deployment true"
bundle "config set --local disable_checksum_validation true"
bundle "install", :artifice => "compact_index"
@ -1177,6 +1297,33 @@ RSpec.describe "bundle install with gems on multiple sources" do
expect(the_bundle).to include_gems("rack 0.9.1", :source => "remote3")
end
it "prints a checksum warning when the checksums from both sources do not match", :bundler => "< 3" do
bundle "config set --local deployment true"
bundle "install", :artifice => "compact_index", :raise_on_error => false
api_checksum1 = checksum_for_repo_gem(gem_repo1, "rack", "0.9.1").split("sha256-").last
api_checksum3 = checksum_for_repo_gem(gem_repo3, "rack", "0.9.1").split("sha256-").last
expect(exitstatus).to eq(37)
expect(err).to eq(<<~E.strip)
[DEPRECATED] Your lockfile contains a single rubygems source section with multiple remotes, which is insecure. Make sure you run `bundle install` in non frozen mode and commit the result to make your lockfile secure.
Bundler found mismatched checksums. This is a potential security risk.
rack (0.9.1) sha256-#{api_checksum3}
from the API at https://gem.repo3/
rack (0.9.1) sha256-#{api_checksum1}
from the API at https://gem.repo1/
Mismatched checksums each have an authoritative source:
1. the API at https://gem.repo3/
2. the API at https://gem.repo1/
You may need to alter your Gemfile sources to resolve this issue.
To ignore checksum security warnings, disable checksum validation with
`bundle config set --local disable_checksum_validation true`
E
end
it "refuses to install the existing lockfile and prints an error", :bundler => "3" do
bundle "config set --local deployment true"

8
spec/bundler/install/gemfile/specific_platform_spec.rb
View File

@ -759,6 +759,11 @@ RSpec.describe "bundle install with specific platforms" do
bundle "update"
expected_checksums = checksum_section do |c|
c.repo_gem gem_repo4, "nokogiri", "1.14.0", "x86_64-linux"
c.repo_gem gem_repo4, "sorbet-static", "0.5.10696", "x86_64-linux"
end
expect(lockfile).to eq <<~L
GEM
remote: #{file_uri_for(gem_repo4)}/
@ -773,6 +778,9 @@ RSpec.describe "bundle install with specific platforms" do
nokogiri
sorbet-static
CHECKSUMS
#{expected_checksums}
BUNDLED WITH
#{Bundler::VERSION}
L

30
spec/bundler/install/gems/compact_index_spec.rb
View File

@ -890,25 +890,21 @@ The checksum of /versions does not match the checksum provided by the server! So
default_cache_path.dirname.join("rack-1.0.0.gem")
end
expect(exitstatus).to eq(19)
expect(err).
to eq <<~E.strip
Bundler cannot continue installing rack (1.0.0).
The checksum for the downloaded `rack-1.0.0.gem` does not match the known checksum for the gem.
This means the contents of the downloaded gem is different from what was uploaded to the server or first used by your teammates, and could be a potential security issue.
expect(exitstatus).to eq(37)
expect(err).to eq <<~E.strip
Bundler found mismatched checksums. This is a potential security risk.
rack (1.0.0) sha256-2222222222222222222222222222222222222222222222222222222222222222
from the API at http://localgemserver.test/
rack (1.0.0) sha256-#{api_checksum}
from the gem at #{gem_path}
To resolve this issue:
1. delete the downloaded gem located at: `#{gem_path}`
If you trust the API at http://localgemserver.test/, to resolve this issue you can:
1. remove the gem at #{gem_path}
2. run `bundle install`
If you are sure that the new checksum is correct, you can remove the `rack (1.0.0)` entry under the lockfile `CHECKSUMS` section and rerun `bundle install`.
If you wish to continue installing the downloaded gem, and are certain it does not pose a security issue despite the mismatching checksum, do the following:
1. run `bundle config set --local disable_checksum_validation true` to turn off checksum verification
2. run `bundle install`
(More info: The expected SHA256 checksum was "69b69b69b69b69b69b69b69b69b69b69b69b69b69b69b69b69b69b69b69b69b69b", but the checksum for the downloaded gem was "#{api_checksum}". The expected checksum came from: API response from http://localgemserver.test/)
E
To ignore checksum security warnings, disable checksum validation with
`bundle config set --local disable_checksum_validation true`
E
end
it "raises when the checksum is the wrong length" do
@ -917,7 +913,7 @@ The checksum of /versions does not match the checksum provided by the server! So
gem "rack"
G
expect(exitstatus).to eq(14)
expect(err).to include("The given checksum for rack-0.9.1 (\"checksum!\") is not a valid SHA256 hexdigest nor base64digest")
expect(err).to include("Invalid checksum for rack-0.9.1: \"checksum!\" is not a valid SHA256 hexdigest nor base64digest")
end
it "does not raise when disable_checksum_validation is set" do

2
spec/bundler/lock/lockfile_spec.rb
View File

@ -1769,7 +1769,7 @@ RSpec.describe "the lockfile format" do
gem "rack"
G
expect(err).to match(/your lockfile contains merge conflicts/i)
expect(err).to match(/your Gemfile.lock contains merge conflicts/i)
expect(err).to match(/git checkout HEAD -- Gemfile.lock/i)
end

3
spec/bundler/support/artifice/compact_index_wrong_gem_checksum.rb
View File

@ -7,7 +7,8 @@ class CompactIndexWrongGemChecksum < CompactIndexAPI
etag_response do
name = params[:name]
gem = gems.find {|g| g.name == name }
checksum = ENV.fetch("BUNDLER_SPEC_#{name.upcase}_CHECKSUM") { "ab" * 22 }
# This generates the hexdigest "2222222222222222222222222222222222222222222222222222222222222222"
checksum = ENV.fetch("BUNDLER_SPEC_#{name.upcase}_CHECKSUM") { "IiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiI=" }
versions = gem ? gem.versions : []
versions.each {|v| v.checksum = checksum }
CompactIndex.info(versions)

12
spec/bundler/support/artifice/helpers/compact_index.rb
View File

@ -79,11 +79,13 @@ class CompactIndexAPI < Endpoint
reqs = d.requirement.requirements.map {|r| r.join(" ") }.join(", ")
CompactIndex::Dependency.new(d.name, reqs)
end
checksum = begin
Digest(:SHA256).file("#{gem_repo}/gems/#{spec.original_name}.gem").hexdigest
rescue StandardError
nil
end
begin
checksum = ENV.fetch("BUNDLER_SPEC_#{name.upcase}_CHECKSUM") do
Digest(:SHA256).file("#{gem_repo}/gems/#{spec.original_name}.gem").hexdigest
end
rescue StandardError
checksum = nil
end
CompactIndex::GemVersion.new(spec.version.version, spec.platform.to_s, checksum, nil,
deps, spec.required_ruby_version.to_s, spec.required_rubygems_version.to_s)
end

20
spec/bundler/support/checksums.rb
View File

@ -9,26 +9,22 @@ module Spec
end
def repo_gem(repo, name, version, platform = Gem::Platform::RUBY)
gem_file = File.join(repo, "gems", "#{Bundler::GemHelpers.spec_full_name(name, version, platform)}.gem")
name_tuple = Gem::NameTuple.new(name, version, platform)
gem_file = File.join(repo, "gems", "#{name_tuple.full_name}.gem")
File.open(gem_file, "rb") do |f|
checksums = Bundler::Checksum.from_io(f, "ChecksumsBuilder")
checksum_entry(checksums, name, version, platform)
@checksums[name_tuple] = Bundler::Checksum.from_gem(f, "#{gem_file} (via ChecksumsBuilder#repo_gem)")
end
end
def no_checksum(name, version, platform = Gem::Platform::RUBY)
checksum_entry(nil, name, version, platform)
end
def checksum_entry(checksums, name, version, platform = Gem::Platform::RUBY)
lock_name = Bundler::GemHelpers.lock_name(name, version, platform)
@checksums[lock_name] = checksums
name_tuple = Gem::NameTuple.new(name, version, platform)
@checksums[name_tuple] = nil
end
def to_lock
@checksums.map do |lock_name, checksums|
checksums &&= " #{checksums.map(&:to_lock).join(",")}"
" #{lock_name}#{checksums}\n"
@checksums.map do |name_tuple, checksum|
checksum &&= " #{checksum.to_lock}"
" #{name_tuple.lock_name}#{checksum}\n"
end.sort.join.strip
end
end