[rubygems/rubygems] Handle base64 encoded checksums in lockfile for future compatibility.

Save checksums using = as separator.

https://github.com/rubygems/rubygems/commit/a36ad7d160

lib/bundler/checksum.rb

@@ -2,6 +2,7 @@
 
 module Bundler
   class Checksum
+    ALGO_SEPARATOR = "="
     DEFAULT_ALGORITHM = "sha256"
     private_constant :DEFAULT_ALGORITHM
     DEFAULT_BLOCK_SIZE = 16_384
@@ -15,20 +16,24 @@ module Bundler
         Checksum.new(algo, digest.hexdigest!, Source.new(:gem, pathname))
       end
 
-      def from_api(digest, source_uri)
+      def from_api(digest, source_uri, algo = DEFAULT_ALGORITHM)
-        # transform the bytes from base64 to hex, switch to unpack1 when we drop older rubies
+        Checksum.new(algo, to_hexdigest(digest, algo), Source.new(:api, source_uri))
-        hexdigest = digest.length == 44 ? digest.unpack("m0").first.unpack("H*").first : digest
-
-        if hexdigest.length != 64
-          raise ArgumentError, "#{digest.inspect} is not a valid SHA256 hexdigest nor base64digest"
-        end
-
-        Checksum.new(DEFAULT_ALGORITHM, hexdigest, Source.new(:api, source_uri))
       end
 
       def from_lock(lock_checksum, lockfile_location)
-        algo, digest = lock_checksum.strip.split("-", 2)
+        algo, digest = lock_checksum.strip.split(ALGO_SEPARATOR, 2)
-        Checksum.new(algo, digest, Source.new(:lock, lockfile_location))
+        Checksum.new(algo, to_hexdigest(digest, algo), Source.new(:lock, lockfile_location))
+      end
+
+      def to_hexdigest(digest, algo = DEFAULT_ALGORITHM)
+        return digest unless algo == DEFAULT_ALGORITHM
+        return digest if digest.match?(/\A[0-9a-f]{64}\z/i)
+        if digest.match?(%r{\A[-0-9a-z_+/]{43}={0,2}\z}i)
+          digest = digest.tr("-_", "+/") # fix urlsafe base64
+          # transform to hex. Use unpack1 when we drop older rubies
+          return digest.unpack("m0").first.unpack("H*").first
+        end
+        raise ArgumentError, "#{digest.inspect} is not a valid SHA256 hex or base64 digest"
       end
     end
 
@@ -59,7 +64,7 @@ module Bundler
     end
 
     def to_lock
-      "#{algo}-#{digest}"
+      "#{algo}#{ALGO_SEPARATOR}#{digest}"
     end
 
     def merge!(other)
@@ -87,7 +92,7 @@ module Bundler
     end
 
     def inspect
-      abbr = "#{algo}-#{digest[0, 8]}"
+      abbr = "#{algo}#{ALGO_SEPARATOR}#{digest[0, 8]}"
       from = "from #{sources.join(" and ")}"
       "#<#{self.class}:#{object_id} #{abbr} #{from}>"
     end
@@ -109,7 +114,7 @@ module Bundler
       end
 
       # phrased so that the usual string format is grammatically correct
-      #   rake (10.3.2) sha256-abc123 from #{to_s}
+      #   rake (10.3.2) sha256=abc123 from #{to_s}
       def to_s
         case type
         when :lock

spec/bundler/bundler/lockfile_parser_spec.rb

@@ -23,7 +23,7 @@ RSpec.describe Bundler::LockfileParser do
       rake
 
     CHECKSUMS
-      rake (10.3.2) sha256-814828c34f1315d7e7b7e8295184577cc4e969bad6156ac069d02d63f58d82e8
+      rake (10.3.2) sha256=814828c34f1315d7e7b7e8295184577cc4e969bad6156ac069d02d63f58d82e8
 
     RUBY VERSION
        ruby 2.1.3p242
@@ -121,8 +121,8 @@ RSpec.describe Bundler::LockfileParser do
     let(:lockfile_path) { Bundler.default_lockfile.relative_path_from(Dir.pwd) }
     let(:rake_checksum) do
       Bundler::Checksum.from_lock(
-        "sha256-814828c34f1315d7e7b7e8295184577cc4e969bad6156ac069d02d63f58d82e8",
+        "sha256=814828c34f1315d7e7b7e8295184577cc4e969bad6156ac069d02d63f58d82e8",
-        "#{lockfile_path}:??:1"
+        "#{lockfile_path}:20:17"
       )
     end
 
@@ -163,8 +163,33 @@ RSpec.describe Bundler::LockfileParser do
       include_examples "parsing"
     end
 
+    context "when the checksum is urlsafe base64 encoded" do
+      let(:lockfile_contents) do
+        super().sub(
+          "sha256=814828c34f1315d7e7b7e8295184577cc4e969bad6156ac069d02d63f58d82e8",
+          "sha256=gUgow08TFdfnt-gpUYRXfMTpabrWFWrAadAtY_WNgug="
+        )
+      end
+      include_examples "parsing"
+    end
+
+    context "when the checksum is of an unknown algorithm" do
+      let(:lockfile_contents) do
+        super().sub(
+          "sha256=",
+          "sha512=pVDn9GLmcFkz8vj1ueiVxj5uGKkAyaqYjEX8zG6L5O4BeVg3wANaKbQdpj/B82Nd/MHVszy6polHcyotUdwilQ==,sha256="
+        )
+      end
+      include_examples "parsing"
+
+      it "preserves the checksum as is" do
+        checksum = subject.sources.last.checksum_store.fetch(specs.last, "sha512")
+        expect(checksum.algo).to eq("sha512")
+      end
+    end
+
     context "when CHECKSUMS has duplicate checksums in the lockfile that don't match" do
-      let(:bad_checksum) { "sha256-c0ffee11c0ffee11c0ffee11c0ffee11c0ffee11c0ffee11c0ffee11c0ffee11" }
+      let(:bad_checksum) { "sha256=c0ffee11c0ffee11c0ffee11c0ffee11c0ffee11c0ffee11c0ffee11c0ffee11" }
       let(:lockfile_contents) { super().split(/(?<=CHECKSUMS\n)/m).insert(1, "  rake (10.3.2) #{bad_checksum}\n").join }
 
       it "raises a security error" do

spec/bundler/commands/lock_spec.rb

@@ -214,7 +214,7 @@ RSpec.describe "bundle lock" do
   end
 
   it "preserves unknown checksum algorithms" do
-    lockfile @lockfile.gsub(/(sha256-[a-f0-9]+)$/, "constant-true,\\1,xyz-123")
+    lockfile @lockfile.gsub(/(sha256=[a-f0-9]+)$/, "constant=true,\\1,xyz=123")
 
     previous_lockfile = read_lockfile
 

spec/bundler/install/gemfile/sources_spec.rb

@@ -129,7 +129,7 @@ RSpec.describe "bundle install with gems on multiple sources" do
     end
 
     it "works in standalone mode", :bundler => "< 3" do
-      gem_checksum = checksum_for_repo_gem(gem_repo4, "foo", "1.0").split("-").last
+      gem_checksum = checksum_for_repo_gem(gem_repo4, "foo", "1.0").split(Bundler::Checksum::ALGO_SEPARATOR).last
       bundle "install --standalone", :artifice => "compact_index", :env => { "BUNDLER_SPEC_FOO_CHECKSUM" => gem_checksum }
     end
   end
@@ -337,7 +337,7 @@ RSpec.describe "bundle install with gems on multiple sources" do
           expect(err).to eq(<<~E.strip)
             [DEPRECATED] Your Gemfile contains multiple global sources. Using `source` more than once without a block is a security risk, and may result in installing unexpected gems. To resolve this warning, use a block to indicate which gems should come from the secondary source.
             Bundler found mismatched checksums. This is a potential security risk.
-              rack (1.0.0) sha256-#{rack_checksum}
+              rack (1.0.0) sha256=#{rack_checksum}
                 from the API at https://gem.repo2/
                 and the API at https://gem.repo1/
               #{checksum_for_repo_gem(gem_repo2, "rack", "1.0.0")}
@@ -354,7 +354,7 @@ RSpec.describe "bundle install with gems on multiple sources" do
         end
 
         it "installs from the other source and warns about ambiguous gems when the sources have the same checksum", :bundler => "< 3" do
-          gem_checksum = checksum_for_repo_gem(gem_repo2, "rack", "1.0.0").split("-").last
+          gem_checksum = checksum_for_repo_gem(gem_repo2, "rack", "1.0.0").split(Bundler::Checksum::ALGO_SEPARATOR).last
           bundle :install, :artifice => "compact_index", :env => { "BUNDLER_SPEC_RACK_CHECKSUM" => gem_checksum, "DEBUG" => "1" }
 
           expect(err).to include("Warning: the gem 'rack' was found in multiple sources.")
@@ -1302,16 +1302,16 @@ RSpec.describe "bundle install with gems on multiple sources" do
 
         bundle "install", :artifice => "compact_index", :raise_on_error => false
 
-        api_checksum1 = checksum_for_repo_gem(gem_repo1, "rack", "0.9.1").split("sha256-").last
+        api_checksum1 = checksum_for_repo_gem(gem_repo1, "rack", "0.9.1").split("sha256=").last
-        api_checksum3 = checksum_for_repo_gem(gem_repo3, "rack", "0.9.1").split("sha256-").last
+        api_checksum3 = checksum_for_repo_gem(gem_repo3, "rack", "0.9.1").split("sha256=").last
 
         expect(exitstatus).to eq(37)
         expect(err).to eq(<<~E.strip)
           [DEPRECATED] Your lockfile contains a single rubygems source section with multiple remotes, which is insecure. Make sure you run `bundle install` in non frozen mode and commit the result to make your lockfile secure.
           Bundler found mismatched checksums. This is a potential security risk.
-            rack (0.9.1) sha256-#{api_checksum3}
+            rack (0.9.1) sha256=#{api_checksum3}
               from the API at https://gem.repo3/
-            rack (0.9.1) sha256-#{api_checksum1}
+            rack (0.9.1) sha256=#{api_checksum1}
               from the API at https://gem.repo1/
 
           Mismatched checksums each have an authoritative source:

spec/bundler/install/gems/compact_index_spec.rb

@@ -876,13 +876,25 @@ The checksum of /versions does not match the checksum provided by the server! So
   end
 
   describe "checksum validation" do
+    it "handles checksums from the server in base64" do
+      api_checksum = checksum_for_repo_gem(gem_repo1, "rack", "1.0.0").split("sha256=").last
+      rack_checksum = [[api_checksum].pack("H*")].pack("m0")
+      install_gemfile <<-G, :artifice => "compact_index", :env => { "BUNDLER_SPEC_RACK_CHECKSUM" => rack_checksum }
+        source "#{source_uri}"
+        gem "rack"
+      G
+
+      expect(out).to include("Fetching gem metadata from #{source_uri}")
+      expect(the_bundle).to include_gems("rack 1.0.0")
+    end
+
     it "raises when the checksum does not match" do
       install_gemfile <<-G, :artifice => "compact_index_wrong_gem_checksum", :raise_on_error => false
         source "#{source_uri}"
         gem "rack"
       G
 
-      api_checksum = checksum_for_repo_gem(gem_repo1, "rack", "1.0.0").split("sha256-").last
+      api_checksum = checksum_for_repo_gem(gem_repo1, "rack", "1.0.0").split("sha256=").last
 
       gem_path = if Bundler.feature_flag.global_gem_cache?
         default_cache_path.dirname.join("cache", "gems", "localgemserver.test.80.dd34752a738ee965a2a4298dc16db6c5", "rack-1.0.0.gem")
@@ -893,9 +905,9 @@ The checksum of /versions does not match the checksum provided by the server! So
       expect(exitstatus).to eq(37)
       expect(err).to eq <<~E.strip
         Bundler found mismatched checksums. This is a potential security risk.
-          rack (1.0.0) sha256-2222222222222222222222222222222222222222222222222222222222222222
+          rack (1.0.0) sha256=2222222222222222222222222222222222222222222222222222222222222222
             from the API at http://localgemserver.test/
-          rack (1.0.0) sha256-#{api_checksum}
+          rack (1.0.0) sha256=#{api_checksum}
             from the gem at #{gem_path}
 
         If you trust the API at http://localgemserver.test/, to resolve this issue you can:
@@ -913,7 +925,7 @@ The checksum of /versions does not match the checksum provided by the server! So
         gem "rack"
       G
       expect(exitstatus).to eq(14)
-      expect(err).to include("Invalid checksum for rack-0.9.1: \"checksum!\" is not a valid SHA256 hexdigest nor base64digest")
+      expect(err).to include('Invalid checksum for rack-0.9.1: "checksum!" is not a valid SHA256 hex or base64 digest')
     end
 
     it "does not raise when disable_checksum_validation is set" do

spec/bundler/support/checksums.rb

@@ -61,7 +61,7 @@ module Spec
 
       checksums = checksums.each_line.map do |line|
         if prefixes.nil? || line.match?(prefixes)
-          line.gsub(/ sha256-[a-f0-9]{64}/i, "")
+          line.gsub(/ sha256=[a-f0-9]{64}/i, "")
         else
           line
         end