[rubygems/rubygems] Upgrading S3 source signature to AWS SigV4

https://github.com/rubygems/rubygems/commit/f289788ca5

lib/rubygems/remote_fetcher.rb

@@ -343,31 +343,71 @@ class Gem::RemoteFetcher
 
   protected
 
+  S3Config = Struct.new :access_key_id, :secret_access_key, :security_token, :region
+
   # we have our own signing code here to avoid a dependency on the aws-sdk gem
   # fortunately, a simple GET request isn't too complex to sign properly
   def sign_s3_url(uri, expiration = nil)
     require 'base64'
+    require 'digest'
     require 'openssl'
 
-    id, secret = s3_source_auth uri
+    s3_config = s3_source_auth uri
+    expiration ||= 3600
 
-    expiration ||= s3_expiration
+    current_time = Time.now.utc
-    canonical_path = "/#{uri.host}#{uri.path}"
+    date_time = current_time.strftime("%Y%m%dT%H%m%SZ")
-    payload = "GET\n\n\n#{expiration}\n#{canonical_path}"
+    date = date_time[0,8]
-    digest = OpenSSL::HMAC.digest('sha1', secret, payload)
-    # URI.escape is deprecated, and there isn't yet a replacement that does quite what we want
-    signature = Base64.encode64(digest).gsub("\n", '').gsub(/[\+\/=]/) { |c| BASE64_URI_TRANSLATE[c] }
-    URI.parse("https://#{uri.host}.s3.amazonaws.com#{uri.path}?AWSAccessKeyId=#{id}&Expires=#{expiration}&Signature=#{signature}")
-  end
 
-  def s3_expiration
+    credential_info = "#{date}/#{s3_config.region}/s3/aws4_request"
-    (Time.now + 3600).to_i # one hour from now
+    canonical_host = "#{uri.host}.s3.#{s3_config.region}.amazonaws.com"
+
+    canonical_params = {}
+    canonical_params['X-Amz-Algorithm'] = 'AWS4-HMAC-SHA256'
+    canonical_params['X-Amz-Credential'] = "#{s3_config.access_key_id}/#{credential_info}"
+    canonical_params['X-Amz-Date'] = date_time
+    canonical_params['X-Amz-Expires'] = expiration.to_s
+    canonical_params['X-Amz-SignedHeaders'] = 'host'
+    canonical_params['X-Amz-Security-Token'] = s3_config.security_token if s3_config.security_token
+
+    query_params = canonical_params.sort.to_h.map do |key, value|
+      "#{base64_uri_escape(key)}=#{base64_uri_escape(value)}"
+    end.join('&')
+
+    canonical_request = [
+      'GET',
+      uri.path,
+      query_params,
+      "host:#{canonical_host}",
+      '', # empty params
+      'host',
+      'UNSIGNED-PAYLOAD',
+    ].join("\n")
+
+    string_to_sign = [
+      "AWS4-HMAC-SHA256",
+      date_time,
+      credential_info,
+      Digest::SHA256.hexdigest(canonical_request)
+    ].join("\n")
+
+    date_key = OpenSSL::HMAC.digest('sha256', "AWS4" + s3_config.secret_access_key, date)
+    date_region_key = OpenSSL::HMAC.digest('sha256', date_key, s3_config.region)
+    date_region_service_key = OpenSSL::HMAC.digest('sha256', date_region_key, "s3")
+    signing_key = OpenSSL::HMAC.digest('sha256', date_region_service_key, "aws4_request")
+    signature = OpenSSL::HMAC.hexdigest('sha256', signing_key, string_to_sign)
+
+    URI.parse("https://#{canonical_host}#{uri.path}?#{query_params}&X-Amz-Signature=#{signature}")
   end
 
   BASE64_URI_TRANSLATE = { '+' => '%2B', '/' => '%2F', '=' => '%3D' }.freeze
 
   private
 
+  def base64_uri_escape(str)
+    str.gsub("\n", '').gsub(/[\+\/=]/) { |c| BASE64_URI_TRANSLATE[c] }
+  end
+
   def proxy_for(proxy, uri)
     Gem::Request.proxy_uri(proxy || Gem::Request.get_proxy_from_env(uri.scheme))
   end
@@ -379,7 +419,7 @@ class Gem::RemoteFetcher
   end
 
   def s3_source_auth(uri)
-    return [uri.user, uri.password] if uri.user && uri.password
+    return S3Config.new(uri.user, uri.password, nil, 'us-east-1') if uri.user && uri.password
 
     s3_source = Gem.configuration[:s3_source] || Gem.configuration['s3_source']
     host = uri.host
@@ -388,11 +428,31 @@ class Gem::RemoteFetcher
     auth = s3_source[host] || s3_source[host.to_sym]
     raise FetchError.new("no key for host #{host} in s3_source in .gemrc", "s3://#{host}") unless auth
 
-    id = auth[:id] || auth['id']
+    provider = auth[:provider] || auth['provider']
-    secret = auth[:secret] || auth['secret']
+    case provider
-    raise FetchError.new("s3_source for #{host} missing id or secret", "s3://#{host}") unless id and secret
+    when 'env'
+      id = ENV['AWS_ACCESS_KEY_ID']
+      secret = ENV['AWS_SECRET_ACCESS_KEY']
+      security_token = ENV['AWS_SESSION_TOKEN']
+    when 'instance_profile'
+      require 'json'
+      credentials_response = fetch_http URI(EC2_METADATA_CREDENTIALS)
+      credentials = JSON.parse(credentials_response)
+      id = credentials['AccessKeyId']
+      secret = credentials['SecretAccessKey']
+      security_token = credentials['Token']
+    else
+      id = auth[:id] || auth['id']
+      secret = auth[:secret] || auth['secret']
+      raise FetchError.new("s3_source for #{host} missing id or secret", "s3://#{host}") unless id and secret
 
-    [id, secret]
+      security_token = auth[:security_token] || auth['security_token']
+    end
+
+    region = auth[:region] || auth['region'] || 'us-east-1'
+    S3Config.new(id, secret, security_token, region)
   end
 
+  EC2_METADATA_CREDENTIALS = "http://169.254.169.254/latest/meta-data/identity-credentials/ec2/security-credentials/ec2-instance"
+
 end

test/rubygems/test_gem_remote_fetcher.rb

@@ -650,25 +650,27 @@ PeIQQkFng2VVot/WAQbv3ePqWq07g1BBcwIBAg==
     assert_equal "murphy", fetcher.fetch_path(@server_uri)
   end
 
-  def assert_fetch_s3(url)
+  def assert_fetch_s3(url, signature, token=nil, region='us-east-1', instance_profile_json=nil)
     fetcher = Gem::RemoteFetcher.new nil
     @fetcher = fetcher
     $fetched_uri = nil
+    $instance_profile = instance_profile_json
 
     def fetcher.request(uri, request_class, last_modified = nil)
       $fetched_uri = uri
       res = Net::HTTPOK.new nil, 200, nil
-      def res.body() 'success' end
+      case uri.to_s
+      when /^http:\/\/169\.254\.169\.254.*/
+        def res.body() $instance_profile end
+      else
+        def res.body() 'success' end
+      end
       res
     end
 
-    def fetcher.s3_expiration
-      1395098371
-    end
-
     data = fetcher.fetch_s3 URI.parse(url)
 
-    assert_equal 'https://my-bucket.s3.amazonaws.com/gems/specs.4.8.gz?AWSAccessKeyId=testuser&Expires=1395098371&Signature=eUTr7NkpZEet%2BJySE%2BfH6qukroI%3D', $fetched_uri.to_s
+    assert_equal "https://my-bucket.s3.#{region}.amazonaws.com/gems/specs.4.8.gz?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=testuser%2F20190624%2F#{region}%2Fs3%2Faws4_request&X-Amz-Date=20190624T050641Z&X-Amz-Expires=3600#{token ? "&X-Amz-Security-Token=" + token : ""}&X-Amz-SignedHeaders=host&X-Amz-Signature=#{signature}", $fetched_uri.to_s
     assert_equal 'success', data
   ensure
     $fetched_uri = nil
@@ -679,14 +681,132 @@ PeIQQkFng2VVot/WAQbv3ePqWq07g1BBcwIBAg==
       'my-bucket' => {:id => 'testuser', :secret => 'testpass'}
     }
     url = 's3://my-bucket/gems/specs.4.8.gz'
-    assert_fetch_s3 url
+    Time.stub :now, Time.at(1561353581) do
+      assert_fetch_s3 url, '8464fcf293454689ec1d3399463c9eeafe2a5b22f2cbd948cba28175a34b073d'
+    end
   ensure
     Gem.configuration[:s3_source] = nil
   end
 
+  def test_fetch_s3_config_creds_with_region
+    Gem.configuration[:s3_source] = {
+      'my-bucket' => {:id => 'testuser', :secret => 'testpass', :region => 'us-west-2'}
+    }
+    url = 's3://my-bucket/gems/specs.4.8.gz'
+    Time.stub :now, Time.at(1561353581) do
+      assert_fetch_s3 url, '800419b6efb5eacabef2c13c7e493f093df417afc10961f6dcc7e085c9c89e93', nil, 'us-west-2'
+    end
+  ensure
+    Gem.configuration[:s3_source] = nil
+  end
+
+  def test_fetch_s3_config_creds_with_token
+    Gem.configuration[:s3_source] = {
+      'my-bucket' => {:id => 'testuser', :secret => 'testpass', :security_token => 'testtoken'}
+    }
+    url = 's3://my-bucket/gems/specs.4.8.gz'
+    Time.stub :now, Time.at(1561353581) do
+      assert_fetch_s3 url, '9686a7b85d29b983f0ee723fd444536258ae1d3a7afa7f8599b3852aef3c8c11', 'testtoken'
+    end
+  ensure
+    Gem.configuration[:s3_source] = nil
+  end
+
+  def test_fetch_s3_env_creds
+    ENV['AWS_ACCESS_KEY_ID'] = 'testuser'
+    ENV['AWS_SECRET_ACCESS_KEY'] = 'testpass'
+    ENV['AWS_SESSION_TOKEN'] = nil
+    Gem.configuration[:s3_source] = {
+      'my-bucket' => {:provider => 'env'}
+    }
+    url = 's3://my-bucket/gems/specs.4.8.gz'
+    Time.stub :now, Time.at(1561353581) do
+      assert_fetch_s3 url, '8464fcf293454689ec1d3399463c9eeafe2a5b22f2cbd948cba28175a34b073d'
+    end
+  ensure
+    ENV.each_key {|key| ENV.delete(key) if key.start_with?('AWS')}
+    Gem.configuration[:s3_source] = nil
+  end
+
+  def test_fetch_s3_env_creds_with_region
+    ENV['AWS_ACCESS_KEY_ID'] = 'testuser'
+    ENV['AWS_SECRET_ACCESS_KEY'] = 'testpass'
+    ENV['AWS_SESSION_TOKEN'] = nil
+    Gem.configuration[:s3_source] = {
+      'my-bucket' => {:provider => 'env', :region => 'us-west-2'}
+    }
+    url = 's3://my-bucket/gems/specs.4.8.gz'
+    Time.stub :now, Time.at(1561353581) do
+      assert_fetch_s3 url, '800419b6efb5eacabef2c13c7e493f093df417afc10961f6dcc7e085c9c89e93', nil, 'us-west-2'
+    end
+  ensure
+    ENV.each_key {|key| ENV.delete(key) if key.start_with?('AWS')}
+    Gem.configuration[:s3_source] = nil
+  end
+
+  def test_fetch_s3_env_creds_with_token
+    ENV['AWS_ACCESS_KEY_ID'] = 'testuser'
+    ENV['AWS_SECRET_ACCESS_KEY'] = 'testpass'
+    ENV['AWS_SESSION_TOKEN'] = 'testtoken'
+    Gem.configuration[:s3_source] = {
+      'my-bucket' => {:provider => 'env'}
+    }
+    url = 's3://my-bucket/gems/specs.4.8.gz'
+    Time.stub :now, Time.at(1561353581) do
+      assert_fetch_s3 url, '9686a7b85d29b983f0ee723fd444536258ae1d3a7afa7f8599b3852aef3c8c11', 'testtoken'
+    end
+  ensure
+    ENV.each_key {|key| ENV.delete(key) if key.start_with?('AWS')}
+    Gem.configuration[:s3_source] = nil
+  end
+
   def test_fetch_s3_url_creds
     url = 's3://testuser:testpass@my-bucket/gems/specs.4.8.gz'
-    assert_fetch_s3 url
+    Time.stub :now, Time.at(1561353581) do
+      assert_fetch_s3 url, '8464fcf293454689ec1d3399463c9eeafe2a5b22f2cbd948cba28175a34b073d'
+    end
+  end
+
+  def test_fetch_s3_instance_profile_creds
+    Gem.configuration[:s3_source] = {
+      'my-bucket' => {:provider => 'instance_profile'}
+    }
+
+    url = 's3://my-bucket/gems/specs.4.8.gz'
+    Time.stub :now, Time.at(1561353581) do
+      assert_fetch_s3 url, '8464fcf293454689ec1d3399463c9eeafe2a5b22f2cbd948cba28175a34b073d', nil, 'us-east-1',
+                      '{"AccessKeyId": "testuser", "SecretAccessKey": "testpass"}'
+    end
+  ensure
+    Gem.configuration[:s3_source] = nil
+  end
+
+  def test_fetch_s3_instance_profile_creds_with_region
+    Gem.configuration[:s3_source] = {
+      'my-bucket' => {:provider => 'instance_profile', :region => 'us-west-2'}
+    }
+
+    url = 's3://my-bucket/gems/specs.4.8.gz'
+    Time.stub :now, Time.at(1561353581) do
+      assert_fetch_s3 url, '800419b6efb5eacabef2c13c7e493f093df417afc10961f6dcc7e085c9c89e93', nil, 'us-west-2',
+                      '{"AccessKeyId": "testuser", "SecretAccessKey": "testpass"}'
+    end
+  ensure
+    Gem.configuration[:s3_source] = nil
+  end
+
+  def test_fetch_s3_instance_profile_creds_with_token
+    Gem.configuration[:s3_source] = {
+      'my-bucket' => {:provider => 'instance_profile'}
+    }
+
+    url = 's3://my-bucket/gems/specs.4.8.gz'
+    Time.stub :now, Time.at(1561353581) do
+      assert_fetch_s3 url, '9686a7b85d29b983f0ee723fd444536258ae1d3a7afa7f8599b3852aef3c8c11', 'testtoken', 'us-east-1',
+                      '{"AccessKeyId": "testuser", "SecretAccessKey": "testpass", "Token": "testtoken"}'
+    end
+  ensure
+    Gem.configuration[:s3_source] = nil
   end
 
   def refute_fetch_s3(url, expected_message)