diff --git a/lib/rubygems/remote_fetcher.rb b/lib/rubygems/remote_fetcher.rb index a40ee55273..f0b997755e 100644 --- a/lib/rubygems/remote_fetcher.rb +++ b/lib/rubygems/remote_fetcher.rb @@ -343,31 +343,71 @@ class Gem::RemoteFetcher protected + S3Config = Struct.new :access_key_id, :secret_access_key, :security_token, :region + # we have our own signing code here to avoid a dependency on the aws-sdk gem # fortunately, a simple GET request isn't too complex to sign properly def sign_s3_url(uri, expiration = nil) require 'base64' + require 'digest' require 'openssl' - id, secret = s3_source_auth uri + s3_config = s3_source_auth uri + expiration ||= 3600 - expiration ||= s3_expiration - canonical_path = "/#{uri.host}#{uri.path}" - payload = "GET\n\n\n#{expiration}\n#{canonical_path}" - digest = OpenSSL::HMAC.digest('sha1', secret, payload) - # URI.escape is deprecated, and there isn't yet a replacement that does quite what we want - signature = Base64.encode64(digest).gsub("\n", '').gsub(/[\+\/=]/) { |c| BASE64_URI_TRANSLATE[c] } - URI.parse("https://#{uri.host}.s3.amazonaws.com#{uri.path}?AWSAccessKeyId=#{id}&Expires=#{expiration}&Signature=#{signature}") - end + current_time = Time.now.utc + date_time = current_time.strftime("%Y%m%dT%H%m%SZ") + date = date_time[0,8] - def s3_expiration - (Time.now + 3600).to_i # one hour from now + credential_info = "#{date}/#{s3_config.region}/s3/aws4_request" + canonical_host = "#{uri.host}.s3.#{s3_config.region}.amazonaws.com" + + canonical_params = {} + canonical_params['X-Amz-Algorithm'] = 'AWS4-HMAC-SHA256' + canonical_params['X-Amz-Credential'] = "#{s3_config.access_key_id}/#{credential_info}" + canonical_params['X-Amz-Date'] = date_time + canonical_params['X-Amz-Expires'] = expiration.to_s + canonical_params['X-Amz-SignedHeaders'] = 'host' + canonical_params['X-Amz-Security-Token'] = s3_config.security_token if s3_config.security_token + + query_params = canonical_params.sort.to_h.map do |key, value| + "#{base64_uri_escape(key)}=#{base64_uri_escape(value)}" + end.join('&') + + canonical_request = [ + 'GET', + uri.path, + query_params, + "host:#{canonical_host}", + '', # empty params + 'host', + 'UNSIGNED-PAYLOAD', + ].join("\n") + + string_to_sign = [ + "AWS4-HMAC-SHA256", + date_time, + credential_info, + Digest::SHA256.hexdigest(canonical_request) + ].join("\n") + + date_key = OpenSSL::HMAC.digest('sha256', "AWS4" + s3_config.secret_access_key, date) + date_region_key = OpenSSL::HMAC.digest('sha256', date_key, s3_config.region) + date_region_service_key = OpenSSL::HMAC.digest('sha256', date_region_key, "s3") + signing_key = OpenSSL::HMAC.digest('sha256', date_region_service_key, "aws4_request") + signature = OpenSSL::HMAC.hexdigest('sha256', signing_key, string_to_sign) + + URI.parse("https://#{canonical_host}#{uri.path}?#{query_params}&X-Amz-Signature=#{signature}") end BASE64_URI_TRANSLATE = { '+' => '%2B', '/' => '%2F', '=' => '%3D' }.freeze private + def base64_uri_escape(str) + str.gsub("\n", '').gsub(/[\+\/=]/) { |c| BASE64_URI_TRANSLATE[c] } + end + def proxy_for(proxy, uri) Gem::Request.proxy_uri(proxy || Gem::Request.get_proxy_from_env(uri.scheme)) end @@ -379,7 +419,7 @@ class Gem::RemoteFetcher end def s3_source_auth(uri) - return [uri.user, uri.password] if uri.user && uri.password + return S3Config.new(uri.user, uri.password, nil, 'us-east-1') if uri.user && uri.password s3_source = Gem.configuration[:s3_source] || Gem.configuration['s3_source'] host = uri.host @@ -388,11 +428,31 @@ class Gem::RemoteFetcher auth = s3_source[host] || s3_source[host.to_sym] raise FetchError.new("no key for host #{host} in s3_source in .gemrc", "s3://#{host}") unless auth - id = auth[:id] || auth['id'] - secret = auth[:secret] || auth['secret'] - raise FetchError.new("s3_source for #{host} missing id or secret", "s3://#{host}") unless id and secret + provider = auth[:provider] || auth['provider'] + case provider + when 'env' + id = ENV['AWS_ACCESS_KEY_ID'] + secret = ENV['AWS_SECRET_ACCESS_KEY'] + security_token = ENV['AWS_SESSION_TOKEN'] + when 'instance_profile' + require 'json' + credentials_response = fetch_http URI(EC2_METADATA_CREDENTIALS) + credentials = JSON.parse(credentials_response) + id = credentials['AccessKeyId'] + secret = credentials['SecretAccessKey'] + security_token = credentials['Token'] + else + id = auth[:id] || auth['id'] + secret = auth[:secret] || auth['secret'] + raise FetchError.new("s3_source for #{host} missing id or secret", "s3://#{host}") unless id and secret - [id, secret] + security_token = auth[:security_token] || auth['security_token'] + end + + region = auth[:region] || auth['region'] || 'us-east-1' + S3Config.new(id, secret, security_token, region) end + EC2_METADATA_CREDENTIALS = "http://169.254.169.254/latest/meta-data/identity-credentials/ec2/security-credentials/ec2-instance" + end diff --git a/test/rubygems/test_gem_remote_fetcher.rb b/test/rubygems/test_gem_remote_fetcher.rb index 7d5bcc23a9..2464a0d513 100644 --- a/test/rubygems/test_gem_remote_fetcher.rb +++ b/test/rubygems/test_gem_remote_fetcher.rb @@ -650,25 +650,27 @@ PeIQQkFng2VVot/WAQbv3ePqWq07g1BBcwIBAg== assert_equal "murphy", fetcher.fetch_path(@server_uri) end - def assert_fetch_s3(url) + def assert_fetch_s3(url, signature, token=nil, region='us-east-1', instance_profile_json=nil) fetcher = Gem::RemoteFetcher.new nil @fetcher = fetcher $fetched_uri = nil + $instance_profile = instance_profile_json def fetcher.request(uri, request_class, last_modified = nil) $fetched_uri = uri res = Net::HTTPOK.new nil, 200, nil - def res.body() 'success' end + case uri.to_s + when /^http:\/\/169\.254\.169\.254.*/ + def res.body() $instance_profile end + else + def res.body() 'success' end + end res end - def fetcher.s3_expiration - 1395098371 - end - data = fetcher.fetch_s3 URI.parse(url) - assert_equal 'https://my-bucket.s3.amazonaws.com/gems/specs.4.8.gz?AWSAccessKeyId=testuser&Expires=1395098371&Signature=eUTr7NkpZEet%2BJySE%2BfH6qukroI%3D', $fetched_uri.to_s + assert_equal "https://my-bucket.s3.#{region}.amazonaws.com/gems/specs.4.8.gz?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=testuser%2F20190624%2F#{region}%2Fs3%2Faws4_request&X-Amz-Date=20190624T050641Z&X-Amz-Expires=3600#{token ? "&X-Amz-Security-Token=" + token : ""}&X-Amz-SignedHeaders=host&X-Amz-Signature=#{signature}", $fetched_uri.to_s assert_equal 'success', data ensure $fetched_uri = nil @@ -679,14 +681,132 @@ PeIQQkFng2VVot/WAQbv3ePqWq07g1BBcwIBAg== 'my-bucket' => {:id => 'testuser', :secret => 'testpass'} } url = 's3://my-bucket/gems/specs.4.8.gz' - assert_fetch_s3 url + Time.stub :now, Time.at(1561353581) do + assert_fetch_s3 url, '8464fcf293454689ec1d3399463c9eeafe2a5b22f2cbd948cba28175a34b073d' + end ensure Gem.configuration[:s3_source] = nil end + def test_fetch_s3_config_creds_with_region + Gem.configuration[:s3_source] = { + 'my-bucket' => {:id => 'testuser', :secret => 'testpass', :region => 'us-west-2'} + } + url = 's3://my-bucket/gems/specs.4.8.gz' + Time.stub :now, Time.at(1561353581) do + assert_fetch_s3 url, '800419b6efb5eacabef2c13c7e493f093df417afc10961f6dcc7e085c9c89e93', nil, 'us-west-2' + end + ensure + Gem.configuration[:s3_source] = nil + end + + def test_fetch_s3_config_creds_with_token + Gem.configuration[:s3_source] = { + 'my-bucket' => {:id => 'testuser', :secret => 'testpass', :security_token => 'testtoken'} + } + url = 's3://my-bucket/gems/specs.4.8.gz' + Time.stub :now, Time.at(1561353581) do + assert_fetch_s3 url, '9686a7b85d29b983f0ee723fd444536258ae1d3a7afa7f8599b3852aef3c8c11', 'testtoken' + end + ensure + Gem.configuration[:s3_source] = nil + end + + def test_fetch_s3_env_creds + ENV['AWS_ACCESS_KEY_ID'] = 'testuser' + ENV['AWS_SECRET_ACCESS_KEY'] = 'testpass' + ENV['AWS_SESSION_TOKEN'] = nil + Gem.configuration[:s3_source] = { + 'my-bucket' => {:provider => 'env'} + } + url = 's3://my-bucket/gems/specs.4.8.gz' + Time.stub :now, Time.at(1561353581) do + assert_fetch_s3 url, '8464fcf293454689ec1d3399463c9eeafe2a5b22f2cbd948cba28175a34b073d' + end + ensure + ENV.each_key {|key| ENV.delete(key) if key.start_with?('AWS')} + Gem.configuration[:s3_source] = nil + end + + def test_fetch_s3_env_creds_with_region + ENV['AWS_ACCESS_KEY_ID'] = 'testuser' + ENV['AWS_SECRET_ACCESS_KEY'] = 'testpass' + ENV['AWS_SESSION_TOKEN'] = nil + Gem.configuration[:s3_source] = { + 'my-bucket' => {:provider => 'env', :region => 'us-west-2'} + } + url = 's3://my-bucket/gems/specs.4.8.gz' + Time.stub :now, Time.at(1561353581) do + assert_fetch_s3 url, '800419b6efb5eacabef2c13c7e493f093df417afc10961f6dcc7e085c9c89e93', nil, 'us-west-2' + end + ensure + ENV.each_key {|key| ENV.delete(key) if key.start_with?('AWS')} + Gem.configuration[:s3_source] = nil + end + + def test_fetch_s3_env_creds_with_token + ENV['AWS_ACCESS_KEY_ID'] = 'testuser' + ENV['AWS_SECRET_ACCESS_KEY'] = 'testpass' + ENV['AWS_SESSION_TOKEN'] = 'testtoken' + Gem.configuration[:s3_source] = { + 'my-bucket' => {:provider => 'env'} + } + url = 's3://my-bucket/gems/specs.4.8.gz' + Time.stub :now, Time.at(1561353581) do + assert_fetch_s3 url, '9686a7b85d29b983f0ee723fd444536258ae1d3a7afa7f8599b3852aef3c8c11', 'testtoken' + end + ensure + ENV.each_key {|key| ENV.delete(key) if key.start_with?('AWS')} + Gem.configuration[:s3_source] = nil + end + def test_fetch_s3_url_creds url = 's3://testuser:testpass@my-bucket/gems/specs.4.8.gz' - assert_fetch_s3 url + Time.stub :now, Time.at(1561353581) do + assert_fetch_s3 url, '8464fcf293454689ec1d3399463c9eeafe2a5b22f2cbd948cba28175a34b073d' + end + end + + def test_fetch_s3_instance_profile_creds + Gem.configuration[:s3_source] = { + 'my-bucket' => {:provider => 'instance_profile'} + } + + url = 's3://my-bucket/gems/specs.4.8.gz' + Time.stub :now, Time.at(1561353581) do + assert_fetch_s3 url, '8464fcf293454689ec1d3399463c9eeafe2a5b22f2cbd948cba28175a34b073d', nil, 'us-east-1', + '{"AccessKeyId": "testuser", "SecretAccessKey": "testpass"}' + end + ensure + Gem.configuration[:s3_source] = nil + end + + def test_fetch_s3_instance_profile_creds_with_region + Gem.configuration[:s3_source] = { + 'my-bucket' => {:provider => 'instance_profile', :region => 'us-west-2'} + } + + url = 's3://my-bucket/gems/specs.4.8.gz' + Time.stub :now, Time.at(1561353581) do + assert_fetch_s3 url, '800419b6efb5eacabef2c13c7e493f093df417afc10961f6dcc7e085c9c89e93', nil, 'us-west-2', + '{"AccessKeyId": "testuser", "SecretAccessKey": "testpass"}' + end + ensure + Gem.configuration[:s3_source] = nil + end + + def test_fetch_s3_instance_profile_creds_with_token + Gem.configuration[:s3_source] = { + 'my-bucket' => {:provider => 'instance_profile'} + } + + url = 's3://my-bucket/gems/specs.4.8.gz' + Time.stub :now, Time.at(1561353581) do + assert_fetch_s3 url, '9686a7b85d29b983f0ee723fd444536258ae1d3a7afa7f8599b3852aef3c8c11', 'testtoken', 'us-east-1', + '{"AccessKeyId": "testuser", "SecretAccessKey": "testpass", "Token": "testtoken"}' + end + ensure + Gem.configuration[:s3_source] = nil end def refute_fetch_s3(url, expected_message)