diff --git a/lib/bundler/source/rubygems.rb b/lib/bundler/source/rubygems.rb
index af57acbbc2..9790753204 100644
--- a/lib/bundler/source/rubygems.rb
+++ b/lib/bundler/source/rubygems.rb
@@ -399,16 +399,11 @@ module Bundler
         @remote_specs ||= Index.build do |idx|
           index_fetchers = fetchers - api_fetchers
 
-          # gather lists from non-api sites
-          fetch_names(index_fetchers, nil, idx, false)
-
-          # legacy multi-remote sources need special logic to figure out
-          # dependency names and that logic can be very costly if one remote
-          # uses the dependency API but others don't. So use full indexes
-          # consistently in that particular case.
-          allow_api = !multiple_remotes?
-
-          fetch_names(api_fetchers, allow_api && dependency_names, idx, false)
+          if index_fetchers.empty?
+            fetch_names(api_fetchers, dependency_names, idx, false)
+          else
+            fetch_names(fetchers, nil, idx, false)
+          end
         end
       end
 
diff --git a/spec/bundler/install/gemfile/sources_spec.rb b/spec/bundler/install/gemfile/sources_spec.rb
index 9095eca03b..1f89f9f0f9 100644
--- a/spec/bundler/install/gemfile/sources_spec.rb
+++ b/spec/bundler/install/gemfile/sources_spec.rb
@@ -35,6 +35,15 @@ RSpec.describe "bundle install with gems on multiple sources" do
         expect(the_bundle).to include_gems("rack-obama 1.0.0", "rack 1.0.0", :source => "remote1")
       end
 
+      it "does not use the full index unnecessarily", :bundler => "< 3" do
+        bundle :install, :artifice => "compact_index", :verbose => true
+
+        expect(out).to include("https://gem.repo1/versions")
+        expect(out).to include("https://gem.repo3/versions")
+        expect(out).not_to include("https://gem.repo1/quick/Marshal.4.8/")
+        expect(out).not_to include("https://gem.repo3/quick/Marshal.4.8/")
+      end
+
       it "fails", :bundler => "3" do
         bundle :install, :artifice => "compact_index", :raise_on_error => false
         expect(err).to include("Each source after the first must include a block")