diff --git a/lib/bundler/rubygems_gem_installer.rb b/lib/bundler/rubygems_gem_installer.rb
index 87b9772c27..df2dcdb454 100644
--- a/lib/bundler/rubygems_gem_installer.rb
+++ b/lib/bundler/rubygems_gem_installer.rb
@@ -90,6 +90,14 @@ module Bundler
       end
     end
 
+    def spec
+      if Bundler.rubygems.provides?("< 3.3.12") # RubyGems implementation rescues and re-raises errors before 3.3.12 and we don't want that
+        @package.spec
+      else
+        super
+      end
+    end
+
     private
 
     def strict_rm_rf(dir)
diff --git a/lib/bundler/rubygems_integration.rb b/lib/bundler/rubygems_integration.rb
index bf93c57e0b..08af0610c6 100644
--- a/lib/bundler/rubygems_integration.rb
+++ b/lib/bundler/rubygems_integration.rb
@@ -203,8 +203,9 @@ module Bundler
       EXT_LOCK
     end
 
-    def spec_from_gem(path, policy = nil)
-      gem_from_path(path, security_policies[policy]).spec
+    def spec_from_gem(path)
+      require "rubygems/package"
+      Gem::Package.new(path).spec
     end
 
     def build_gem(gem_dir, spec)
@@ -498,13 +499,6 @@ module Bundler
       Gem::RemoteFetcher.new(proxy)
     end
 
-    def gem_from_path(path, policy = nil)
-      require "rubygems/package"
-      p = Gem::Package.new(path)
-      p.security_policy = policy if policy
-      p
-    end
-
     def build(spec, skip_validation = false)
       require "rubygems/package"
       Gem::Package.build(spec, skip_validation)
diff --git a/lib/bundler/source/rubygems.rb b/lib/bundler/source/rubygems.rb
index 3cb79667d8..f78e6a443b 100644
--- a/lib/bundler/source/rubygems.rb
+++ b/lib/bundler/source/rubygems.rb
@@ -155,27 +155,11 @@ module Bundler
           Installer.ambiguous_gems << [spec.name, *uris] if uris.length > 1
 
           path = fetch_gem(spec, options[:previous_spec])
-          begin
-            s = Bundler.rubygems.spec_from_gem(path, Bundler.settings["trust-policy"])
-          rescue Gem::Security::Exception => e
-            raise SecurityError,
-              "The gem #{File.basename(path, ".gem")} can't be installed because " \
-              "the security policy didn't allow it, with the message: #{e.message}"
-          rescue Gem::Package::FormatError
-            Bundler.rm_rf(path)
-            raise
-          end
-
-          spec.__swap__(s)
         else
           path = cached_gem(spec)
           raise GemNotFound, "Could not find #{spec.file_name} for installation" unless path
         end
 
-        message = "Installing #{version_message(spec, options[:previous_spec])}"
-        message += " with native extensions" if spec.extensions.any?
-        Bundler.ui.confirm message
-
         if requires_sudo?
           install_path = Bundler.tmp(spec.full_name)
           bin_path     = install_path.join("bin")
@@ -188,8 +172,9 @@ module Bundler
 
         require_relative "../rubygems_gem_installer"
 
-        installed_spec = Bundler::RubyGemsGemInstaller.at(
+        installer = Bundler::RubyGemsGemInstaller.at(
           path,
+          :security_policy     => Bundler.rubygems.security_policies[Bundler.settings["trust-policy"]],
           :install_dir         => install_path.to_s,
           :bin_dir             => bin_path.to_s,
           :ignore_dependencies => true,
@@ -198,7 +183,29 @@ module Bundler
           :build_args          => options[:build_args],
           :bundler_expected_checksum => spec.respond_to?(:checksum) && spec.checksum,
           :bundler_extension_cache_path => extension_cache_path(spec)
-        ).install
+        )
+
+        if spec.remote
+          s = begin
+            installer.spec
+          rescue Gem::Package::FormatError
+            Bundler.rm_rf(path)
+            raise
+          rescue Gem::Security::Exception => e
+            raise SecurityError,
+             "The gem #{File.basename(path, ".gem")} can't be installed because " \
+             "the security policy didn't allow it, with the message: #{e.message}"
+          end
+
+          spec.__swap__(s)
+        end
+
+        message = "Installing #{version_message(spec, options[:previous_spec])}"
+        message += " with native extensions" if spec.extensions.any?
+        Bundler.ui.confirm message
+
+        installed_spec = installer.install
+
         spec.full_gem_path = installed_spec.full_gem_path
         spec.loaded_from = installed_spec.loaded_from