diff --git a/lib/bundler/rubygems_gem_installer.rb b/lib/bundler/rubygems_gem_installer.rb
index 3ec73452b3..858ebf32cf 100644
--- a/lib/bundler/rubygems_gem_installer.rb
+++ b/lib/bundler/rubygems_gem_installer.rb
@@ -150,6 +150,7 @@ module Bundler
 
     def strict_rm_rf(dir)
       return unless File.exist?(dir)
+      return if Dir.empty?(dir)
 
       parent = File.dirname(dir)
       parent_st = File.stat(parent)
diff --git a/spec/bundler/commands/install_spec.rb b/spec/bundler/commands/install_spec.rb
index d4bc1d2922..9353d24d95 100644
--- a/spec/bundler/commands/install_spec.rb
+++ b/spec/bundler/commands/install_spec.rb
@@ -1059,6 +1059,34 @@ RSpec.describe "bundle install with gem sources" do
     end
   end
 
+  describe "when gems path is world writable (no sticky bit set), but previous install is just an empty dir (like it happens with default gems)", :permissions do
+    let(:gems_path) { bundled_app("vendor/#{Bundler.ruby_scope}/gems") }
+    let(:full_path) { gems_path.join("foo-1.0.0") }
+
+    before do
+      build_repo4 do
+        build_gem "foo", "1.0.0" do |s|
+          s.write "CHANGELOG.md", "foo"
+        end
+      end
+
+      gemfile <<-G
+        source "https://gem.repo4"
+        gem 'foo'
+      G
+    end
+
+    it "does not try to remove the directory and thus don't abort with an error about unsafe directory removal" do
+      bundle "config set --local path vendor"
+
+      FileUtils.mkdir_p(gems_path)
+      FileUtils.chmod(0o777, gems_path)
+      Dir.mkdir(full_path)
+
+      bundle "install"
+    end
+  end
+
   describe "when bundle cache path does not have write access", :permissions do
     let(:cache_path) { bundled_app("vendor/#{Bundler.ruby_scope}/cache") }