From 6c252912af4981f016a9abdb4c1689307a4f1d2f Mon Sep 17 00:00:00 2001
From: Peter Zhu <peter@peterzhu.ca>
Date: Wed, 3 Jan 2024 15:59:37 -0500
Subject: [PATCH] Memory leak when duplicating identhash

[Bug #20145]

Before this commit, both copy_compare_by_id and hash_copy will create a
copy of the ST table, so the ST table created in copy_compare_by_id will
be leaked.

    h = { 1 => 2 }.compare_by_identity

    10.times do
      1_000_000.times do
        h.select { false }
      end

      puts `ps -o rss= -p #{$$}`
    end

Before:

    110736
    204352
    300272
    395520
    460704
    476736
    542000
    604704
    682624
    770528

After:

    15504
    16048
    16144
    16256
    16320
    16320
    16752
    16752
    16752
    16752
---
 hash.c                 | 10 +++++++++-
 test/ruby/test_hash.rb | 10 ++++++++++
 2 files changed, 19 insertions(+), 1 deletion(-)

diff --git a/hash.c b/hash.c
index 51f8f85d63..97b5b52c10 100644
--- a/hash.c
+++ b/hash.c
@@ -1558,7 +1558,15 @@ hash_copy(VALUE ret, VALUE hash)
 static VALUE
 hash_dup_with_compare_by_id(VALUE hash)
 {
-    return hash_copy(copy_compare_by_id(rb_hash_new(), hash), hash);
+    VALUE dup = hash_alloc_flags(rb_cHash, 0, Qnil, RHASH_ST_TABLE_P(hash));
+    if (RHASH_ST_TABLE_P(hash)) {
+        RHASH_SET_ST_FLAG(dup);
+    }
+    else {
+        RHASH_UNSET_ST_FLAG(dup);
+    }
+
+    return hash_copy(dup, hash);
 }
 
 static VALUE
diff --git a/test/ruby/test_hash.rb b/test/ruby/test_hash.rb
index 70daea0ef1..c72b256bab 100644
--- a/test/ruby/test_hash.rb
+++ b/test/ruby/test_hash.rb
@@ -1458,6 +1458,16 @@ class TestHash < Test::Unit::TestCase
     assert_predicate(h.dup, :compare_by_identity?, bug8703)
   end
 
+  def test_compare_by_identy_memory_leak
+    assert_no_memory_leak([], "", "#{<<~"begin;"}\n#{<<~'end;'}", "[Bug #20145]", rss: true)
+    begin;
+      h = { 1 => 2 }.compare_by_identity
+      1_000_000.times do
+        h.select { false }
+      end
+    end;
+  end
+
   def test_same_key
     bug9646 = '[ruby-dev:48047] [Bug #9646] Infinite loop at Hash#each'
     h = @cls[a=[], 1]