From 82b57d7bfeefd717c10f7a5a3484aca6b3e708a3 Mon Sep 17 00:00:00 2001
From: Peter Zhu <peter@peterzhu.ca>
Date: Mon, 8 Jan 2024 11:32:48 -0500
Subject: [PATCH] Fix memory leak when duplicating too complex object

[Bug #20162]

Creating a ST table then calling st_replace leaks memory because the
st_replace overwrites the ST table without freeing any of the existing
memory. This commit changes it to use st_copy instead.

For example:

    RubyVM::Shape.exhaust_shapes

    o = Object.new
    o.instance_variable_set(:@a, 0)

    10.times do
      100_000.times { o.dup }

      puts `ps -o rss= -p #{$$}`
    end

Before:

    23264
    33600
    42672
    52160
    61600
    71728
    81056
    90528
    100560
    109840

After:

    14752
    14816
    15584
    15584
    15664
    15664
    15664
    15664
    15664
    15664
---
 object.c                 |  3 +--
 test/ruby/test_shapes.rb | 13 +++++++++++++
 2 files changed, 14 insertions(+), 2 deletions(-)

diff --git a/object.c b/object.c
index 9a5a8fbe8a..a0783e9ef7 100644
--- a/object.c
+++ b/object.c
@@ -302,8 +302,7 @@ rb_obj_copy_ivar(VALUE dest, VALUE obj)
 
     if (rb_shape_obj_too_complex(obj)) {
         // obj is TOO_COMPLEX so we can copy its iv_hash
-        st_table * table = rb_st_init_numtable_with_size(rb_st_table_size(ROBJECT_IV_HASH(obj)));
-        st_replace(table, ROBJECT_IV_HASH(obj));
+        st_table *table = st_copy(ROBJECT_IV_HASH(obj));
         rb_obj_convert_to_too_complex(dest, table);
 
         return;
diff --git a/test/ruby/test_shapes.rb b/test/ruby/test_shapes.rb
index 885b762eb9..ee99fbad6d 100644
--- a/test/ruby/test_shapes.rb
+++ b/test/ruby/test_shapes.rb
@@ -942,6 +942,19 @@ class TestShapes < Test::Unit::TestCase
     assert_shape_equal(RubyVM::Shape.of(obj), RubyVM::Shape.of(obj2))
   end
 
+  def test_duplicating_too_complex_objects_memory_leak
+    assert_no_memory_leak([], "#{<<~'begin;'}", "#{<<~'end;'}", "[Bug #20162]", rss: true)
+      RubyVM::Shape.exhaust_shapes
+
+      o = Object.new
+      o.instance_variable_set(:@a, 0)
+    begin;
+      1_000_000.times do
+        o.dup
+      end
+    end;
+  end
+
   def test_freezing_and_duplicating_object
     obj = Object.new.freeze
     obj2 = obj.dup