From b14674b236445fb70f484603e678722760f678f4 Mon Sep 17 00:00:00 2001
From: Peter Zhu <peter@peterzhu.ca>
Date: Fri, 19 Jan 2024 10:26:24 -0500
Subject: [PATCH] Memory leak with TracePoint on bmethod

[Bug #20194]

When disabling the TracePoint on bmethod, the hooks list is not freed.

For example:

    obj = Object.new
    obj.define_singleton_method(:foo) {}
    bmethod = obj.method(:foo)
    tp = TracePoint.new(:return) {}

    10.times do
      100_000.times do
        tp.enable(target: bmethod) {}
      end

      puts `ps -o rss= -p #{$$}`
    end

Before:

    18208
    22832
    26528
    29728
    34000
    37776
    40864
    44400
    47680
    51504

After:

    16688
    17168
    17168
    17248
    17696
    17760
    17824
    17824
    17856
    17920
---
 test/ruby/test_settracefunc.rb | 13 +++++++++++++
 vm_trace.c                     |  1 +
 2 files changed, 14 insertions(+)

diff --git a/test/ruby/test_settracefunc.rb b/test/ruby/test_settracefunc.rb
index b7711e191d..dbaf0aaf09 100644
--- a/test/ruby/test_settracefunc.rb
+++ b/test/ruby/test_settracefunc.rb
@@ -625,6 +625,19 @@ PREP
 CODE
   end
 
+  def test_tracepoint_bmethod_memory_leak
+    assert_no_memory_leak([], '', "#{<<~"begin;"}\n#{<<~'end;'}", "[Bug #20194]", rss: true)
+      obj = Object.new
+      obj.define_singleton_method(:foo) {}
+      bmethod = obj.method(:foo)
+      tp = TracePoint.new(:return) {}
+    begin;
+      1_000_000.times do
+        tp.enable(target: bmethod) {}
+      end
+    end;
+  end
+
   def trace_by_set_trace_func
     events = []
     trace = nil
diff --git a/vm_trace.c b/vm_trace.c
index fb896fd423..9d0507879f 100644
--- a/vm_trace.c
+++ b/vm_trace.c
@@ -1259,6 +1259,7 @@ rb_tracepoint_enable_for_target(VALUE tpval, VALUE target, VALUE target_line)
             (tp->events & (RUBY_EVENT_CALL | RUBY_EVENT_RETURN))) {
             if (def->body.bmethod.hooks == NULL) {
                 def->body.bmethod.hooks = ZALLOC(rb_hook_list_t);
+                def->body.bmethod.hooks->is_local = true;
             }
             rb_hook_list_connect_tracepoint(target, def->body.bmethod.hooks, tpval, 0);
             rb_hash_aset(tp->local_target_set, target, Qfalse);