From dddef93bbd0327e24dff8c87caeb66b6f64800e6 Mon Sep 17 00:00:00 2001
From: Peter Zhu <peter@peterzhu.ca>
Date: Tue, 30 Jan 2024 10:12:06 -0500
Subject: [PATCH] Fix memory leak in File.expand_path

File.expand_path leaks the dir if the encodings are not compatible.

For example:

    Encoding.default_external = Encoding::UTF_16BE

    10.times do
      100_000.times do
        File.expand_path("./a")
      rescue
      end

      puts `ps -o rss= -p #{$$}`
    end

Before:

    12288
    15488
    18656
    21872
    25056
    28240
    31392
    34688
    37856
    41056

After:

    9680
    9728
    9728
    9792
    9792
    9792
    9792
    9792
    9792
---
 file.c | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/file.c b/file.c
index c2a78cc75d..cfcb4954b1 100644
--- a/file.c
+++ b/file.c
@@ -3710,7 +3710,15 @@ append_fspath(VALUE result, VALUE fname, char *dir, rb_encoding **enc, rb_encodi
     size_t dirlen = strlen(dir), buflen = rb_str_capacity(result);
 
     if (NORMALIZE_UTF8PATH || *enc != fsenc) {
-        rb_encoding *direnc = fs_enc_check(fname, dirname = ospath_new(dir, dirlen, fsenc));
+        dirname = ospath_new(dir, dirlen, fsenc);
+        if (!rb_enc_compatible(fname, dirname)) {
+            xfree(dir);
+            /* rb_enc_check must raise because the two encodings are not
+             * compatible. */
+            rb_enc_check(fname, dirname);
+            rb_bug("unreachable");
+        }
+        rb_encoding *direnc = fs_enc_check(fname, dirname);
         if (direnc != fsenc) {
             dirname = rb_str_conv_enc(dirname, fsenc, direnc);
             RSTRING_GETMEM(dirname, cwdp, dirlen);