marshal.c: fix infinite recursion

* marshal.c (check_userdump_arg): marshal_dump should not return
  an instance of the same class, otherwise it causes infinite
  recursion.  [ruby-core:78289] [Bug #12974]

git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@56894 b2dd03c8-39d4-4d8f-98ff-823fe69b080e

marshal.c

@@ -178,8 +178,22 @@ check_dump_arg(VALUE ret, struct dump_arg *arg, const char *name)
     }
     return ret;
 }
+
+static VALUE
+check_userdump_arg(VALUE obj, ID sym, int argc, const VALUE *argv,
+		   struct dump_arg *arg, const char *name)
+{
+    VALUE ret = rb_funcallv(obj, sym, argc, argv);
+    VALUE klass = CLASS_OF(obj);
+    if (CLASS_OF(ret) == klass) {
+        rb_raise(rb_eRuntimeError, "%"PRIsVALUE"#%s returned same class instance",
+		 klass, name);
+    }
+    return check_dump_arg(ret, arg, name);
+}
+
 #define dump_funcall(arg, obj, sym, argc, argv) \
-    check_dump_arg(rb_funcallv(obj, sym, argc, argv), arg, name_##sym)
+    check_userdump_arg(obj, sym, argc, argv, arg, name_##sym)
 #define dump_check_funcall(arg, obj, sym, argc, argv) \
     check_dump_arg(rb_check_funcall(obj, sym, argc, argv), arg, name_##sym)
 

test/ruby/test_marshal.rb

@@ -757,4 +757,16 @@ class TestMarshal < Test::Unit::TestCase
     end
     assert_equal(obj, Marshal.load(dump))
   end
+
+  class Bug12974
+    def marshal_dump
+      dup
+    end
+  end
+
+  def test_marshal_dump_recursion
+    assert_raise_with_message(RuntimeError, /same class instance/) do
+      Marshal.dump(Bug12974.new)
+    end
+  end
 end