diff --git a/ext/json/generator/generator.c b/ext/json/generator/generator.c
index 503baca65f..e4761fcb8b 100644
--- a/ext/json/generator/generator.c
+++ b/ext/json/generator/generator.c
@@ -71,6 +71,28 @@ static void generate_json_float(FBuffer *buffer, struct generate_json_data *data
 
 static int usascii_encindex, utf8_encindex, binary_encindex;
 
+#ifdef RBIMPL_ATTR_NORETURN
+RBIMPL_ATTR_NORETURN()
+#endif
+static void raise_generator_error_str(VALUE invalid_object, VALUE str)
+{
+    VALUE exc = rb_exc_new_str(eGeneratorError, str);
+    rb_ivar_set(exc, rb_intern("@invalid_object"), invalid_object);
+    rb_exc_raise(exc);
+}
+
+#ifdef RBIMPL_ATTR_NORETURN
+RBIMPL_ATTR_NORETURN()
+#endif
+static void raise_generator_error(VALUE invalid_object, const char *fmt, ...)
+{
+    va_list args;
+    va_start(args, fmt);
+    VALUE str = rb_vsprintf(fmt, args);
+    va_end(args);
+    raise_generator_error_str(invalid_object, str);
+}
+
 /* Converts in_string to a JSON string (without the wrapping '"'
  * characters) in FBuffer out_buffer.
  *
@@ -867,6 +889,17 @@ static inline int enc_utf8_compatible_p(int enc_idx)
     return 0;
 }
 
+static VALUE encode_json_string_try(VALUE str)
+{
+    return rb_funcall(str, i_encode, 1, Encoding_UTF_8);
+}
+
+static VALUE encode_json_string_rescue(VALUE str, VALUE exception)
+{
+    raise_generator_error_str(str, rb_funcall(exception, rb_intern("message"), 0));
+    return Qundef;
+}
+
 static inline VALUE ensure_valid_encoding(VALUE str)
 {
     int encindex = RB_ENCODING_GET(str);
@@ -886,7 +919,7 @@ static inline VALUE ensure_valid_encoding(VALUE str)
             }
         }
 
-        str = rb_funcall(str, i_encode, 1, Encoding_UTF_8);
+        str = rb_rescue(encode_json_string_try, str, encode_json_string_rescue, str);
     }
     return str;
 }
@@ -909,7 +942,7 @@ static void generate_json_string(FBuffer *buffer, struct generate_json_data *dat
             }
             break;
         default:
-            rb_raise(rb_path2class("JSON::GeneratorError"), "source sequence is illegal/malformed utf-8");
+            raise_generator_error(obj, "source sequence is illegal/malformed utf-8");
             break;
     }
     fbuffer_append_char(buffer, '"');
@@ -957,10 +990,8 @@ static void generate_json_float(FBuffer *buffer, struct generate_json_data *data
     char allow_nan = state->allow_nan;
     VALUE tmp = rb_funcall(obj, i_to_s, 0);
     if (!allow_nan) {
-        if (isinf(value)) {
-            rb_raise(eGeneratorError, "%"PRIsVALUE" not allowed in JSON", tmp);
-        } else if (isnan(value)) {
-            rb_raise(eGeneratorError, "%"PRIsVALUE" not allowed in JSON", tmp);
+        if (isinf(value) || isnan(value)) {
+            raise_generator_error(obj, "%"PRIsVALUE" not allowed in JSON", tmp);
         }
     }
     fbuffer_append_str(buffer, tmp);
@@ -1008,7 +1039,7 @@ static void generate_json(FBuffer *buffer, struct generate_json_data *data, JSON
             default:
             general:
                 if (state->strict) {
-                    rb_raise(eGeneratorError, "%"PRIsVALUE" not allowed in JSON", CLASS_OF(obj));
+                    raise_generator_error(obj, "%"PRIsVALUE" not allowed in JSON", CLASS_OF(obj));
                 } else if (rb_respond_to(obj, i_to_json)) {
                     tmp = rb_funcall(obj, i_to_json, 1, vstate_get(data));
                     Check_Type(tmp, T_STRING);
@@ -1036,10 +1067,6 @@ static VALUE generate_json_rescue(VALUE d, VALUE exc)
     struct generate_json_data *data = (struct generate_json_data *)d;
     fbuffer_free(data->buffer);
 
-    if (RBASIC_CLASS(exc) == rb_path2class("Encoding::UndefinedConversionError")) {
-        exc = rb_exc_new_str(eGeneratorError, rb_funcall(exc, rb_intern("message"), 0));
-    }
-
     rb_exc_raise(exc);
 
     return Qundef;
@@ -1537,10 +1564,11 @@ void Init_generator(void)
     VALUE mExt = rb_define_module_under(mJSON, "Ext");
     VALUE mGenerator = rb_define_module_under(mExt, "Generator");
 
+    rb_global_variable(&eGeneratorError);
     eGeneratorError = rb_path2class("JSON::GeneratorError");
+
+    rb_global_variable(&eNestingError);
     eNestingError = rb_path2class("JSON::NestingError");
-    rb_gc_register_mark_object(eGeneratorError);
-    rb_gc_register_mark_object(eNestingError);
 
     cState = rb_define_class_under(mGenerator, "State", rb_cObject);
     rb_define_alloc_func(cState, cState_s_allocate);
diff --git a/ext/json/lib/json/common.rb b/ext/json/lib/json/common.rb
index a88a3fffa5..197ae11f6a 100644
--- a/ext/json/lib/json/common.rb
+++ b/ext/json/lib/json/common.rb
@@ -143,7 +143,23 @@ module JSON
   # :startdoc:
 
   # This exception is raised if a generator or unparser error occurs.
-  class GeneratorError < JSONError; end
+  class GeneratorError < JSONError
+    attr_reader :invalid_object
+
+    def initialize(message, invalid_object = nil)
+      super(message)
+      @invalid_object = invalid_object
+    end
+
+    def detailed_message(...)
+      if @invalid_object.nil?
+        super
+      else
+        "#{super}\nInvalid object: #{@invalid_object.inspect}"
+      end
+    end
+  end
+
   # For backwards compatibility
   UnparserError = GeneratorError # :nodoc:
 
diff --git a/test/json/json_generator_test.rb b/test/json/json_generator_test.rb
index 700220a152..6e4e293db3 100755
--- a/test/json/json_generator_test.rb
+++ b/test/json/json_generator_test.rb
@@ -250,17 +250,20 @@ class JSONGeneratorTest < Test::Unit::TestCase
   end
 
   def test_allow_nan
-    assert_raise(GeneratorError) { generate([JSON::NaN]) }
+    error = assert_raise(GeneratorError) { generate([JSON::NaN]) }
+    assert_same JSON::NaN, error.invalid_object
     assert_equal '[NaN]', generate([JSON::NaN], :allow_nan => true)
     assert_raise(GeneratorError) { fast_generate([JSON::NaN]) }
     assert_raise(GeneratorError) { pretty_generate([JSON::NaN]) }
     assert_equal "[\n  NaN\n]", pretty_generate([JSON::NaN], :allow_nan => true)
-    assert_raise(GeneratorError) { generate([JSON::Infinity]) }
+    error = assert_raise(GeneratorError) { generate([JSON::Infinity]) }
+    assert_same JSON::Infinity, error.invalid_object
     assert_equal '[Infinity]', generate([JSON::Infinity], :allow_nan => true)
     assert_raise(GeneratorError) { fast_generate([JSON::Infinity]) }
     assert_raise(GeneratorError) { pretty_generate([JSON::Infinity]) }
     assert_equal "[\n  Infinity\n]", pretty_generate([JSON::Infinity], :allow_nan => true)
-    assert_raise(GeneratorError) { generate([JSON::MinusInfinity]) }
+    error = assert_raise(GeneratorError) { generate([JSON::MinusInfinity]) }
+    assert_same JSON::MinusInfinity, error.invalid_object
     assert_equal '[-Infinity]', generate([JSON::MinusInfinity], :allow_nan => true)
     assert_raise(GeneratorError) { fast_generate([JSON::MinusInfinity]) }
     assert_raise(GeneratorError) { pretty_generate([JSON::MinusInfinity]) }
@@ -487,9 +490,13 @@ class JSONGeneratorTest < Test::Unit::TestCase
       ["\x82\xAC\xEF".b].to_json
     end
 
-    assert_raise(JSON::GeneratorError) do
-      { foo: "\x82\xAC\xEF".b }.to_json
+    badly_encoded = "\x82\xAC\xEF".b
+    exception = assert_raise(JSON::GeneratorError) do
+      { foo: badly_encoded }.to_json
     end
+
+    assert_kind_of EncodingError, exception.cause
+    assert_same badly_encoded, exception.invalid_object
   end
 
   class MyCustomString < String