JSON.generate: call to_json on String subclasses

Fix: https://github.com/ruby/json/issues/667

This is yet another behavior on which the various implementations
differed, but the C implementation used to call `to_json` on String
subclasses used as keys.

This was optimized out in e125072130229e54a651f7b11d7d5a782ae7fb65
but there is an Active Support test case for it, so it's best to
make all 3 implementation respect this behavior.

ext/json/fbuffer/fbuffer.h

@@ -42,6 +42,10 @@ static VALUE fbuffer_to_s(FBuffer *fb);
 #define RB_UNLIKELY(expr) expr
 #endif
 
+#ifndef RB_LIKELY
+#define RB_LIKELY(expr) expr
+#endif
+
 static void fbuffer_stack_init(FBuffer *fb, unsigned long initial_length, char *stack_buffer, long stack_buffer_size)
 {
     fb->initial_length = (initial_length > 0) ? initial_length : FBUFFER_INITIAL_LENGTH_DEFAULT;

ext/json/generator/generator.c

@@ -737,7 +737,11 @@ json_object_i(VALUE key, VALUE val, VALUE _arg)
             break;
     }
 
-    generate_json_string(buffer, data, state, key_to_s);
+    if (RB_LIKELY(RBASIC_CLASS(key_to_s) == rb_cString)) {
+        generate_json_string(buffer, data, state, key_to_s);
+    } else {
+        generate_json(buffer, data, state, key_to_s);
+    }
     if (RB_UNLIKELY(state->space_before)) fbuffer_append_str(buffer, state->space_before);
     fbuffer_append_char(buffer, ':');
     if (RB_UNLIKELY(state->space)) fbuffer_append_str(buffer, state->space);

test/json/json_generator_test.rb

@@ -486,6 +486,41 @@ class JSONGeneratorTest < Test::Unit::TestCase
     end
   end
 
+  class MyCustomString < String
+    def to_json(_state = nil)
+      '"my_custom_key"'
+    end
+
+    def to_s
+      self
+    end
+  end
+
+  def test_string_subclass_as_keys
+    # Ref: https://github.com/ruby/json/issues/667
+    # if key.to_s doesn't return a bare string, we call `to_json` on it.
+    key = MyCustomString.new("won't be used")
+    assert_equal '{"my_custom_key":1}', JSON.generate(key => 1)
+  end
+
+  class FakeString
+    def to_json(_state = nil)
+      raise "Shouldn't be called"
+    end
+
+    def to_s
+      self
+    end
+  end
+
+  def test_custom_object_as_keys
+    key = FakeString.new
+    error = assert_raise(TypeError) do
+      JSON.generate(key => 1)
+    end
+    assert_match "FakeString", error.message
+  end
+
   def test_to_json_called_with_state_object
     object = Object.new
     called = false