util: allow returning this from custom inspect

If a custom inspection function returned `this`, use that value
for further formatting instead of going into infinite recursion.

This is particularly useful when combined with `util.inspect.custom`
because returning `this` from such a method makes it easy to
have an `inspect()` function that is ignored by `util.inspect` without
actually having to provide an alternative for custom inspection.

PR-URL: https://github.com/nodejs/node/pull/8174
Reviewed-By: James M Snell <jasnell@gmail.com>
Reviewed-By: Michaël Zasso <mic.besace@gmail.com>

lib/util.js

@@ -362,12 +362,17 @@ function formatValue(ctx, value, recurseTimes) {
         // Also filter out any prototype objects using the circular check.
         !(value.constructor && value.constructor.prototype === value)) {
       let ret = maybeCustomInspect.call(value, recurseTimes, ctx);
+
+      // If the custom inspection method returned `this`, don't go into
+      // infinite recursion.
+      if (ret !== value) {
         if (typeof ret !== 'string') {
           ret = formatValue(ctx, ret, recurseTimes);
         }
         return ret;
       }
     }
+  }
 
   // Primitive types cannot have properties
   var primitive = formatPrimitive(ctx, value);

test/parallel/test-util-inspect.js

@@ -567,6 +567,16 @@ assert.doesNotThrow(function() {
   );
 }
 
+{
+  // Returning `this` from a custom inspection function works.
+  assert.strictEqual(util.inspect({ a: 123, inspect() { return this; } }),
+                     '{ a: 123, inspect: [Function: inspect] }');
+
+  const subject = { a: 123, [util.inspect.custom]() { return this; } };
+  assert.strictEqual(util.inspect(subject),
+                     '{ a: 123 }');
+}
+
 // util.inspect with "colors" option should produce as many lines as without it
 function test_lines(input) {
   var count_lines = function(str) {