repl: add mode detection, cli persistent history

this creates a new internal module responsible for providing
the repl created via "iojs" or "iojs -i," and adds the following
options to the readline and repl subsystems:

* "repl mode" - determine whether a repl is strict mode, sloppy mode,
  or auto-detect mode.
* historySize - determine the maximum number of lines a repl will store
  as history.

The built-in repl gains persistent history support when the
NODE_REPL_HISTORY_FILE environment variable is set. This functionality
is not exposed to userland repl instances.

PR-URL: https://github.com/iojs/io.js/pull/1513
Reviewed-By: Fedor Indutny <fedor@indutny.com>

doc/api/readline.markdown

@@ -39,6 +39,8 @@ the following values:
    treated like a TTY, and have ANSI/VT100 escape codes written to it.
    Defaults to checking `isTTY` on the `output` stream upon instantiation.
 
+ - `historySize` - maximum number of history lines retained. Defaults to `30`.
+
 The `completer` function is given the current line entered by the user, and
 is supposed to return an Array with 2 entries:
 

doc/api/repl.markdown

@@ -29,6 +29,18 @@ For example, you could add this to your bashrc file:
 
     alias iojs="env NODE_NO_READLINE=1 rlwrap iojs"
 
+The built-in repl (invoked by running `iojs` or `iojs -i`) may be controlled
+via the following environment variables:
+
+ - `NODE_REPL_HISTORY_FILE` - if given, must be a path to a user-writable,
+   user-readable file. When a valid path is given, persistent history support
+   is enabled: REPL history will persist across `iojs` repl sessions.
+ - `NODE_REPL_HISTORY_SIZE` - defaults to `1000`. In conjunction with
+   `NODE_REPL_HISTORY_FILE`, controls how many lines of history will be
+   persisted. Must be a positive number.
+ - `NODE_REPL_MODE` - may be any of `sloppy`, `strict`, or `magic`. Defaults
+   to `magic`, which will automatically run "strict mode only" statements in
+   strict mode.
 
 ## repl.start(options)
 
@@ -64,6 +76,14 @@ the following values:
    returns the formatting (including coloring) to display. Defaults to
    `util.inspect`.
 
+ - `replMode` - controls whether the repl runs all commands in strict mode,
+   default mode, or a hybrid mode ("magic" mode.) Acceptable values are:
+  * `repl.REPL_MODE_SLOPPY` - run commands in sloppy mode.
+  * `repl.REPL_MODE_STRICT` - run commands in strict mode. This is equivalent to
+  prefacing every repl statement with `'use strict'`.
+  * `repl.REPL_MODE_MAGIC` - attempt to run commands in default mode. If they
+  fail to parse, re-try in strict mode.
+
 You can use your own `eval` function if it has following signature:
 
     function eval(cmd, context, filename, callback) {

lib/internal/repl.js

@@ -0,0 +1,168 @@
+'use strict';
+
+module.exports = {createRepl: createRepl};
+
+const Interface = require('readline').Interface;
+const REPL = require('repl');
+const path = require('path');
+
+// XXX(chrisdickinson): The 15ms debounce value is somewhat arbitrary.
+// The debounce is to guard against code pasted into the REPL.
+const kDebounceHistoryMS = 15;
+
+try {
+  // hack for require.resolve("./relative") to work properly.
+  module.filename = path.resolve('repl');
+} catch (e) {
+  // path.resolve('repl') fails when the current working directory has been
+  // deleted.  Fall back to the directory name of the (absolute) executable
+  // path.  It's not really correct but what are the alternatives?
+  const dirname = path.dirname(process.execPath);
+  module.filename = path.resolve(dirname, 'repl');
+}
+
+// hack for repl require to work properly with node_modules folders
+module.paths = require('module')._nodeModulePaths(module.filename);
+
+function createRepl(env, cb) {
+  const opts = {
+    useGlobal: true,
+    ignoreUndefined: false
+  };
+
+  if (parseInt(env.NODE_NO_READLINE)) {
+    opts.terminal = false;
+  }
+  if (parseInt(env.NODE_DISABLE_COLORS)) {
+    opts.useColors = false;
+  }
+
+  opts.replMode = {
+    'strict': REPL.REPL_MODE_STRICT,
+    'sloppy': REPL.REPL_MODE_SLOPPY,
+    'magic': REPL.REPL_MODE_MAGIC
+  }[String(env.NODE_REPL_MODE).toLowerCase().trim()];
+
+  if (opts.replMode === undefined) {
+    opts.replMode = REPL.REPL_MODE_MAGIC;
+  }
+
+  const historySize = Number(env.NODE_REPL_HISTORY_SIZE);
+  if (!isNaN(historySize) && historySize > 0) {
+    opts.historySize = historySize;
+  } else {
+    // XXX(chrisdickinson): set here to avoid affecting existing applications
+    // using repl instances.
+    opts.historySize = 1000;
+  }
+
+  const repl = REPL.start(opts);
+  if (env.NODE_REPL_HISTORY_PATH) {
+    return setupHistory(repl, env.NODE_REPL_HISTORY_PATH, cb);
+  }
+  repl._historyPrev = _replHistoryMessage;
+  cb(null, repl);
+}
+
+function setupHistory(repl, historyPath, ready) {
+  const fs = require('fs');
+  var timer = null;
+  var writing = false;
+  var pending = false;
+  repl.pause();
+  fs.open(historyPath, 'a+', oninit);
+
+  function oninit(err, hnd) {
+    if (err) {
+      return ready(err);
+    }
+    fs.close(hnd, onclose);
+  }
+
+  function onclose(err) {
+    if (err) {
+      return ready(err);
+    }
+    fs.readFile(historyPath, 'utf8', onread);
+  }
+
+  function onread(err, data) {
+    if (err) {
+      return ready(err);
+    }
+
+    if (data) {
+      try {
+        repl.history = JSON.parse(data);
+        if (!Array.isArray(repl.history)) {
+          throw new Error('Expected array, got ' + typeof repl.history);
+        }
+        repl.history.slice(-repl.historySize);
+      } catch (err) {
+        return ready(
+            new Error(`Could not parse history data in ${historyPath}.`));
+      }
+    }
+
+    fs.open(historyPath, 'w', onhandle);
+  }
+
+  function onhandle(err, hnd) {
+    if (err) {
+      return ready(err);
+    }
+    repl._historyHandle = hnd;
+    repl.on('line', online);
+    repl.resume();
+    return ready(null, repl);
+  }
+
+  // ------ history listeners ------
+  function online() {
+    repl._flushing = true;
+
+    if (timer) {
+      clearTimeout(timer);
+    }
+
+    timer = setTimeout(flushHistory, kDebounceHistoryMS);
+  }
+
+  function flushHistory() {
+    timer = null;
+    if (writing) {
+      pending = true;
+      return;
+    }
+    writing = true;
+    const historyData = JSON.stringify(repl.history, null, 2);
+    fs.write(repl._historyHandle, historyData, 0, 'utf8', onwritten);
+  }
+
+  function onwritten(err, data) {
+    writing = false;
+    if (pending) {
+      pending = false;
+      online();
+    } else {
+      repl._flushing = Boolean(timer);
+      if (!repl._flushing) {
+        repl.emit('flushHistory');
+      }
+    }
+  }
+}
+
+
+function _replHistoryMessage() {
+  if (this.history.length === 0) {
+    this._writeToOutput(
+        '\nPersistent history support disabled. ' +
+        'Set the NODE_REPL_HISTORY_PATH environment variable to ' +
+        'a valid, user-writable path to enable.\n'
+    );
+    this._refreshLine();
+  }
+  this._historyPrev = Interface.prototype._historyPrev;
+  return this._historyPrev();
+}

lib/module.js

@@ -273,6 +273,17 @@ Module._load = function(request, parent, isMain) {
     debug('Module._load REQUEST  ' + (request) + ' parent: ' + parent.id);
   }
 
+  // REPL is a special case, because it needs the real require.
+  if (request === 'internal/repl' || request === 'repl') {
+    if (Module._cache[request]) {
+      return Module._cache[request];
+    }
+    var replModule = new Module(request);
+    replModule._compile(NativeModule.getSource(request), `${request}.js`);
+    NativeModule._cache[request] = replModule;
+    return replModule.exports;
+  }
+
   var filename = Module._resolveFilename(request, parent);
 
   var cachedModule = Module._cache[filename];
@@ -281,14 +292,6 @@ Module._load = function(request, parent, isMain) {
   }
 
   if (NativeModule.nonInternalExists(filename)) {
-    // REPL is a special case, because it needs the real require.
-    if (filename == 'repl') {
-      var replModule = new Module('repl');
-      replModule._compile(NativeModule.getSource('repl'), 'repl.js');
-      NativeModule._cache.repl = replModule;
-      return replModule.exports;
-    }
-
     debug('load native module ' + request);
     return NativeModule.require(filename);
   }
@@ -502,7 +505,7 @@ Module._initPaths = function() {
 
 // bootstrap repl
 Module.requireRepl = function() {
-  return Module._load('repl', '.');
+  return Module._load('internal/repl', '.');
 };
 
 Module._initPaths();

lib/readline.js

@@ -35,14 +35,17 @@ function Interface(input, output, completer, terminal) {
   this._sawReturn = false;
 
   EventEmitter.call(this);
+  var historySize;
 
   if (arguments.length === 1) {
     // an options object was given
     output = input.output;
     completer = input.completer;
     terminal = input.terminal;
+    historySize = input.historySize;
     input = input.input;
   }
+  historySize = historySize || kHistorySize;
 
   completer = completer || function() { return []; };
 
@@ -50,6 +53,12 @@ function Interface(input, output, completer, terminal) {
     throw new TypeError('Argument \'completer\' must be a function');
   }
 
+  if (typeof historySize !== 'number' ||
+      isNaN(historySize) ||
+      historySize < 0) {
+    throw new TypeError('Argument \'historySize\' must be a positive number');
+  }
+
   // backwards compat; check the isTTY prop of the output stream
   //  when `terminal` was not specified
   if (terminal === undefined && !(output === null || output === undefined)) {
@@ -60,6 +69,7 @@ function Interface(input, output, completer, terminal) {
 
   this.output = output;
   this.input = input;
+  this.historySize = historySize;
 
   // Check arity, 2 - for async, 1 for sync
   this.completer = completer.length === 2 ? completer : function(v, callback) {
@@ -214,7 +224,7 @@ Interface.prototype._addHistory = function() {
     this.history.unshift(this.line);
 
     // Only store so many
-    if (this.history.length > kHistorySize) this.history.pop();
+    if (this.history.length > this.historySize) this.history.pop();
   }
 
   this.historyIndex = -1;

lib/repl.js

@@ -40,20 +40,6 @@ function hasOwnProperty(obj, prop) {
 }
 
 
-try {
-  // hack for require.resolve("./relative") to work properly.
-  module.filename = path.resolve('repl');
-} catch (e) {
-  // path.resolve('repl') fails when the current working directory has been
-  // deleted.  Fall back to the directory name of the (absolute) executable
-  // path.  It's not really correct but what are the alternatives?
-  const dirname = path.dirname(process.execPath);
-  module.filename = path.resolve(dirname, 'repl');
-}
-
-// hack for repl require to work properly with node_modules folders
-module.paths = require('module')._nodeModulePaths(module.filename);
-
 // Can overridden with custom print functions, such as `probe` or `eyes.js`.
 // This is the default "writer" value if none is passed in the REPL options.
 exports.writer = util.inspect;
@@ -65,9 +51,23 @@ exports._builtinLibs = ['assert', 'buffer', 'child_process', 'cluster',
   'smalloc'];
 
 
-function REPLServer(prompt, stream, eval_, useGlobal, ignoreUndefined) {
+const BLOCK_SCOPED_ERROR = 'Block-scoped declarations (let, ' +
+    'const, function, class) not yet supported outside strict mode';
+
+
+function REPLServer(prompt,
+                    stream,
+                    eval_,
+                    useGlobal,
+                    ignoreUndefined,
+                    replMode) {
   if (!(this instanceof REPLServer)) {
-    return new REPLServer(prompt, stream, eval_, useGlobal, ignoreUndefined);
+    return new REPLServer(prompt,
+                          stream,
+                          eval_,
+                          useGlobal,
+                          ignoreUndefined,
+                          replMode);
   }
 
   var options, input, output, dom;
@@ -82,6 +82,7 @@ function REPLServer(prompt, stream, eval_, useGlobal, ignoreUndefined) {
     ignoreUndefined = options.ignoreUndefined;
     prompt = options.prompt;
     dom = options.domain;
+    replMode = options.replMode;
   } else if (typeof prompt !== 'string') {
     throw new Error('An options Object, or a prompt String are required');
   } else {
@@ -94,6 +95,7 @@ function REPLServer(prompt, stream, eval_, useGlobal, ignoreUndefined) {
 
   self.useGlobal = !!useGlobal;
   self.ignoreUndefined = !!ignoreUndefined;
+  self.replMode = replMode || exports.REPL_MODE_SLOPPY;
 
   self._inTemplateLiteral = false;
 
@@ -103,19 +105,34 @@ function REPLServer(prompt, stream, eval_, useGlobal, ignoreUndefined) {
   eval_ = eval_ || defaultEval;
 
   function defaultEval(code, context, file, cb) {
-    var err, result;
+    var err, result, retry = false;
     // first, create the Script object to check the syntax
-    try {
-      var script = vm.createScript(code, {
-        filename: file,
-        displayErrors: false
-      });
-    } catch (e) {
-      debug('parse error %j', code, e);
-      if (isRecoverableError(e, self))
-        err = new Recoverable(e);
-      else
-        err = e;
+    while (true) {
+      try {
+        if (!/^\s*$/.test(code) &&
+            (self.replMode === exports.REPL_MODE_STRICT || retry)) {
+          // "void 0" keeps the repl from returning "use strict" as the
+          // result value for let/const statements.
+          code = `'use strict'; void 0; ${code}`;
+        }
+        var script = vm.createScript(code, {
+          filename: file,
+          displayErrors: false
+        });
+      } catch (e) {
+        debug('parse error %j', code, e);
+        if (self.replMode === exports.REPL_MODE_MAGIC &&
+            e.message === BLOCK_SCOPED_ERROR &&
+            !retry) {
+          retry = true;
+          continue;
+        }
+        if (isRecoverableError(e, self))
+          err = new Recoverable(e);
+        else
+          err = e;
+      }
+      break;
     }
 
     if (!err) {
@@ -177,12 +194,13 @@ function REPLServer(prompt, stream, eval_, useGlobal, ignoreUndefined) {
     self.complete(text, callback);
   }
 
-  rl.Interface.apply(this, [
-    self.inputStream,
-    self.outputStream,
-    complete,
-    options.terminal
-  ]);
+  rl.Interface.call(this, {
+    input: self.inputStream,
+    output: self.outputStream,
+    completer: complete,
+    terminal: options.terminal,
+    historySize: options.historySize
+  });
 
   self.setPrompt(prompt !== undefined ? prompt : '> ');
 
@@ -330,11 +348,24 @@ function REPLServer(prompt, stream, eval_, useGlobal, ignoreUndefined) {
 inherits(REPLServer, rl.Interface);
 exports.REPLServer = REPLServer;
 
+exports.REPL_MODE_SLOPPY = Symbol('repl-sloppy');
+exports.REPL_MODE_STRICT = Symbol('repl-strict');
+exports.REPL_MODE_MAGIC = Symbol('repl-magic');
 
 // prompt is a string to print on each line for the prompt,
 // source is a stream to use for I/O, defaulting to stdin/stdout.
-exports.start = function(prompt, source, eval_, useGlobal, ignoreUndefined) {
-  var repl = new REPLServer(prompt, source, eval_, useGlobal, ignoreUndefined);
+exports.start = function(prompt,
+                         source,
+                         eval_,
+                         useGlobal,
+                         ignoreUndefined,
+                         replMode) {
+  var repl = new REPLServer(prompt,
+                            source,
+                            eval_,
+                            useGlobal,
+                            ignoreUndefined,
+                            replMode);
   if (!exports.repl) exports.repl = repl;
   return repl;
 };

node.gyp

@@ -72,6 +72,7 @@
 
       'lib/internal/freelist.js',
       'lib/internal/smalloc.js',
+      'lib/internal/repl.js',
     ],
   },
 

src/node.js

@@ -130,21 +130,20 @@
         // If -i or --interactive were passed, or stdin is a TTY.
         if (process._forceRepl || NativeModule.require('tty').isatty(0)) {
           // REPL
-          var opts = {
-            useGlobal: true,
-            ignoreUndefined: false
-          };
-          if (parseInt(process.env['NODE_NO_READLINE'], 10)) {
-            opts.terminal = false;
-          }
-          if (parseInt(process.env['NODE_DISABLE_COLORS'], 10)) {
-            opts.useColors = false;
-          }
-          var repl = Module.requireRepl().start(opts);
-          repl.on('exit', function() {
-            process.exit();
+          Module.requireRepl().createRepl(process.env, function(err, repl) {
+            if (err) {
+              throw err;
+            }
+            repl.on('exit', function() {
+              if (repl._flushing) {
+                repl.pause();
+                return repl.once('flushHistory', function() {
+                  process.exit();
+                });
+              }
+              process.exit();
+            });
           });
-
         } else {
           // Read all of stdin - execute it.
           process.stdin.setEncoding('utf8');

test/parallel/test-repl-mode.js

@@ -0,0 +1,84 @@
+var common = require('../common');
+var assert = require('assert');
+var Stream = require('stream');
+var repl = require('repl');
+
+common.globalCheck = false;
+
+var tests = [
+  testSloppyMode,
+  testStrictMode,
+  testAutoMode
+];
+
+tests.forEach(function(test) {
+  test();
+});
+
+function testSloppyMode() {
+  var cli = initRepl(repl.REPL_MODE_SLOPPY);
+
+  cli.input.emit('data', `
+    x = 3
+  `.trim() + '\n');
+  assert.equal(cli.output.accumulator.join(''), '> 3\n> ')
+  cli.output.accumulator.length = 0;
+
+  cli.input.emit('data', `
+    let y = 3
+  `.trim() + '\n');
+  assert.ok(/SyntaxError: Block-scoped/.test(
+      cli.output.accumulator.join('')));
+}
+
+function testStrictMode() {
+  var cli = initRepl(repl.REPL_MODE_STRICT);
+
+  cli.input.emit('data', `
+    x = 3
+  `.trim() + '\n');
+  assert.ok(/ReferenceError: x is not defined/.test(
+      cli.output.accumulator.join('')));
+  cli.output.accumulator.length = 0;
+
+  cli.input.emit('data', `
+    let y = 3
+  `.trim() + '\n');
+  assert.equal(cli.output.accumulator.join(''), 'undefined\n> ');
+}
+
+function testAutoMode() {
+  var cli = initRepl(repl.REPL_MODE_MAGIC);
+
+  cli.input.emit('data', `
+    x = 3
+  `.trim() + '\n');
+  assert.equal(cli.output.accumulator.join(''), '> 3\n> ')
+  cli.output.accumulator.length = 0;
+
+  cli.input.emit('data', `
+    let y = 3
+  `.trim() + '\n');
+  assert.equal(cli.output.accumulator.join(''), 'undefined\n> ');
+}
+
+function initRepl(mode) {
+  var input = new Stream();
+  input.write = input.pause = input.resume = function(){};
+  input.readable = true;
+
+  var output = new Stream();
+  output.write = output.pause = output.resume = function(buf) {
+    output.accumulator.push(buf);
+  };
+  output.accumulator = [];
+  output.writable = true;
+
+  return repl.start({
+    input: input,
+    output: output,
+    useColors: false,
+    terminal: false,
+    replMode: mode
+  });
+}

test/parallel/test-repl-options.js

@@ -25,6 +25,8 @@ assert.equal(r1.terminal, true);
 assert.equal(r1.useColors, r1.terminal);
 assert.equal(r1.useGlobal, false);
 assert.equal(r1.ignoreUndefined, false);
+assert.equal(r1.replMode, repl.REPL_MODE_SLOPPY);
+assert.equal(r1.historySize, 30);
 
 // test r1 for backwards compact
 assert.equal(r1.rli.input, stream);
@@ -45,7 +47,8 @@ var r2 = repl.start({
   useGlobal: true,
   ignoreUndefined: true,
   eval: evaler,
-  writer: writer
+  writer: writer,
+  replMode: repl.REPL_MODE_STRICT
 });
 assert.equal(r2.input, stream);
 assert.equal(r2.output, stream);
@@ -56,6 +59,7 @@ assert.equal(r2.useColors, true);
 assert.equal(r2.useGlobal, true);
 assert.equal(r2.ignoreUndefined, true);
 assert.equal(r2.writer, writer);
+assert.equal(r2.replMode, repl.REPL_MODE_STRICT);
 
 // test r2 for backwards compact
 assert.equal(r2.rli.input, stream);
@@ -64,3 +68,14 @@ assert.equal(r2.rli.input, r2.inputStream);
 assert.equal(r2.rli.output, r2.outputStream);
 assert.equal(r2.rli.terminal, false);
 
+// testing out "magic" replMode
+var r3 = repl.start({
+  input: stream,
+  output: stream,
+  writer: writer,
+  replMode: repl.REPL_MODE_MAGIC,
+  historySize: 50
+})
+
+assert.equal(r3.replMode, repl.REPL_MODE_MAGIC);
+assert.equal(r3.historySize, 50);