crypto: improve error handling in parseKeyEncoding

This change only affects KeyObject.export().

PR-URL: https://github.com/nodejs/node/pull/26455
Reviewed-By: Luigi Pinca <luigipinca@gmail.com>
Reviewed-By: Anna Henningsen <anna@addaleax.net>
Reviewed-By: Ruben Bridgewater <ruben@bridgewater.de>
Reviewed-By: Rich Trott <rtrott@gmail.com>
Reviewed-By: Michaël Zasso <targos@protonmail.com>

lib/internal/crypto/keys.js

@@ -178,6 +178,9 @@ function isStringOrBuffer(val) {
 }
 
 function parseKeyEncoding(enc, keyType, isPublic, objName) {
+  if (enc === null || typeof enc !== 'object')
+    throw new ERR_INVALID_ARG_TYPE('options', 'object', enc);
+
   const isInput = keyType === undefined;
 
   const {

test/parallel/test-crypto-key-objects.js

@@ -107,6 +107,16 @@ const privatePem = fixtures.readSync('test_rsa_privkey.pem', 'ascii');
   assert.strictEqual(derivedPublicKey.asymmetricKeyType, 'rsa');
   assert.strictEqual(derivedPublicKey.symmetricKeySize, undefined);
 
+  // Test exporting with an invalid options object, this should throw.
+  for (const opt of [undefined, null, 'foo', 0, NaN]) {
+    common.expectsError(() => publicKey.export(opt), {
+      type: TypeError,
+      code: 'ERR_INVALID_ARG_TYPE',
+      message: 'The "options" argument must be of type object. Received type ' +
+               typeof opt
+    });
+  }
+
   const publicDER = publicKey.export({
     format: 'der',
     type: 'pkcs1'