From 0e11d2c3f8a74d2310efa67179b50c12acb3fa42 Mon Sep 17 00:00:00 2001
From: Joe Truba <joe@bannable.net>
Date: Sun, 20 Nov 2022 00:54:32 +0000
Subject: [PATCH] [ruby/openssl] pkey/ec: check private key validity with
 OpenSSL 3

The behavior of EVP_PKEY_public_check changed between OpenSSL 1.1.1
and 3.0 so that it no longer validates the private key. Instead, private
keys can be validated through EVP_PKEY_private_check and
EVP_PKEY_pairwise_check.

[ky: simplified condition to use either EVP_PKEY_check() or
EVP_PKEY_public_check().]

https://github.com/ruby/openssl/commit/e38a63ab3d
---
 ext/openssl/ossl_pkey_ec.c                    | 22 ++++++++++++++-----
 test/openssl/fixtures/pkey/p256_too_large.pem |  5 +++++
 test/openssl/fixtures/pkey/p384_invalid.pem   |  6 +++++
 test/openssl/test_pkey_ec.rb                  |  7 ++++++
 4 files changed, 35 insertions(+), 5 deletions(-)
 create mode 100644 test/openssl/fixtures/pkey/p256_too_large.pem
 create mode 100644 test/openssl/fixtures/pkey/p384_invalid.pem

diff --git a/ext/openssl/ossl_pkey_ec.c b/ext/openssl/ossl_pkey_ec.c
index 2ae1ee32d3..a59849f719 100644
--- a/ext/openssl/ossl_pkey_ec.c
+++ b/ext/openssl/ossl_pkey_ec.c
@@ -483,16 +483,28 @@ static VALUE ossl_ec_key_check_key(VALUE self)
 #ifdef HAVE_EVP_PKEY_CHECK
     EVP_PKEY *pkey;
     EVP_PKEY_CTX *pctx;
-    int ret;
+    EC_KEY *ec;
 
     GetPKey(self, pkey);
+    GetEC(self, ec);
     pctx = EVP_PKEY_CTX_new(pkey, /* engine */NULL);
     if (!pctx)
-        ossl_raise(eDHError, "EVP_PKEY_CTX_new");
-    ret = EVP_PKEY_public_check(pctx);
+        ossl_raise(eECError, "EVP_PKEY_CTX_new");
+
+    if (EC_KEY_get0_private_key(ec) != NULL) {
+        if (EVP_PKEY_check(pctx) != 1) {
+            EVP_PKEY_CTX_free(pctx);
+            ossl_raise(eECError, "EVP_PKEY_check");
+        }
+    }
+    else {
+        if (EVP_PKEY_public_check(pctx) != 1) {
+            EVP_PKEY_CTX_free(pctx);
+            ossl_raise(eECError, "EVP_PKEY_public_check");
+        }
+    }
+
     EVP_PKEY_CTX_free(pctx);
-    if (ret != 1)
-        ossl_raise(eECError, "EVP_PKEY_public_check");
 #else
     EC_KEY *ec;
 
diff --git a/test/openssl/fixtures/pkey/p256_too_large.pem b/test/openssl/fixtures/pkey/p256_too_large.pem
new file mode 100644
index 0000000000..a73ac37f87
--- /dev/null
+++ b/test/openssl/fixtures/pkey/p256_too_large.pem
@@ -0,0 +1,5 @@
+-----BEGIN EC PRIVATE KEY-----
+MHcCAQEEIP+TT0V8Fndsnacji9tyf6hmhHywcOWTee9XkiBeJoVloAoGCCqGSM49
+AwEHoUQDQgAEBkhhJIU/2/YdPSlY2I1k25xjK4trr5OXSgXvBC21PtY0HQ7lor7A
+jzT0giJITqmcd81fwGw5+96zLcdxTF1hVQ==
+-----END EC PRIVATE KEY-----
diff --git a/test/openssl/fixtures/pkey/p384_invalid.pem b/test/openssl/fixtures/pkey/p384_invalid.pem
new file mode 100644
index 0000000000..d5cdc9a3af
--- /dev/null
+++ b/test/openssl/fixtures/pkey/p384_invalid.pem
@@ -0,0 +1,6 @@
+-----BEGIN EC PRIVATE KEY-----
+MIGkAgEBBDDA1Tm0m7YhkfeVpFuarAJYVlHp2tQj+1fOBiLa10t9E8TiQO/hVfxB
+vGaVEQwOheWgBwYFK4EEACKhZANiAASyGqmryZGqdpsq5gEDIfNvgC3AwSJxiBCL
+XKHBTFRp+tCezLDOK/6V8KK/vVGBJlGFW6/I7ahyXprxS7xs7hPA9iz5YiuqXlu+
+lbrIpZOz7b73hyQQCkvbBO/Avg+hPAk=
+-----END EC PRIVATE KEY-----
diff --git a/test/openssl/test_pkey_ec.rb b/test/openssl/test_pkey_ec.rb
index 451bab0321..e5fef940a6 100644
--- a/test/openssl/test_pkey_ec.rb
+++ b/test/openssl/test_pkey_ec.rb
@@ -90,6 +90,13 @@ class OpenSSL::TestEC < OpenSSL::PKeyTestCase
     assert_equal(true, key2.public?)
     assert_equal(true, key2.check_key)
 
+    # Behavior of EVP_PKEY_public_check changes between OpenSSL 1.1.1 and 3.0
+    key4 = Fixtures.pkey("p256_too_large")
+    assert_raise(OpenSSL::PKey::ECError) { key4.check_key }
+
+    key5 = Fixtures.pkey("p384_invalid")
+    assert_raise(OpenSSL::PKey::ECError) { key5.check_key }
+
     # EC#private_key= is deprecated in 3.0 and won't work on OpenSSL 3.0
     if !openssl?(3, 0, 0)
       key2.private_key += 1