[release/8.0-staging] Re-try loading ENGINE keys with a non-NULL UI_METHOD

Co-authored-by: Kevin Jones <[email protected]>

Author: github-actions[bot]

src/native/libs/System.Security.Cryptography.Native/opensslshim.h

@@ -31,6 +31,7 @@
 #include <openssl/sha.h>
 #include <openssl/ssl.h>
 #include <openssl/tls1.h>
+#include <openssl/ui.h>
 #include <openssl/x509.h>
 #include <openssl/x509v3.h>
 
@@ -607,6 +608,8 @@ EVP_PKEY *ENGINE_load_public_key(ENGINE *e, const char *key_id,
     LIGHTUP_FUNCTION(SSL_verify_client_post_handshake) \
     LIGHTUP_FUNCTION(SSL_set_post_handshake_auth) \
     REQUIRED_FUNCTION(SSL_version) \
+    REQUIRED_FUNCTION(UI_create_method) \
+    REQUIRED_FUNCTION(UI_destroy_method) \
     FALLBACK_FUNCTION(X509_check_host) \
     REQUIRED_FUNCTION(X509_check_purpose) \
     REQUIRED_FUNCTION(X509_cmp_current_time) \
@@ -1124,6 +1127,8 @@ FOR_ALL_OPENSSL_FUNCTIONS
 #define SSL_set_post_handshake_auth SSL_set_post_handshake_auth_ptr
 #define SSL_version SSL_version_ptr
 #define TLS_method TLS_method_ptr
+#define UI_create_method UI_create_method_ptr
+#define UI_destroy_method UI_destroy_method_ptr
 #define X509_check_host X509_check_host_ptr
 #define X509_check_purpose X509_check_purpose_ptr
 #define X509_cmp_current_time X509_cmp_current_time_ptr

src/native/libs/System.Security.Cryptography.Native/pal_evp_pkey.c

@@ -526,6 +526,7 @@ static EVP_PKEY* LoadKeyFromEngine(
         *haveEngine = 1;
         EVP_PKEY* ret = NULL;
         ENGINE* engine = NULL;
+        UI_METHOD* ui = NULL;
 
         // Per https://github.com/openssl/openssl/discussions/21427
         // using EVP_PKEY after freeing ENGINE is correct.
@@ -537,12 +538,30 @@ static EVP_PKEY* LoadKeyFromEngine(
             {
                 ret = load_func(engine, keyName, NULL, NULL);
 
+                if (ret == NULL)
+                {
+                    // Some engines do not tolerate having NULL passed to the ui_method parameter.
+                    // We re-try with a non-NULL UI_METHOD.
+                    ERR_clear_error();
+                    ui = UI_create_method(".NET NULL UI");
+
+                    if (ui)
+                    {
+                        ret = load_func(engine, keyName, ui, NULL);
+                    }
+                }
+
                 ENGINE_finish(engine);
             }
 
             ENGINE_free(engine);
         }
 
+        if (ui)
+        {
+            UI_destroy_method(ui);
+        }
+
         return ret;
     }