Verify default named groups before using them with native SSL impleme… (

#14441)

…… (#14434)

…ntation

Motivation:

We should verify that the default named groups are actually supported by
our native SSL implementation. This might not always be the case as for
example when FIPS is used.

Modifications:

- Verify that default named groups are supported
- Fail creation of ReferenceCountedOpenSslContext if setting of groups
fails and also include details about why it failed if possible

Result:

Easier to debug miss-configuration of groups and make things work out of
the box even if FIPS is used. Related to
netty/netty-tcnative#883

handler/src/main/java/io/netty5/handler/ssl/OpenSsl.java

@@ -36,6 +36,7 @@
 import java.util.Collection;
 import java.util.Collections;
 import java.util.HashSet;
+import java.util.Iterator;
 import java.util.LinkedHashSet;
 import java.util.List;
 import java.util.Set;
@@ -149,9 +150,9 @@ public final class OpenSsl {
             boolean supportsKeyManagerFactory = false;
             boolean tlsv13Supported = false;
             String[] namedGroups = DEFAULT_NAMED_GROUPS;
-            String[] defaultConvertedNamedGroups = new String[namedGroups.length];
+            Set<String> defaultConvertedNamedGroups = new HashSet<String>(namedGroups.length);
             for (int i = 0; i < namedGroups.length; i++) {
-                defaultConvertedNamedGroups[i] = GroupsConverter.toOpenSsl(namedGroups[i]);
+                defaultConvertedNamedGroups.add(GroupsConverter.toOpenSsl(namedGroups[i]));
             }
 
             IS_BORINGSSL = "BoringSSL".equals(versionString());
@@ -173,6 +174,19 @@ public final class OpenSsl {
 
             try {
                 final long sslCtx = SSLContext.make(SSL.SSL_PROTOCOL_ALL, SSL.SSL_MODE_SERVER);
+
+                // Let's filter out any group that is not supported from the default.
+                Iterator<String> defaultGroupsIter = defaultConvertedNamedGroups.iterator();
+                while (defaultGroupsIter.hasNext()) {
+                    if (!SSLContext.setCurvesList(sslCtx, defaultGroupsIter.next())) {
+                        // Not supported, let's remove it. This could for example be the case if we use
+                        // fips and the configure group is not supported when using FIPS.
+                        // See https://github.com/netty/netty-tcnative/issues/883
+                        defaultGroupsIter.remove();
+                    }
+                }
+                namedGroups = defaultConvertedNamedGroups.toArray(EmptyArrays.EMPTY_STRINGS);
+
                 long certBio = 0;
                 long keyBio = 0;
                 long cert = 0;
@@ -279,7 +293,7 @@ public final class OpenSsl {
                         }
 
                         if (supportedNamedGroups.isEmpty()) {
-                            namedGroups = defaultConvertedNamedGroups;
+                            namedGroups = defaultConvertedNamedGroups.toArray(EmptyArrays.EMPTY_STRINGS);
                             logger.info("All configured namedGroups are not supported: {}. Use default: {}.",
                                     Arrays.toString(unsupportedNamedGroups.toArray(EmptyArrays.EMPTY_STRINGS)),
                                     Arrays.toString(DEFAULT_NAMED_GROUPS));
@@ -296,7 +310,7 @@ public final class OpenSsl {
                             namedGroups =  supportedConvertedNamedGroups.toArray(EmptyArrays.EMPTY_STRINGS);
                         }
                     } else {
-                        namedGroups = defaultConvertedNamedGroups;
+                        namedGroups = defaultConvertedNamedGroups.toArray(EmptyArrays.EMPTY_STRINGS);
                     }
                 } finally {
                     SSLContext.free(sslCtx);

handler/src/main/java/io/netty5/handler/ssl/ReferenceCountedOpenSslContext.java

@@ -59,6 +59,7 @@
 import java.security.cert.CertificateRevokedException;
 import java.security.cert.X509Certificate;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collections;
 import java.util.LinkedHashSet;
 import java.util.List;
@@ -432,8 +433,17 @@ public ApplicationProtocolConfig.SelectedListenerFailureBehavior selectedListene
             if (maxCertificateList != null) {
                 SSLContext.setMaxCertList(ctx, maxCertificateList);
             }
-            // Set the curves.
-            SSLContext.setCurvesList(ctx, OpenSsl.NAMED_GROUPS);
+
+            // Set the curves / groups if anything is configured.
+            if (OpenSsl.NAMED_GROUPS.length > 0 && !SSLContext.setCurvesList(ctx, OpenSsl.NAMED_GROUPS)) {
+                String msg = "failed to set curves / groups suite: " + Arrays.toString(OpenSsl.NAMED_GROUPS);
+                int err = SSL.getLastErrorNumber();
+                if (err != 0) {
+                    // We have some more details about why the operations failed, include these into the message.
+                    msg += ". " + SSL.getErrorString(err);
+                }
+                throw new SSLException(msg);
+            }
             success = true;
         } finally {
             if (!success) {