From ebd9bd8b30da55ac5f3890b9aef811128d39d0cf Mon Sep 17 00:00:00 2001 From: Apurva Telang Date: Fri, 26 Jul 2024 15:40:30 -0700 Subject: [PATCH] pip-337-impl --- .../pulsar/broker/ServiceConfiguration.java | 19 + .../jetty/tls/JettySslContextFactory.java | 58 +-- .../jetty/tls/JettySslContextFactoryTest.java | 94 +++-- ...ettySslContextFactoryWithKeyStoreTest.java | 82 +++- .../apache/pulsar/broker/PulsarService.java | 10 +- .../broker/namespace/NamespaceService.java | 4 +- .../pulsar/broker/service/BrokerService.java | 30 +- .../service/PulsarChannelInitializer.java | 90 ++--- .../apache/pulsar/broker/web/WebService.java | 78 ++-- .../pulsar/compaction/CompactorTool.java | 4 +- .../pulsar/broker/PulsarServiceTest.java | 3 + .../pulsar/broker/admin/TopicsTest.java | 3 + .../broker/loadbalance/LoadBalancerTest.java | 12 + .../loadbalance/SimpleBrokerStartTest.java | 14 + .../SimpleLoadManagerImplTest.java | 15 + .../ExtensibleLoadManagerImplBaseTest.java | 10 + .../impl/BundleSplitterTaskTest.java | 11 + .../impl/ModularLoadManagerImplTest.java | 18 + .../broker/service/BrokerServiceTest.java | 4 +- .../service/BrokerServiceThrottlingTest.java | 2 +- ...econnectZKClientPulsarServiceBaseTest.java | 10 + .../broker/service/ClusterMigrationTest.java | 3 + ...licationWithConfigurationSyncTestBase.java | 10 + .../broker/service/NetworkErrorTestBase.java | 10 + .../api/InjectedClientCnxClientBuilder.java | 2 +- .../client/api/TlsProducerConsumerTest.java | 6 +- .../client/impl/ConnectionPoolTest.java | 31 +- .../pulsar/client/impl/PulsarTestClient.java | 2 +- .../pulsar/compaction/CompactorToolTest.java | 1 + .../security/MockedPulsarStandalone.java | 3 + .../client/admin/PulsarAdminBuilder.java | 14 + .../common/policies/data/ClusterData.java | 8 + .../internal/PulsarAdminBuilderImpl.java | 14 + .../internal/http/AsyncHttpConnector.java | 126 +++--- .../pulsar/client/api/ClientBuilder.java | 35 +- .../apache/pulsar/admin/cli/CmdClusters.java | 14 + .../pulsar/admin/cli/PulsarAdminTool.java | 8 +- .../pulsar/client/cli/PulsarClientTool.java | 7 + .../pulsar/client/impl/ClientBuilderImpl.java | 23 ++ .../pulsar/client/impl/ConnectionPool.java | 19 +- .../apache/pulsar/client/impl/HttpClient.java | 123 +++--- .../client/impl/PulsarChannelInitializer.java | 144 ++++--- .../pulsar/client/impl/PulsarClientImpl.java | 7 +- .../impl/conf/ClientConfigurationData.java | 14 + ...a => PulsarHttpAsyncSslEngineFactory.java} | 31 +- .../client/impl/ClientInitializationTest.java | 6 +- .../client/impl/PulsarClientImplTest.java | 4 +- .../common/policies/data/ClusterDataImpl.java | 28 ++ .../common/util/DefaultPulsarSslFactory.java | 366 ++++++++++++++++++ .../common/util/DefaultSslContextBuilder.java | 76 ---- .../util/NettyClientSslContextRefresher.java | 96 ----- .../util/NettyServerSslContextBuilder.java | 89 ----- .../common/util/PulsarSslConfiguration.java | 167 ++++++++ .../pulsar/common/util/PulsarSslFactory.java | 106 +++++ .../util/SslContextAutoRefreshBuilder.java | 100 ----- .../keystoretls/NetSslContextBuilder.java | 90 ----- .../NettySSLContextAutoRefreshBuilder.java | 155 -------- .../util/DefaultPulsarSslFactoryTest.java | 282 ++++++++++++++ .../util/FileModifiedTimeUpdaterTest.java | 24 +- .../common/util/netty/SslContextTest.java | 109 ++++-- .../functions/worker/rest/WorkerServer.java | 86 ++-- .../proxy/server/AdminProxyHandler.java | 127 ++++-- .../proxy/server/DirectProxyHandler.java | 82 ++-- .../proxy/server/ProxyConfiguration.java | 22 ++ .../pulsar/proxy/server/ProxyConnection.java | 2 +- .../server/ServiceChannelInitializer.java | 98 ++--- .../apache/pulsar/proxy/server/WebServer.java | 83 ++-- .../proxy/server/AdminProxyHandlerTest.java | 2 +- .../pulsar/proxy/server/ProxyParserTest.java | 2 +- .../server/ProxyServiceTlsStarterTest.java | 2 + .../apache/pulsar/proxy/server/ProxyTest.java | 2 +- .../pulsar/testclient/PerfClientUtils.java | 10 + .../testclient/PerformanceBaseArguments.java | 9 + .../pulsar/websocket/service/ProxyServer.java | 84 ++-- 74 files changed, 2231 insertions(+), 1304 deletions(-) rename pulsar-client/src/main/java/org/apache/pulsar/client/util/{WithSNISslEngineFactory.java => PulsarHttpAsyncSslEngineFactory.java} (53%) create mode 100644 pulsar-common/src/main/java/org/apache/pulsar/common/util/DefaultPulsarSslFactory.java delete mode 100644 pulsar-common/src/main/java/org/apache/pulsar/common/util/DefaultSslContextBuilder.java delete mode 100644 pulsar-common/src/main/java/org/apache/pulsar/common/util/NettyClientSslContextRefresher.java delete mode 100644 pulsar-common/src/main/java/org/apache/pulsar/common/util/NettyServerSslContextBuilder.java create mode 100644 pulsar-common/src/main/java/org/apache/pulsar/common/util/PulsarSslConfiguration.java create mode 100644 pulsar-common/src/main/java/org/apache/pulsar/common/util/PulsarSslFactory.java delete mode 100644 pulsar-common/src/main/java/org/apache/pulsar/common/util/SslContextAutoRefreshBuilder.java delete mode 100644 pulsar-common/src/main/java/org/apache/pulsar/common/util/keystoretls/NetSslContextBuilder.java delete mode 100644 pulsar-common/src/main/java/org/apache/pulsar/common/util/keystoretls/NettySSLContextAutoRefreshBuilder.java create mode 100644 pulsar-common/src/test/java/org/apache/pulsar/common/util/DefaultPulsarSslFactoryTest.java diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java index 26b2f99abf5451..20addc3924bf3b 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java @@ -49,6 +49,7 @@ import org.apache.pulsar.common.policies.data.TopicType; import org.apache.pulsar.common.protocol.Commands; import org.apache.pulsar.common.sasl.SaslConstants; +import org.apache.pulsar.common.util.DefaultPulsarSslFactory; import org.apache.pulsar.common.util.DirectMemoryUtils; import org.apache.pulsar.metadata.api.MetadataStoreFactory; import org.apache.pulsar.metadata.impl.ZKMetadataStore; @@ -1581,6 +1582,15 @@ The max allowed delay for delayed delivery (in milliseconds). If the broker rece doc = "Specify whether Client certificates are required for TLS Reject.\n" + "the Connection if the Client Certificate is not trusted") private boolean tlsRequireTrustedClientCertOnConnect = false; + @FieldContext( + category = CATEGORY_TLS, + doc = "SSL Factory Plugin class to provide SSLEngine and SSLContext objects. The default " + + " class used is DefaultSslFactory.") + private String sslFactoryPlugin = DefaultPulsarSslFactory.class.getName(); + @FieldContext( + category = CATEGORY_TLS, + doc = "SSL Factory plugin configuration parameters.") + private String sslFactoryPluginParams = ""; /***** --- Authentication. --- ****/ @FieldContext( @@ -3546,6 +3556,15 @@ public double getLoadBalancerBandwidthOutResourceWeight() { + " used by the internal client to authenticate with Pulsar brokers" ) private Set brokerClientTlsProtocols = new TreeSet<>(); + @FieldContext( + category = CATEGORY_TLS, + doc = "SSL Factory Plugin class used by internal client to provide SSLEngine and SSLContext objects. " + + "The default class used is DefaultSslFactory.") + private String brokerClientSslFactoryPlugin = DefaultPulsarSslFactory.class.getName(); + @FieldContext( + category = CATEGORY_TLS, + doc = "SSL Factory plugin configuration parameters used by internal client.") + private String brokerClientSslFactoryPluginParams = ""; /* packages management service configurations (begin) */ diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/jetty/tls/JettySslContextFactory.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/jetty/tls/JettySslContextFactory.java index 46a86045995f99..0ac1b78ca993fc 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/jetty/tls/JettySslContextFactory.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/jetty/tls/JettySslContextFactory.java @@ -21,10 +21,8 @@ import java.util.Set; import javax.net.ssl.SSLContext; import lombok.extern.slf4j.Slf4j; -import org.apache.pulsar.common.util.DefaultSslContextBuilder; +import org.apache.pulsar.common.util.PulsarSslFactory; import org.apache.pulsar.common.util.SecurityUtility; -import org.apache.pulsar.common.util.SslContextAutoRefreshBuilder; -import org.apache.pulsar.common.util.keystoretls.NetSslContextBuilder; import org.eclipse.jetty.util.ssl.SslContextFactory; @Slf4j @@ -35,57 +33,21 @@ public class JettySslContextFactory { } } - public static SslContextFactory.Server createServerSslContextWithKeystore(String sslProviderString, - String keyStoreTypeString, - String keyStore, - String keyStorePassword, - boolean allowInsecureConnection, - String trustStoreTypeString, - String trustStore, - String trustStorePassword, - boolean requireTrustedClientCertOnConnect, - Set ciphers, - Set protocols, - long certRefreshInSec) { - NetSslContextBuilder sslCtxRefresher = new NetSslContextBuilder( - sslProviderString, - keyStoreTypeString, - keyStore, - keyStorePassword, - allowInsecureConnection, - trustStoreTypeString, - trustStore, - trustStorePassword, - requireTrustedClientCertOnConnect, - certRefreshInSec); - - return new JettySslContextFactory.Server(sslProviderString, sslCtxRefresher, + public static SslContextFactory.Server createSslContextFactory(String sslProviderString, + PulsarSslFactory pulsarSslFactory, + boolean requireTrustedClientCertOnConnect, + Set ciphers, Set protocols) { + return new JettySslContextFactory.Server(sslProviderString, pulsarSslFactory, requireTrustedClientCertOnConnect, ciphers, protocols); } - public static SslContextFactory createServerSslContext(String sslProviderString, boolean tlsAllowInsecureConnection, - String tlsTrustCertsFilePath, - String tlsCertificateFilePath, - String tlsKeyFilePath, - boolean tlsRequireTrustedClientCertOnConnect, - Set ciphers, - Set protocols, - long certRefreshInSec) { - DefaultSslContextBuilder sslCtxRefresher = - new DefaultSslContextBuilder(tlsAllowInsecureConnection, tlsTrustCertsFilePath, tlsCertificateFilePath, - tlsKeyFilePath, tlsRequireTrustedClientCertOnConnect, certRefreshInSec, sslProviderString); - - return new JettySslContextFactory.Server(sslProviderString, sslCtxRefresher, - tlsRequireTrustedClientCertOnConnect, ciphers, protocols); - } - private static class Server extends SslContextFactory.Server { - private final SslContextAutoRefreshBuilder sslCtxRefresher; + private final PulsarSslFactory pulsarSslFactory; - public Server(String sslProviderString, SslContextAutoRefreshBuilder sslCtxRefresher, + public Server(String sslProviderString, PulsarSslFactory pulsarSslFactory, boolean requireTrustedClientCertOnConnect, Set ciphers, Set protocols) { super(); - this.sslCtxRefresher = sslCtxRefresher; + this.pulsarSslFactory = pulsarSslFactory; if (ciphers != null && ciphers.size() > 0) { this.setIncludeCipherSuites(ciphers.toArray(new String[0])); @@ -110,7 +72,7 @@ public Server(String sslProviderString, SslContextAutoRefreshBuilder @Override public SSLContext getSslContext() { - return sslCtxRefresher.get(); + return this.pulsarSslFactory.getInternalSslContext(); } } } diff --git a/pulsar-broker-common/src/test/java/org/apache/pulsar/jetty/tls/JettySslContextFactoryTest.java b/pulsar-broker-common/src/test/java/org/apache/pulsar/jetty/tls/JettySslContextFactoryTest.java index 2f0c8b627d5815..019627f52cbcf9 100644 --- a/pulsar-broker-common/src/test/java/org/apache/pulsar/jetty/tls/JettySslContextFactoryTest.java +++ b/pulsar-broker-common/src/test/java/org/apache/pulsar/jetty/tls/JettySslContextFactoryTest.java @@ -42,6 +42,9 @@ import org.eclipse.jetty.server.ServerConnector; import org.eclipse.jetty.util.ssl.SslContextFactory; import org.testng.annotations.Test; +import org.apache.pulsar.common.util.DefaultPulsarSslFactory; +import org.apache.pulsar.common.util.PulsarSslConfiguration; +import org.apache.pulsar.common.util.PulsarSslFactory; @Slf4j public class JettySslContextFactoryTest { @@ -51,16 +54,20 @@ public void testJettyTlsServerTls() throws Exception { @Cleanup("stop") Server server = new Server(); List connectors = new ArrayList<>(); - SslContextFactory factory = JettySslContextFactory.createServerSslContext( - null, - false, - Resources.getResource("ssl/my-ca/ca.pem").getPath(), - Resources.getResource("ssl/my-ca/server-ca.pem").getPath(), - Resources.getResource("ssl/my-ca/server-key.pem").getPath(), - true, - null, - null, - 600); + PulsarSslConfiguration sslConfiguration = PulsarSslConfiguration.builder() + .tlsTrustCertsFilePath(Resources.getResource("ssl/my-ca/ca.pem").getPath()) + .tlsCertificateFilePath(Resources.getResource("ssl/my-ca/server-ca.pem").getPath()) + .tlsKeyFilePath(Resources.getResource("ssl/my-ca/server-key.pem").getPath()) + .allowInsecureConnection(false) + .requireTrustedClientCertOnConnect(true) + .tlsEnabledWithKeystore(false) + .isHttps(true) + .build(); + PulsarSslFactory sslFactory = new DefaultPulsarSslFactory(); + sslFactory.initialize(sslConfiguration); + sslFactory.createInternalSslContext(); + SslContextFactory factory = JettySslContextFactory.createSslContextFactory(null, + sslFactory, true, null, null); ServerConnector connector = new ServerConnector(server, factory); connector.setPort(0); @@ -85,20 +92,30 @@ public void testJettyTlsServerInvalidTlsProtocol() throws Exception { @Cleanup("stop") Server server = new Server(); List connectors = new ArrayList<>(); - SslContextFactory factory = JettySslContextFactory.createServerSslContext( - null, - false, - Resources.getResource("ssl/my-ca/ca.pem").getPath(), - Resources.getResource("ssl/my-ca/server-ca.pem").getPath(), - Resources.getResource("ssl/my-ca/server-key.pem").getPath(), - true, - null, + PulsarSslConfiguration sslConfiguration = PulsarSslConfiguration.builder() + .tlsProtocols(new HashSet() { + { + this.add("TLSv1.3"); + } + }) + .tlsTrustCertsFilePath(Resources.getResource("ssl/my-ca/ca.pem").getPath()) + .tlsCertificateFilePath(Resources.getResource("ssl/my-ca/server-ca.pem").getPath()) + .tlsKeyFilePath(Resources.getResource("ssl/my-ca/server-key.pem").getPath()) + .allowInsecureConnection(false) + .requireTrustedClientCertOnConnect(true) + .tlsEnabledWithKeystore(false) + .isHttps(true) + .build(); + PulsarSslFactory sslFactory = new DefaultPulsarSslFactory(); + sslFactory.initialize(sslConfiguration); + sslFactory.createInternalSslContext(); + SslContextFactory factory = JettySslContextFactory.createSslContextFactory(null, + sslFactory, true, null, new HashSet() { { this.add("TLSv1.3"); } - }, - 600); + }); factory.setHostnameVerifier((s, sslSession) -> true); ServerConnector connector = new ServerConnector(server, factory); connector.setPort(0); @@ -123,13 +140,30 @@ public void testJettyTlsServerInvalidCipher() throws Exception { @Cleanup("stop") Server server = new Server(); List connectors = new ArrayList<>(); - SslContextFactory factory = JettySslContextFactory.createServerSslContext( - null, - false, - Resources.getResource("ssl/my-ca/ca.pem").getPath(), - Resources.getResource("ssl/my-ca/server-ca.pem").getPath(), - Resources.getResource("ssl/my-ca/server-key.pem").getPath(), - true, + PulsarSslConfiguration sslConfiguration = PulsarSslConfiguration.builder() + .tlsCiphers(new HashSet() { + { + this.add("TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256"); + } + }) + .tlsProtocols(new HashSet() { + { + this.add("TLSv1.3"); + } + }) + .tlsTrustCertsFilePath(Resources.getResource("ssl/my-ca/ca.pem").getPath()) + .tlsCertificateFilePath(Resources.getResource("ssl/my-ca/server-ca.pem").getPath()) + .tlsKeyFilePath(Resources.getResource("ssl/my-ca/server-key.pem").getPath()) + .allowInsecureConnection(false) + .requireTrustedClientCertOnConnect(true) + .isHttps(true) + .tlsEnabledWithKeystore(false) + .build(); + PulsarSslFactory sslFactory = new DefaultPulsarSslFactory(); + sslFactory.initialize(sslConfiguration); + sslFactory.createInternalSslContext(); + SslContextFactory factory = JettySslContextFactory.createSslContextFactory(null, + sslFactory, true, new HashSet() { { this.add("TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256"); @@ -137,11 +171,9 @@ public void testJettyTlsServerInvalidCipher() throws Exception { }, new HashSet() { { - this.add("TLSv1.2"); + this.add("TLSv1.3"); } - }, - 600); - + }); factory.setHostnameVerifier((s, sslSession) -> true); ServerConnector connector = new ServerConnector(server, factory); connector.setPort(0); diff --git a/pulsar-broker-common/src/test/java/org/apache/pulsar/jetty/tls/JettySslContextFactoryWithKeyStoreTest.java b/pulsar-broker-common/src/test/java/org/apache/pulsar/jetty/tls/JettySslContextFactoryWithKeyStoreTest.java index f08f62c480c000..30fbc50257d4c5 100644 --- a/pulsar-broker-common/src/test/java/org/apache/pulsar/jetty/tls/JettySslContextFactoryWithKeyStoreTest.java +++ b/pulsar-broker-common/src/test/java/org/apache/pulsar/jetty/tls/JettySslContextFactoryWithKeyStoreTest.java @@ -43,6 +43,9 @@ import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; import org.apache.logging.log4j.Level; import org.apache.logging.log4j.core.config.Configurator; +import org.apache.pulsar.common.util.DefaultPulsarSslFactory; +import org.apache.pulsar.common.util.PulsarSslConfiguration; +import org.apache.pulsar.common.util.PulsarSslFactory; import org.eclipse.jetty.server.Server; import org.eclipse.jetty.server.ServerConnector; import org.eclipse.jetty.util.ssl.SslContextFactory; @@ -66,10 +69,22 @@ public void testJettyTlsServerTls() throws Exception { @Cleanup("stop") Server server = new Server(); List connectors = new ArrayList<>(); - SslContextFactory.Server factory = JettySslContextFactory.createServerSslContextWithKeystore(null, - keyStoreType, brokerKeyStorePath, keyStorePassword, false, keyStoreType, - clientTrustStorePath, keyStorePassword, true, null, - null, 600); + PulsarSslConfiguration sslConfiguration = PulsarSslConfiguration.builder() + .tlsKeyStoreType(keyStoreType) + .tlsKeyStorePath(brokerKeyStorePath) + .tlsKeyStorePassword(keyStorePassword) + .tlsTrustStoreType(keyStoreType) + .tlsTrustStorePath(clientTrustStorePath) + .tlsTrustStorePassword(keyStorePassword) + .requireTrustedClientCertOnConnect(true) + .tlsEnabledWithKeystore(true) + .isHttps(true) + .build(); + PulsarSslFactory sslFactory = new DefaultPulsarSslFactory(); + sslFactory.initialize(sslConfiguration); + sslFactory.createInternalSslContext(); + SslContextFactory.Server factory = JettySslContextFactory.createSslContextFactory(null, + sslFactory, true, null, null); factory.setHostnameVerifier((s, sslSession) -> true); ServerConnector connector = new ServerConnector(server, factory); connector.setPort(0); @@ -95,14 +110,32 @@ public void testJettyTlsServerInvalidTlsProtocol() throws Exception { @Cleanup("stop") Server server = new Server(); List connectors = new ArrayList<>(); - SslContextFactory.Server factory = JettySslContextFactory.createServerSslContextWithKeystore(null, - keyStoreType, brokerKeyStorePath, keyStorePassword, false, keyStoreType, clientTrustStorePath, - keyStorePassword, true, null, + PulsarSslConfiguration sslConfiguration = PulsarSslConfiguration.builder() + .tlsKeyStoreType(keyStoreType) + .tlsKeyStorePath(brokerKeyStorePath) + .tlsKeyStorePassword(keyStorePassword) + .tlsTrustStoreType(keyStoreType) + .tlsTrustStorePath(clientTrustStorePath) + .tlsTrustStorePassword(keyStorePassword) + .tlsProtocols(new HashSet() { + { + this.add("TLSv1.3"); + } + }) + .requireTrustedClientCertOnConnect(true) + .tlsEnabledWithKeystore(true) + .isHttps(true) + .build(); + PulsarSslFactory sslFactory = new DefaultPulsarSslFactory(); + sslFactory.initialize(sslConfiguration); + sslFactory.createInternalSslContext(); + SslContextFactory.Server factory = JettySslContextFactory.createSslContextFactory(null, + sslFactory, true, null, new HashSet() { { this.add("TLSv1.3"); } - }, 600); + }); factory.setHostnameVerifier((s, sslSession) -> true); ServerConnector connector = new ServerConnector(server, factory); connector.setPort(0); @@ -127,9 +160,33 @@ public void testJettyTlsServerInvalidCipher() throws Exception { @Cleanup("stop") Server server = new Server(); List connectors = new ArrayList<>(); - SslContextFactory.Server factory = JettySslContextFactory.createServerSslContextWithKeystore(null, - keyStoreType, brokerKeyStorePath, keyStorePassword, false, keyStoreType, clientTrustStorePath, - keyStorePassword, true, new HashSet() { + PulsarSslConfiguration sslConfiguration = PulsarSslConfiguration.builder() + .tlsKeyStoreType(keyStoreType) + .tlsKeyStorePath(brokerKeyStorePath) + .tlsKeyStorePassword(keyStorePassword) + .tlsTrustStoreType(keyStoreType) + .tlsTrustStorePath(clientTrustStorePath) + .tlsTrustStorePassword(keyStorePassword) + .tlsCiphers(new HashSet() { + { + this.add("TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256"); + } + }) + .tlsProtocols(new HashSet() { + { + this.add("TLSv1.3"); + } + }) + .requireTrustedClientCertOnConnect(true) + .tlsEnabledWithKeystore(true) + .isHttps(true) + .build(); + PulsarSslFactory sslFactory = new DefaultPulsarSslFactory(); + sslFactory.initialize(sslConfiguration); + sslFactory.createInternalSslContext(); + SslContextFactory.Server factory = JettySslContextFactory.createSslContextFactory(null, + sslFactory, true, + new HashSet() { { this.add("TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256"); } @@ -137,8 +194,9 @@ public void testJettyTlsServerInvalidCipher() throws Exception { new HashSet() { { this.add("TLSv1.2"); + this.add("TLSv1.3"); } - }, 600); + }); factory.setHostnameVerifier((s, sslSession) -> true); ServerConnector connector = new ServerConnector(server, factory); connector.setPort(0); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java index 3d57a3bc010426..9e147517ac724f 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java @@ -1689,6 +1689,8 @@ public synchronized PulsarClient getClient() throws PulsarServerException { conf.setTlsProtocols(this.getConfiguration().getBrokerClientTlsProtocols()); conf.setTlsAllowInsecureConnection(this.getConfiguration().isTlsAllowInsecureConnection()); conf.setTlsHostnameVerificationEnable(this.getConfiguration().isTlsHostnameVerificationEnabled()); + conf.setSslFactoryPlugin(this.getConfiguration().getBrokerClientSslFactoryPlugin()); + conf.setSslFactoryPluginParams(this.getConfiguration().getBrokerClientSslFactoryPluginParams()); if (this.getConfiguration().isBrokerClientTlsEnabledWithKeyStore()) { conf.setUseKeyStoreTls(true); conf.setTlsTrustStoreType(this.getConfiguration().getBrokerClientTlsTrustStoreType()); @@ -1739,15 +1741,17 @@ public synchronized PulsarAdmin getAdminClient() throws PulsarServerException { // Apply all arbitrary configuration. This must be called before setting any fields annotated as // @Secret on the ClientConfigurationData object because of the way they are serialized. // See https://github.com/apache/pulsar/issues/8509 for more information. - builder.loadConf(PropertiesUtils.filterAndMapProperties(config.getProperties(), "brokerClient_")); + builder.loadConf(PropertiesUtils.filterAndMapProperties(conf.getProperties(), "brokerClient_")); builder.authentication( conf.getBrokerClientAuthenticationPlugin(), conf.getBrokerClientAuthenticationParameters()); if (conf.isBrokerClientTlsEnabled()) { - builder.tlsCiphers(config.getBrokerClientTlsCiphers()) - .tlsProtocols(config.getBrokerClientTlsProtocols()); + builder.tlsCiphers(conf.getBrokerClientTlsCiphers()) + .tlsProtocols(conf.getBrokerClientTlsProtocols()) + .sslFactoryPlugin(conf.getBrokerClientSslFactoryPlugin()) + .sslFactoryPluginParams(conf.getBrokerClientSslFactoryPluginParams()); if (conf.isBrokerClientTlsEnabledWithKeyStore()) { builder.useKeyStoreTls(true).tlsTrustStoreType(conf.getBrokerClientTlsTrustStoreType()) .tlsTrustStorePath(conf.getBrokerClientTlsTrustStore()) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java index f3fb17c02fcee4..92188f5e6eeee5 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java @@ -1677,7 +1677,9 @@ public PulsarClientImpl getNamespaceClient(ClusterDataImpl cluster) { .enableTls(true) .tlsTrustCertsFilePath(pulsar.getConfiguration().getBrokerClientTrustCertsFilePath()) .allowTlsInsecureConnection(pulsar.getConfiguration().isTlsAllowInsecureConnection()) - .enableTlsHostnameVerification(pulsar.getConfiguration().isTlsHostnameVerificationEnabled()); + .enableTlsHostnameVerification(pulsar.getConfiguration().isTlsHostnameVerificationEnabled()) + .sslFactoryPlugin(pulsar.getConfiguration().getBrokerClientSslFactoryPlugin()) + .sslFactoryPluginParams(pulsar.getConfiguration().getBrokerClientSslFactoryPluginParams()); } else { clientBuilder.serviceUrl(isNotBlank(cluster.getBrokerServiceUrl()) ? cluster.getBrokerServiceUrl() : cluster.getServiceUrl()); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java index 8460fe23ac3b79..37302a9aef3b05 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java @@ -1389,7 +1389,9 @@ public PulsarClient getReplicationClient(String cluster, Optional c data.getBrokerClientTrustCertsFilePath(), data.getBrokerClientKeyFilePath(), data.getBrokerClientCertificateFilePath(), - pulsar.getConfiguration().isTlsHostnameVerificationEnabled() + pulsar.getConfiguration().isTlsHostnameVerificationEnabled(), + data.getBrokerClientSslFactoryPlugin(), + data.getBrokerClientSslFactoryPluginParams() ); } else if (pulsar.getConfiguration().isBrokerClientTlsEnabled()) { configTlsSettings(clientBuilder, serviceUrlTls, @@ -1404,7 +1406,9 @@ public PulsarClient getReplicationClient(String cluster, Optional c pulsar.getConfiguration().getBrokerClientTrustCertsFilePath(), pulsar.getConfiguration().getBrokerClientKeyFilePath(), pulsar.getConfiguration().getBrokerClientCertificateFilePath(), - pulsar.getConfiguration().isTlsHostnameVerificationEnabled() + pulsar.getConfiguration().isTlsHostnameVerificationEnabled(), + pulsar.getConfiguration().getBrokerClientSslFactoryPlugin(), + pulsar.getConfiguration().getBrokerClientSslFactoryPluginParams() ); } else { clientBuilder.serviceUrl( @@ -1435,11 +1439,16 @@ private void configTlsSettings(ClientBuilder clientBuilder, String serviceUrl, String brokerClientTlsKeyStore, String brokerClientTlsKeyStorePassword, String brokerClientTrustCertsFilePath, String brokerClientKeyFilePath, String brokerClientCertificateFilePath, - boolean isTlsHostnameVerificationEnabled) { + boolean isTlsHostnameVerificationEnabled, String brokerClientSslFactoryPlugin, + String brokerClientSslFactoryPluginParams) { clientBuilder .serviceUrl(serviceUrl) .allowTlsInsecureConnection(isTlsAllowInsecureConnection) .enableTlsHostnameVerification(isTlsHostnameVerificationEnabled); + if (StringUtils.isNotBlank(brokerClientSslFactoryPlugin)) { + clientBuilder.sslFactoryPlugin(brokerClientSslFactoryPlugin) + .sslFactoryPluginParams(brokerClientSslFactoryPluginParams); + } if (brokerClientTlsEnabledWithKeyStore) { clientBuilder.useKeyStoreTls(true) .tlsTrustStoreType(brokerClientTlsTrustStoreType) @@ -1462,7 +1471,8 @@ private void configAdminTlsSettings(PulsarAdminBuilder adminBuilder, boolean bro String brokerClientTlsKeyStore, String brokerClientTlsKeyStorePassword, String brokerClientTrustCertsFilePath, String brokerClientKeyFilePath, String brokerClientCertificateFilePath, - boolean isTlsHostnameVerificationEnabled) { + boolean isTlsHostnameVerificationEnabled, String brokerClientSslFactoryPlugin, + String brokerClientSslFactoryPluginParams) { if (brokerClientTlsEnabledWithKeyStore) { adminBuilder.useKeyStoreTls(true) .tlsTrustStoreType(brokerClientTlsTrustStoreType) @@ -1477,7 +1487,9 @@ private void configAdminTlsSettings(PulsarAdminBuilder adminBuilder, boolean bro .tlsCertificateFilePath(brokerClientCertificateFilePath); } adminBuilder.allowTlsInsecureConnection(isTlsAllowInsecureConnection) - .enableTlsHostnameVerification(isTlsHostnameVerificationEnabled); + .enableTlsHostnameVerification(isTlsHostnameVerificationEnabled) + .sslFactoryPlugin(brokerClientSslFactoryPlugin) + .sslFactoryPluginParams(brokerClientSslFactoryPluginParams); } public PulsarAdmin getClusterPulsarAdmin(String cluster, Optional clusterDataOp) { @@ -1524,7 +1536,9 @@ public PulsarAdmin getClusterPulsarAdmin(String cluster, Optional c data.getBrokerClientTrustCertsFilePath(), data.getBrokerClientKeyFilePath(), data.getBrokerClientCertificateFilePath(), - pulsar.getConfiguration().isTlsHostnameVerificationEnabled() + pulsar.getConfiguration().isTlsHostnameVerificationEnabled(), + data.getBrokerClientSslFactoryPlugin(), + data.getBrokerClientSslFactoryPluginParams() ); } else if (conf.isBrokerClientTlsEnabled()) { configAdminTlsSettings(builder, @@ -1539,7 +1553,9 @@ public PulsarAdmin getClusterPulsarAdmin(String cluster, Optional c conf.getBrokerClientTrustCertsFilePath(), conf.getBrokerClientKeyFilePath(), conf.getBrokerClientCertificateFilePath(), - pulsar.getConfiguration().isTlsHostnameVerificationEnabled() + pulsar.getConfiguration().isTlsHostnameVerificationEnabled(), + conf.getBrokerClientSslFactoryPlugin(), + conf.getBrokerClientSslFactoryPluginParams() ); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PulsarChannelInitializer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PulsarChannelInitializer.java index f15f6d67766f1a..a7be060f7754d1 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PulsarChannelInitializer.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/PulsarChannelInitializer.java @@ -25,9 +25,8 @@ import io.netty.handler.codec.LengthFieldBasedFrameDecoder; import io.netty.handler.flow.FlowControlHandler; import io.netty.handler.flush.FlushConsolidationHandler; -import io.netty.handler.ssl.SslContext; import io.netty.handler.ssl.SslHandler; -import io.netty.handler.ssl.SslProvider; +import java.util.concurrent.TimeUnit; import lombok.Builder; import lombok.Data; import lombok.extern.slf4j.Slf4j; @@ -36,9 +35,8 @@ import org.apache.pulsar.common.protocol.ByteBufPair; import org.apache.pulsar.common.protocol.Commands; import org.apache.pulsar.common.protocol.OptionalProxyProtocolDecoder; -import org.apache.pulsar.common.util.NettyServerSslContextBuilder; -import org.apache.pulsar.common.util.SslContextAutoRefreshBuilder; -import org.apache.pulsar.common.util.keystoretls.NettySSLContextAutoRefreshBuilder; +import org.apache.pulsar.common.util.PulsarSslConfiguration; +import org.apache.pulsar.common.util.PulsarSslFactory; @Slf4j public class PulsarChannelInitializer extends ChannelInitializer { @@ -48,10 +46,8 @@ public class PulsarChannelInitializer extends ChannelInitializer private final PulsarService pulsar; private final String listenerName; private final boolean enableTls; - private final boolean tlsEnabledWithKeyStore; - private SslContextAutoRefreshBuilder sslCtxRefresher; private final ServiceConfiguration brokerConf; - private NettySSLContextAutoRefreshBuilder nettySSLContextAutoRefreshBuilder; + private PulsarSslFactory sslFactory; /** * @param pulsar @@ -65,40 +61,16 @@ public PulsarChannelInitializer(PulsarService pulsar, PulsarChannelOptions opts) this.listenerName = opts.getListenerName(); this.enableTls = opts.isEnableTLS(); ServiceConfiguration serviceConfig = pulsar.getConfiguration(); - this.tlsEnabledWithKeyStore = serviceConfig.isTlsEnabledWithKeyStore(); if (this.enableTls) { - if (tlsEnabledWithKeyStore) { - nettySSLContextAutoRefreshBuilder = new NettySSLContextAutoRefreshBuilder( - serviceConfig.getTlsProvider(), - serviceConfig.getTlsKeyStoreType(), - serviceConfig.getTlsKeyStore(), - serviceConfig.getTlsKeyStorePassword(), - serviceConfig.isTlsAllowInsecureConnection(), - serviceConfig.getTlsTrustStoreType(), - serviceConfig.getTlsTrustStore(), - serviceConfig.getTlsTrustStorePassword(), - serviceConfig.isTlsRequireTrustedClientCertOnConnect(), - serviceConfig.getTlsCiphers(), - serviceConfig.getTlsProtocols(), - serviceConfig.getTlsCertRefreshCheckDurationSec()); - } else { - SslProvider sslProvider = null; - if (serviceConfig.getTlsProvider() != null) { - sslProvider = SslProvider.valueOf(serviceConfig.getTlsProvider()); - } - sslCtxRefresher = new NettyServerSslContextBuilder( - sslProvider, - serviceConfig.isTlsAllowInsecureConnection(), - serviceConfig.getTlsTrustCertsFilePath(), - serviceConfig.getTlsCertificateFilePath(), - serviceConfig.getTlsKeyFilePath(), - serviceConfig.getTlsCiphers(), - serviceConfig.getTlsProtocols(), - serviceConfig.isTlsRequireTrustedClientCertOnConnect(), - serviceConfig.getTlsCertRefreshCheckDurationSec()); - } - } else { - this.sslCtxRefresher = null; + PulsarSslConfiguration pulsarSslConfig = buildSslConfiguration(serviceConfig); + this.sslFactory = (PulsarSslFactory) Class.forName(serviceConfig.getSslFactoryPlugin()) + .getConstructor().newInstance(); + this.sslFactory.initialize(pulsarSslConfig); + this.sslFactory.createInternalSslContext(); + this.pulsar.getExecutor().scheduleWithFixedDelay(this::refreshSslContext, + serviceConfig.getTlsCertRefreshCheckDurationSec(), + serviceConfig.getTlsCertRefreshCheckDurationSec(), + TimeUnit.SECONDS); } this.brokerConf = pulsar.getConfiguration(); } @@ -110,12 +82,7 @@ protected void initChannel(SocketChannel ch) throws Exception { ch.config().setAutoRead(false); ch.pipeline().addLast("consolidation", new FlushConsolidationHandler(1024, true)); if (this.enableTls) { - if (this.tlsEnabledWithKeyStore) { - ch.pipeline().addLast(TLS_HANDLER, - new SslHandler(nettySSLContextAutoRefreshBuilder.get().createSSLEngine())); - } else { - ch.pipeline().addLast(TLS_HANDLER, sslCtxRefresher.get().newHandler(ch.alloc())); - } + ch.pipeline().addLast(TLS_HANDLER, new SslHandler(this.sslFactory.createServerSslEngine(ch.alloc()))); } ch.pipeline().addLast("ByteBufPairEncoder", ByteBufPair.getEncoder(this.enableTls)); @@ -161,4 +128,33 @@ public static class PulsarChannelOptions { */ private String listenerName; } + + protected PulsarSslConfiguration buildSslConfiguration(ServiceConfiguration serviceConfig) { + return PulsarSslConfiguration.builder() + .tlsKeyStoreType(serviceConfig.getTlsKeyStoreType()) + .tlsKeyStorePath(serviceConfig.getTlsKeyStore()) + .tlsKeyStorePassword(serviceConfig.getTlsKeyStorePassword()) + .tlsTrustStoreType(serviceConfig.getTlsTrustStoreType()) + .tlsTrustStorePath(serviceConfig.getTlsTrustStore()) + .tlsTrustStorePassword(serviceConfig.getTlsTrustStorePassword()) + .tlsCiphers(serviceConfig.getTlsCiphers()) + .tlsProtocols(serviceConfig.getTlsProtocols()) + .tlsTrustCertsFilePath(serviceConfig.getTlsTrustCertsFilePath()) + .tlsCertificateFilePath(serviceConfig.getTlsCertificateFilePath()) + .tlsKeyFilePath(serviceConfig.getTlsKeyFilePath()) + .allowInsecureConnection(serviceConfig.isTlsAllowInsecureConnection()) + .requireTrustedClientCertOnConnect(serviceConfig.isTlsRequireTrustedClientCertOnConnect()) + .tlsEnabledWithKeystore(serviceConfig.isTlsEnabledWithKeyStore()) + .tlsCustomParams(serviceConfig.getSslFactoryPluginParams()) + .serverMode(true) + .build(); + } + + protected void refreshSslContext() { + try { + this.sslFactory.update(); + } catch (Exception e) { + log.error("Failed to refresh SSL context", e); + } + } } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/WebService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/WebService.java index d95e88661ae8c6..ece317469e5f91 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/WebService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/web/WebService.java @@ -27,6 +27,8 @@ import java.util.Map; import java.util.Optional; import java.util.concurrent.ExecutionException; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; import javax.servlet.DispatcherType; import javax.servlet.Filter; import javax.servlet.FilterChain; @@ -39,6 +41,8 @@ import org.apache.pulsar.broker.PulsarServerException; import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.ServiceConfiguration; +import org.apache.pulsar.common.util.PulsarSslConfiguration; +import org.apache.pulsar.common.util.PulsarSslFactory; import org.apache.pulsar.jetty.tls.JettySslContextFactory; import org.eclipse.jetty.server.ConnectionFactory; import org.eclipse.jetty.server.ConnectionLimit; @@ -93,6 +97,8 @@ public class WebService implements AutoCloseable { private final ServerConnector httpsConnector; private final FilterInitializer filterInitializer; private JettyStatisticsCollector jettyStatisticsCollector; + private PulsarSslFactory sslFactory; + private ScheduledExecutorService scheduledExecutorService; @Getter private static final DynamicSkipUnknownPropertyHandler sharedUnknownPropertyHandler = @@ -144,34 +150,20 @@ public WebService(PulsarService pulsar) throws PulsarServerException { Optional tlsPort = config.getWebServicePortTls(); if (tlsPort.isPresent()) { try { - SslContextFactory sslCtxFactory; - if (config.isTlsEnabledWithKeyStore()) { - sslCtxFactory = JettySslContextFactory.createServerSslContextWithKeystore( - config.getWebServiceTlsProvider(), - config.getTlsKeyStoreType(), - config.getTlsKeyStore(), - config.getTlsKeyStorePassword(), - config.isTlsAllowInsecureConnection(), - config.getTlsTrustStoreType(), - config.getTlsTrustStore(), - config.getTlsTrustStorePassword(), - config.isTlsRequireTrustedClientCertOnConnect(), - config.getWebServiceTlsCiphers(), - config.getWebServiceTlsProtocols(), - config.getTlsCertRefreshCheckDurationSec() - ); - } else { - sslCtxFactory = JettySslContextFactory.createServerSslContext( - config.getWebServiceTlsProvider(), - config.isTlsAllowInsecureConnection(), - config.getTlsTrustCertsFilePath(), - config.getTlsCertificateFilePath(), - config.getTlsKeyFilePath(), - config.isTlsRequireTrustedClientCertOnConnect(), - config.getWebServiceTlsCiphers(), - config.getWebServiceTlsProtocols(), - config.getTlsCertRefreshCheckDurationSec()); - } + PulsarSslConfiguration sslConfiguration = buildSslConfiguration(config); + this.sslFactory = (PulsarSslFactory) Class.forName(config.getSslFactoryPlugin()) + .getConstructor().newInstance(); + this.sslFactory.initialize(sslConfiguration); + this.sslFactory.createInternalSslContext(); + this.scheduledExecutorService = this.pulsar.getExecutor(); + this.scheduledExecutorService.scheduleWithFixedDelay(this::refreshSslContext, + config.getTlsCertRefreshCheckDurationSec(), + config.getTlsCertRefreshCheckDurationSec(), + TimeUnit.SECONDS); + SslContextFactory sslCtxFactory = + JettySslContextFactory.createSslContextFactory(config.getWebServiceTlsProvider(), + this.sslFactory, config.isTlsRequireTrustedClientCertOnConnect(), + config.getTlsCiphers(), config.getTlsProtocols()); List connectionFactories = new ArrayList<>(); if (config.isWebServiceHaProxyProtocolEnabled()) { connectionFactories.add(new ProxyConnectionFactory()); @@ -455,5 +447,35 @@ public Optional getListenPortHTTPS() { } } + protected PulsarSslConfiguration buildSslConfiguration(ServiceConfiguration serviceConfig) { + return PulsarSslConfiguration.builder() + .tlsKeyStoreType(serviceConfig.getTlsKeyStoreType()) + .tlsKeyStorePath(serviceConfig.getTlsKeyStore()) + .tlsKeyStorePassword(serviceConfig.getTlsKeyStorePassword()) + .tlsTrustStoreType(serviceConfig.getTlsTrustStoreType()) + .tlsTrustStorePath(serviceConfig.getTlsTrustStore()) + .tlsTrustStorePassword(serviceConfig.getTlsTrustStorePassword()) + .tlsCiphers(serviceConfig.getTlsCiphers()) + .tlsProtocols(serviceConfig.getTlsProtocols()) + .tlsTrustCertsFilePath(serviceConfig.getTlsTrustCertsFilePath()) + .tlsCertificateFilePath(serviceConfig.getTlsCertificateFilePath()) + .tlsKeyFilePath(serviceConfig.getTlsKeyFilePath()) + .allowInsecureConnection(serviceConfig.isTlsAllowInsecureConnection()) + .requireTrustedClientCertOnConnect(serviceConfig.isTlsRequireTrustedClientCertOnConnect()) + .tlsEnabledWithKeystore(serviceConfig.isTlsEnabledWithKeyStore()) + .tlsCustomParams(serviceConfig.getSslFactoryPluginParams()) + .serverMode(true) + .isHttps(true) + .build(); + } + + protected void refreshSslContext() { + try { + this.sslFactory.update(); + } catch (Exception e) { + log.error("Failed to refresh SSL context", e); + } + } + private static final Logger log = LoggerFactory.getLogger(WebService.class); } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/compaction/CompactorTool.java b/pulsar-broker/src/main/java/org/apache/pulsar/compaction/CompactorTool.java index 7d35c2c0f7b9e8..fe77db33692b98 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/compaction/CompactorTool.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/compaction/CompactorTool.java @@ -86,7 +86,9 @@ public static PulsarClient createClient(ServiceConfiguration brokerConfig) throw if (internalListener.getBrokerServiceUrlTls() != null && brokerConfig.isBrokerClientTlsEnabled()) { clientBuilder.serviceUrl(internalListener.getBrokerServiceUrlTls().toString()) .allowTlsInsecureConnection(brokerConfig.isTlsAllowInsecureConnection()) - .enableTlsHostnameVerification(brokerConfig.isTlsHostnameVerificationEnabled()); + .enableTlsHostnameVerification(brokerConfig.isTlsHostnameVerificationEnabled()) + .sslFactoryPlugin(brokerConfig.getBrokerClientSslFactoryPlugin()) + .sslFactoryPluginParams(brokerConfig.getBrokerClientSslFactoryPluginParams()); if (brokerConfig.isBrokerClientTlsEnabledWithKeyStore()) { clientBuilder.useKeyStoreTls(true) .tlsKeyStoreType(brokerConfig.getBrokerClientTlsKeyStoreType()) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/PulsarServiceTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/PulsarServiceTest.java index 3bbf423da6ef37..dd2f9288071a56 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/PulsarServiceTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/PulsarServiceTest.java @@ -67,6 +67,9 @@ protected void doInitConf() throws Exception { conf.setWebServicePort(Optional.of(8081)); conf.setWebServicePortTls(Optional.of(8082)); } + conf.setTlsTrustCertsFilePath(CA_CERT_FILE_PATH); + conf.setTlsCertificateFilePath(BROKER_CERT_FILE_PATH); + conf.setTlsKeyFilePath(BROKER_KEY_FILE_PATH); } @Test diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/TopicsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/TopicsTest.java index c9457e1a8883fa..8940fe4a1f3c3c 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/TopicsTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/TopicsTest.java @@ -326,6 +326,9 @@ public void testLookUpWithRedirect() throws Exception { conf.setBrokerServicePortTls(Optional.of(0)); conf.setWebServicePort(Optional.of(0)); conf.setWebServicePortTls(Optional.of(0)); + conf.setTlsTrustCertsFilePath(CA_CERT_FILE_PATH); + conf.setTlsCertificateFilePath(BROKER_CERT_FILE_PATH); + conf.setTlsKeyFilePath(BROKER_KEY_FILE_PATH); @Cleanup PulsarTestContext pulsarTestContext2 = createAdditionalPulsarTestContext(conf); PulsarService pulsar2 = pulsarTestContext2.getPulsarService(); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/LoadBalancerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/LoadBalancerTest.java index 95aafd84ae406b..acd918b55fe1b3 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/LoadBalancerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/LoadBalancerTest.java @@ -63,6 +63,7 @@ import org.apache.pulsar.policies.data.loadbalancer.NamespaceBundleStats; import org.apache.pulsar.policies.data.loadbalancer.ResourceUsage; import org.apache.pulsar.policies.data.loadbalancer.SystemResourceUsage; +import org.apache.pulsar.utils.ResourceUtils; import org.apache.pulsar.zookeeper.LocalBookkeeperEnsemble; import org.apache.zookeeper.CreateMode; import org.apache.zookeeper.ZooDefs; @@ -83,6 +84,14 @@ */ @Test(groups = "broker") public class LoadBalancerTest { + + public final static String CA_CERT_FILE_PATH = + ResourceUtils.getAbsolutePath("certificate-authority/certs/ca.cert.pem"); + public final static String BROKER_CERT_FILE_PATH = + ResourceUtils.getAbsolutePath("certificate-authority/server-keys/broker.cert.pem"); + public final static String BROKER_KEY_FILE_PATH = + ResourceUtils.getAbsolutePath("certificate-authority/server-keys/broker.key-pk8.pem"); + LocalBookkeeperEnsemble bkEnsemble; private static final Logger log = LoggerFactory.getLogger(LoadBalancerTest.class); @@ -126,6 +135,9 @@ void setup() throws Exception { config.setLoadManagerClassName(SimpleLoadManagerImpl.class.getName()); config.setAdvertisedAddress(localhost+i); config.setLoadBalancerEnabled(false); + config.setTlsTrustCertsFilePath(CA_CERT_FILE_PATH); + config.setTlsCertificateFilePath(BROKER_CERT_FILE_PATH); + config.setTlsKeyFilePath(BROKER_KEY_FILE_PATH); pulsarServices[i] = new PulsarService(config); pulsarServices[i].start(); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/SimpleBrokerStartTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/SimpleBrokerStartTest.java index 28dde8b7f559d3..31c8c9f3bccc1c 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/SimpleBrokerStartTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/SimpleBrokerStartTest.java @@ -20,6 +20,7 @@ import static org.testng.Assert.assertTrue; import static org.testng.Assert.fail; +import com.google.common.io.Resources; import java.nio.file.Files; import java.nio.file.Paths; import java.util.Optional; @@ -37,6 +38,13 @@ @Test(groups = "broker") public class SimpleBrokerStartTest { + final static String caCertPath = Resources.getResource("certificate-authority/certs/ca.cert.pem") + .getPath(); + final static String brokerCertPath = + Resources.getResource("certificate-authority/server-keys/broker.cert.pem").getPath(); + final static String brokerKeyPath = + Resources.getResource("certificate-authority/server-keys/broker.key-pk8.pem").getPath(); + public void testHasNICSpeed() throws Exception { if (!LinuxInfoUtils.isLinux()) { return; @@ -57,6 +65,9 @@ public void testHasNICSpeed() throws Exception { config.setBrokerServicePortTls(Optional.of(0)); config.setWebServicePortTls(Optional.of(0)); config.setAdvertisedAddress("localhost"); + config.setTlsTrustCertsFilePath(caCertPath); + config.setTlsCertificateFilePath(brokerCertPath); + config.setTlsKeyFilePath(brokerKeyPath); boolean hasNicSpeeds = LinuxInfoUtils.checkHasNicSpeeds(); if (hasNicSpeeds) { @Cleanup @@ -85,6 +96,9 @@ public void testNoNICSpeed() throws Exception { config.setBrokerServicePortTls(Optional.of(0)); config.setWebServicePortTls(Optional.of(0)); config.setAdvertisedAddress("localhost"); + config.setTlsTrustCertsFilePath(caCertPath); + config.setTlsCertificateFilePath(brokerCertPath); + config.setTlsKeyFilePath(brokerKeyPath); boolean hasNicSpeeds = LinuxInfoUtils.checkHasNicSpeeds(); if (!hasNicSpeeds) { @Cleanup diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/SimpleLoadManagerImplTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/SimpleLoadManagerImplTest.java index acf096751d769f..1e91230559b0a8 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/SimpleLoadManagerImplTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/SimpleLoadManagerImplTest.java @@ -77,6 +77,7 @@ import org.apache.pulsar.policies.data.loadbalancer.ResourceUnitRanking; import org.apache.pulsar.policies.data.loadbalancer.ResourceUsage; import org.apache.pulsar.policies.data.loadbalancer.SystemResourceUsage; +import org.apache.pulsar.utils.ResourceUtils; import org.apache.pulsar.zookeeper.LocalBookkeeperEnsemble; import org.testng.Assert; import org.testng.annotations.AfterMethod; @@ -86,6 +87,14 @@ @Slf4j @Test(groups = "broker") public class SimpleLoadManagerImplTest { + + public final static String CA_CERT_FILE_PATH = + ResourceUtils.getAbsolutePath("certificate-authority/certs/ca.cert.pem"); + public final static String BROKER_CERT_FILE_PATH = + ResourceUtils.getAbsolutePath("certificate-authority/server-keys/broker.cert.pem"); + public final static String BROKER_KEY_FILE_PATH = + ResourceUtils.getAbsolutePath("certificate-authority/server-keys/broker.key-pk8.pem"); + LocalBookkeeperEnsemble bkEnsemble; URL url1; @@ -129,6 +138,9 @@ void setup() throws Exception { config1.setLoadManagerClassName(SimpleLoadManagerImpl.class.getName()); config1.setBrokerServicePortTls(Optional.of(0)); config1.setAdvertisedAddress("localhost"); + config1.setTlsTrustCertsFilePath(CA_CERT_FILE_PATH); + config1.setTlsCertificateFilePath(BROKER_CERT_FILE_PATH); + config1.setTlsKeyFilePath(BROKER_KEY_FILE_PATH); pulsar1 = new PulsarService(config1); pulsar1.start(); @@ -150,6 +162,9 @@ void setup() throws Exception { config2.setLoadManagerClassName(SimpleLoadManagerImpl.class.getName()); config2.setBrokerServicePortTls(Optional.of(0)); config2.setWebServicePortTls(Optional.of(0)); + config2.setTlsTrustCertsFilePath(CA_CERT_FILE_PATH); + config2.setTlsCertificateFilePath(BROKER_CERT_FILE_PATH); + config2.setTlsKeyFilePath(BROKER_KEY_FILE_PATH); config2.setAdvertisedAddress("localhost"); pulsar2 = new PulsarService(config2); pulsar2.start(); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplBaseTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplBaseTest.java index cce16061506a1a..e9fafa9c30317d 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplBaseTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/extensions/ExtensibleLoadManagerImplBaseTest.java @@ -21,6 +21,7 @@ import static org.mockito.Mockito.reset; import static org.mockito.Mockito.spy; import com.google.common.collect.Sets; +import com.google.common.io.Resources; import java.util.Optional; import java.util.concurrent.CompletableFuture; import org.apache.commons.lang3.reflect.FieldUtils; @@ -44,6 +45,12 @@ public abstract class ExtensibleLoadManagerImplBaseTest extends MockedPulsarServiceBaseTest { + final static String caCertPath = Resources.getResource("certificate-authority/certs/ca.cert.pem").getPath(); + final static String brokerCertPath = + Resources.getResource("certificate-authority/server-keys/broker.cert.pem").getPath(); + final static String brokerKeyPath = + Resources.getResource("certificate-authority/server-keys/broker.key-pk8.pem").getPath(); + protected PulsarService pulsar1; protected PulsarService pulsar2; @@ -79,6 +86,9 @@ protected ServiceConfiguration updateConfig(ServiceConfiguration conf) { conf.setLoadBalancerDebugModeEnabled(true); conf.setWebServicePortTls(Optional.of(0)); conf.setBrokerServicePortTls(Optional.of(0)); + conf.setTlsCertificateFilePath(brokerCertPath); + conf.setTlsKeyFilePath(brokerKeyPath); + conf.setTlsTrustCertsFilePath(caCertPath); return conf; } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/impl/BundleSplitterTaskTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/impl/BundleSplitterTaskTest.java index bc49352f41d218..74e692e3d7de0b 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/impl/BundleSplitterTaskTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/impl/BundleSplitterTaskTest.java @@ -28,6 +28,7 @@ import org.apache.pulsar.policies.data.loadbalancer.LocalBrokerData; import org.apache.pulsar.policies.data.loadbalancer.NamespaceBundleStats; import org.apache.pulsar.policies.data.loadbalancer.TimeAverageMessageData; +import org.apache.pulsar.utils.ResourceUtils; import org.apache.pulsar.zookeeper.LocalBookkeeperEnsemble; import org.testng.Assert; import org.testng.annotations.AfterMethod; @@ -45,6 +46,13 @@ @Test(groups = "broker") public class BundleSplitterTaskTest { + public final static String CA_CERT_FILE_PATH = + ResourceUtils.getAbsolutePath("certificate-authority/certs/ca.cert.pem"); + public final static String BROKER_CERT_FILE_PATH = + ResourceUtils.getAbsolutePath("certificate-authority/server-keys/broker.cert.pem"); + public final static String BROKER_KEY_FILE_PATH = + ResourceUtils.getAbsolutePath("certificate-authority/server-keys/broker.key-pk8.pem"); + private LocalBookkeeperEnsemble bkEnsemble; private PulsarService pulsar; @@ -67,6 +75,9 @@ void setup() throws Exception { config.setBrokerServicePort(Optional.of(0)); config.setBrokerServicePortTls(Optional.of(0)); config.setWebServicePortTls(Optional.of(0)); + config.setTlsCertificateFilePath(CA_CERT_FILE_PATH); + config.setTlsCertificateFilePath(BROKER_CERT_FILE_PATH); + config.setTlsKeyFilePath(BROKER_KEY_FILE_PATH); pulsar = new PulsarService(config); pulsar.start(); } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImplTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImplTest.java index 68bef8b241c7b1..aceeefe304b622 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImplTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/impl/ModularLoadManagerImplTest.java @@ -99,6 +99,7 @@ import org.apache.pulsar.policies.data.loadbalancer.SystemResourceUsage; import org.apache.pulsar.policies.data.loadbalancer.TimeAverageBrokerData; import org.apache.pulsar.policies.data.loadbalancer.TimeAverageMessageData; +import org.apache.pulsar.utils.ResourceUtils; import org.apache.pulsar.zookeeper.LocalBookkeeperEnsemble; import org.awaitility.Awaitility; import org.mockito.Mockito; @@ -111,6 +112,14 @@ @Slf4j @Test(groups = "broker") public class ModularLoadManagerImplTest { + + public final static String CA_CERT_FILE_PATH = + ResourceUtils.getAbsolutePath("certificate-authority/certs/ca.cert.pem"); + public final static String BROKER_CERT_FILE_PATH = + ResourceUtils.getAbsolutePath("certificate-authority/server-keys/broker.cert.pem"); + public final static String BROKER_KEY_FILE_PATH = + ResourceUtils.getAbsolutePath("certificate-authority/server-keys/broker.key-pk8.pem"); + private LocalBookkeeperEnsemble bkEnsemble; private URL url1; @@ -180,6 +189,9 @@ void setup() throws Exception { config1.setLoadBalancerOverrideBrokerNicSpeedGbps(Optional.of(1.0d)); config1.setBrokerServicePort(Optional.of(0)); config1.setBrokerServicePortTls(Optional.of(0)); + config1.setTlsTrustCertsFilePath(CA_CERT_FILE_PATH); + config1.setTlsCertificateFilePath(BROKER_CERT_FILE_PATH); + config1.setTlsKeyFilePath(BROKER_KEY_FILE_PATH); pulsar1 = new PulsarService(config1); pulsar1.start(); @@ -200,6 +212,9 @@ void setup() throws Exception { config2.setLoadBalancerOverrideBrokerNicSpeedGbps(Optional.of(1.0d)); config2.setBrokerServicePort(Optional.of(0)); config2.setBrokerServicePortTls(Optional.of(0)); + config2.setTlsTrustCertsFilePath(CA_CERT_FILE_PATH); + config2.setTlsCertificateFilePath(BROKER_CERT_FILE_PATH); + config2.setTlsKeyFilePath(BROKER_KEY_FILE_PATH); pulsar2 = new PulsarService(config2); pulsar2.start(); @@ -215,6 +230,9 @@ void setup() throws Exception { config.setLoadBalancerOverrideBrokerNicSpeedGbps(Optional.of(1.0d)); config.setBrokerServicePort(Optional.of(0)); config.setBrokerServicePortTls(Optional.of(0)); + config.setTlsTrustCertsFilePath(CA_CERT_FILE_PATH); + config.setTlsCertificateFilePath(BROKER_CERT_FILE_PATH); + config.setTlsKeyFilePath(BROKER_KEY_FILE_PATH); pulsar3 = new PulsarService(config); secondaryBrokerId = pulsar2.getBrokerId(); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerServiceTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerServiceTest.java index 172842b5ed3bfe..aa236e09da99d3 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerServiceTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerServiceTest.java @@ -748,7 +748,7 @@ public void testTlsEnabled() throws Exception { fail("should fail"); } catch (Exception e) { - assertTrue(e.getMessage().contains("General OpenSslEngine problem")); + assertTrue(e.getMessage().contains("unable to find valid certification path to requested target")); } finally { pulsarClient.close(); } @@ -1034,7 +1034,7 @@ protected void handlePartitionResponse(CommandPartitionedTopicMetadataResponse l } super.handlePartitionResponse(lookupResult); } - })) { + }, null)) { // for PMR // 2 lookup will succeed long reqId1 = reqId++; diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerServiceThrottlingTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerServiceThrottlingTest.java index c6a94833c4c62f..ddf0fae13545ee 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerServiceThrottlingTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerServiceThrottlingTest.java @@ -190,7 +190,7 @@ public void testLookupThrottlingForClientByBroker() throws Exception { EventLoopGroup eventLoop = EventLoopUtil.newEventLoopGroup(20, false, new DefaultThreadFactory("test-pool", Thread.currentThread().isDaemon())); ExecutorService executor = Executors.newFixedThreadPool(10); - try (ConnectionPool pool = new ConnectionPool(InstrumentProvider.NOOP, conf, eventLoop)) { + try (ConnectionPool pool = new ConnectionPool(InstrumentProvider.NOOP, conf, eventLoop, null)) { final int totalConsumers = 20; List> futures = new ArrayList<>(); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/CanReconnectZKClientPulsarServiceBaseTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/CanReconnectZKClientPulsarServiceBaseTest.java index a1cb4abc4c30ba..787b4d3154e906 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/CanReconnectZKClientPulsarServiceBaseTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/CanReconnectZKClientPulsarServiceBaseTest.java @@ -19,6 +19,7 @@ package org.apache.pulsar.broker.service; import com.google.common.collect.Sets; +import com.google.common.io.Resources; import io.netty.channel.Channel; import java.net.URL; import java.nio.channels.SelectionKey; @@ -45,6 +46,12 @@ public abstract class CanReconnectZKClientPulsarServiceBaseTest extends TestRetrySupport { protected final String defaultTenant = "public"; protected final String defaultNamespace = defaultTenant + "/default"; + final static String caCertPath = Resources.getResource("certificate-authority/certs/ca.cert.pem") + .getPath(); + final static String brokerCertPath = + Resources.getResource("certificate-authority/server-keys/broker.cert.pem").getPath(); + final static String brokerKeyPath = + Resources.getResource("certificate-authority/server-keys/broker.key-pk8.pem").getPath(); protected int numberOfBookies = 3; protected final String clusterName = "r1"; protected URL url; @@ -188,6 +195,9 @@ private void setConfigDefaults(ServiceConfiguration config, String clusterName, config.setAllowAutoTopicCreationType(TopicType.NON_PARTITIONED); config.setEnableReplicatedSubscriptions(true); config.setReplicatedSubscriptionsSnapshotFrequencyMillis(1000); + config.setTlsTrustCertsFilePath(caCertPath); + config.setTlsCertificateFilePath(brokerCertPath); + config.setTlsKeyFilePath(brokerKeyPath); } @Override diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ClusterMigrationTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ClusterMigrationTest.java index 20e13023cacfb5..8ec565f7d45666 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ClusterMigrationTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ClusterMigrationTest.java @@ -1104,6 +1104,9 @@ protected void doInitConf() throws Exception { this.conf.setLoadManagerClassName(loadManagerClassName); this.conf.setWebServicePortTls(Optional.of(0)); this.conf.setBrokerServicePortTls(Optional.of(0)); + this.conf.setTlsTrustCertsFilePath(CA_CERT_FILE_PATH); + this.conf.setTlsCertificateFilePath(BROKER_CERT_FILE_PATH); + this.conf.setTlsKeyFilePath(BROKER_KEY_FILE_PATH); } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/GeoReplicationWithConfigurationSyncTestBase.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/GeoReplicationWithConfigurationSyncTestBase.java index 9b4dd5192e1ec8..1362a046247d87 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/GeoReplicationWithConfigurationSyncTestBase.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/GeoReplicationWithConfigurationSyncTestBase.java @@ -19,6 +19,7 @@ package org.apache.pulsar.broker.service; import com.google.common.collect.Sets; +import com.google.common.io.Resources; import java.net.URL; import java.util.Collections; import java.util.Optional; @@ -39,6 +40,12 @@ public abstract class GeoReplicationWithConfigurationSyncTestBase extends TestRe protected final String defaultTenant = "public"; protected final String defaultNamespace = defaultTenant + "/default"; + final static String caCertPath = Resources.getResource("certificate-authority/certs/ca.cert.pem") + .getPath(); + final static String brokerCertPath = + Resources.getResource("certificate-authority/server-keys/broker.cert.pem").getPath(); + final static String brokerKeyPath = + Resources.getResource("certificate-authority/server-keys/broker.key-pk8.pem").getPath(); protected final String cluster1 = "r1"; protected URL url1; @@ -175,6 +182,9 @@ protected void setConfigDefaults(ServiceConfiguration config, String clusterName config.setEnableReplicatedSubscriptions(true); config.setReplicatedSubscriptionsSnapshotFrequencyMillis(1000); config.setLoadBalancerSheddingEnabled(false); + config.setTlsTrustCertsFilePath(caCertPath); + config.setTlsCertificateFilePath(brokerCertPath); + config.setTlsKeyFilePath(brokerKeyPath); } @Override diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/NetworkErrorTestBase.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/NetworkErrorTestBase.java index 742194d9b12a11..0161a4a63cfc62 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/NetworkErrorTestBase.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/NetworkErrorTestBase.java @@ -44,6 +44,7 @@ import org.apache.pulsar.common.policies.data.TenantInfoImpl; import org.apache.pulsar.common.policies.data.TopicType; import org.apache.pulsar.tests.TestRetrySupport; +import org.apache.pulsar.utils.ResourceUtils; import org.apache.pulsar.zookeeper.LocalBookkeeperEnsemble; import org.apache.pulsar.zookeeper.ZookeeperServerTest; import org.awaitility.reflect.WhiteboxImpl; @@ -51,6 +52,12 @@ @Slf4j public abstract class NetworkErrorTestBase extends TestRetrySupport { + protected final static String CA_CERT_FILE_PATH = + ResourceUtils.getAbsolutePath("certificate-authority/certs/ca.cert.pem"); + protected final static String BROKER_CERT_FILE_PATH = + ResourceUtils.getAbsolutePath("certificate-authority/server-keys/broker.cert.pem"); + protected final static String BROKER_KEY_FILE_PATH = + ResourceUtils.getAbsolutePath("certificate-authority/server-keys/broker.key-pk8.pem"); protected final String defaultTenant = "public"; protected final String defaultNamespace = defaultTenant + "/default"; protected final String cluster1 = "r1"; @@ -176,6 +183,9 @@ protected void setConfigDefaults(ServiceConfiguration config, String clusterName config.setForceDeleteNamespaceAllowed(true); config.setLoadManagerClassName(PreferBrokerModularLoadManager.class.getName()); config.setMetadataStoreSessionTimeoutMillis(5000); + config.setTlsTrustCertsFilePath(CA_CERT_FILE_PATH); + config.setTlsCertificateFilePath(BROKER_CERT_FILE_PATH); + config.setTlsKeyFilePath(BROKER_KEY_FILE_PATH); } @Override diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/InjectedClientCnxClientBuilder.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/InjectedClientCnxClientBuilder.java index 13447e089eab89..288bdba9b3846b 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/InjectedClientCnxClientBuilder.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/InjectedClientCnxClientBuilder.java @@ -44,7 +44,7 @@ public static PulsarClientImpl create(final ClientBuilderImpl clientBuilder, // Inject into ClientCnx. ConnectionPool pool = new ConnectionPool(InstrumentProvider.NOOP, conf, eventLoopGroup, - () -> clientCnxFactory.generate(conf, eventLoopGroup)); + () -> clientCnxFactory.generate(conf, eventLoopGroup), null); return new InjectedClientCnxPulsarClientImpl(conf, eventLoopGroup, pool); } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/TlsProducerConsumerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/TlsProducerConsumerTest.java index 879289eb65dc86..44af37ca90f511 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/TlsProducerConsumerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/TlsProducerConsumerTest.java @@ -199,6 +199,7 @@ public void testTlsCertsFromDynamicStreamExpiredAndRenewCert() throws Exception log.info("-- Starting {} test --", methodName); ClientBuilder clientBuilder = PulsarClient.builder().serviceUrl(pulsar.getBrokerServiceUrlTls()) .enableTls(true).allowTlsInsecureConnection(false) + .autoCertRefreshSeconds(1) .operationTimeout(1000, TimeUnit.MILLISECONDS); AtomicInteger certIndex = new AtomicInteger(1); AtomicInteger keyIndex = new AtomicInteger(0); @@ -223,7 +224,7 @@ public void testTlsCertsFromDynamicStreamExpiredAndRenewCert() throws Exception } catch (PulsarClientException e) { // Ok.. } - + sleepSeconds(2); certIndex.set(0); try { consumer = pulsarClient.newConsumer().topic("persistent://my-property/use/my-ns/my-topic1") @@ -232,8 +233,9 @@ public void testTlsCertsFromDynamicStreamExpiredAndRenewCert() throws Exception } catch (PulsarClientException e) { // Ok.. } - + sleepSeconds(2); trustStoreIndex.set(0); + sleepSeconds(2); consumer = pulsarClient.newConsumer().topic("persistent://my-property/use/my-ns/my-topic1") .subscriptionName("my-subscriber-name").subscribe(); consumer.close(); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/ConnectionPoolTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/ConnectionPoolTest.java index 1037019d608ab3..7767235d0367e9 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/ConnectionPoolTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/ConnectionPoolTest.java @@ -36,6 +36,7 @@ import org.apache.pulsar.client.api.PulsarClient; import org.apache.pulsar.client.impl.conf.ClientConfigurationData; import org.apache.pulsar.client.impl.metrics.InstrumentProvider; +import org.apache.pulsar.client.util.ScheduledExecutorProvider; import org.apache.pulsar.common.api.proto.CommandCloseProducer; import org.apache.pulsar.common.util.netty.EventLoopUtil; import org.awaitility.Awaitility; @@ -69,8 +70,11 @@ protected void cleanup() throws Exception { public void testSingleIpAddress() throws Exception { ClientConfigurationData conf = new ClientConfigurationData(); EventLoopGroup eventLoop = EventLoopUtil.newEventLoopGroup(1, false, new DefaultThreadFactory("test")); + ScheduledExecutorProvider executorProvider = new ScheduledExecutorProvider(conf.getNumIoThreads(), + "test-pulsar-client-scheduled"); ConnectionPool pool = - spyWithClassAndConstructorArgs(ConnectionPool.class, InstrumentProvider.NOOP, conf, eventLoop); + spyWithClassAndConstructorArgs(ConnectionPool.class, InstrumentProvider.NOOP, conf, eventLoop, + executorProvider); conf.setServiceUrl(serviceUrl); PulsarClientImpl client = new PulsarClientImpl(conf, eventLoop, pool); @@ -84,6 +88,7 @@ public void testSingleIpAddress() throws Exception { client.close(); eventLoop.shutdownGracefully(); + executorProvider.shutdownNow(); } @Test @@ -119,8 +124,11 @@ public void testSelectConnectionForSameProducer() throws Exception { @Test public void testDoubleIpAddress() throws Exception { ClientConfigurationData conf = new ClientConfigurationData(); + ScheduledExecutorProvider executorProvider = new ScheduledExecutorProvider(conf.getNumIoThreads(), + "test-pulsar-client-scheduled"); EventLoopGroup eventLoop = EventLoopUtil.newEventLoopGroup(1, false, new DefaultThreadFactory("test")); - ConnectionPool pool = spyWithClassAndConstructorArgs(ConnectionPool.class, InstrumentProvider.NOOP, conf, eventLoop); + ConnectionPool pool = spyWithClassAndConstructorArgs(ConnectionPool.class, InstrumentProvider.NOOP, conf, + eventLoop, executorProvider); conf.setServiceUrl(serviceUrl); PulsarClientImpl client = new PulsarClientImpl(conf, eventLoop, pool); @@ -138,6 +146,7 @@ public void testDoubleIpAddress() throws Exception { client.close(); eventLoop.shutdownGracefully(); + executorProvider.shutdownNow(); } @Test @@ -145,8 +154,11 @@ public void testNoConnectionPool() throws Exception { ClientConfigurationData conf = new ClientConfigurationData(); conf.setConnectionsPerBroker(0); EventLoopGroup eventLoop = EventLoopUtil.newEventLoopGroup(8, false, new DefaultThreadFactory("test")); + ScheduledExecutorProvider executorProvider = new ScheduledExecutorProvider(conf.getNumIoThreads(), + "test-pulsar-client-scheduled"); ConnectionPool pool = - spyWithClassAndConstructorArgs(ConnectionPool.class, InstrumentProvider.NOOP, conf, eventLoop); + spyWithClassAndConstructorArgs(ConnectionPool.class, InstrumentProvider.NOOP, conf, eventLoop, + executorProvider); InetSocketAddress brokerAddress = InetSocketAddress.createUnresolved("127.0.0.1", brokerPort); @@ -162,6 +174,7 @@ public void testNoConnectionPool() throws Exception { pool.closeAllConnections(); pool.close(); eventLoop.shutdownGracefully(); + executorProvider.shutdownNow(); } @Test @@ -169,8 +182,11 @@ public void testEnableConnectionPool() throws Exception { ClientConfigurationData conf = new ClientConfigurationData(); conf.setConnectionsPerBroker(5); EventLoopGroup eventLoop = EventLoopUtil.newEventLoopGroup(8, false, new DefaultThreadFactory("test")); + ScheduledExecutorProvider executorProvider = new ScheduledExecutorProvider(conf.getNumIoThreads(), + "test-pulsar-client-scheduled"); ConnectionPool pool = - spyWithClassAndConstructorArgs(ConnectionPool.class, InstrumentProvider.NOOP, conf, eventLoop); + spyWithClassAndConstructorArgs(ConnectionPool.class, InstrumentProvider.NOOP, conf, eventLoop, + executorProvider); InetSocketAddress brokerAddress = InetSocketAddress.createUnresolved("127.0.0.1", brokerPort); @@ -186,6 +202,7 @@ public void testEnableConnectionPool() throws Exception { pool.closeAllConnections(); pool.close(); eventLoop.shutdownGracefully(); + executorProvider.shutdownNow(); } @@ -193,7 +210,8 @@ public void testEnableConnectionPool() throws Exception { public void testSetProxyToTargetBrokerAddress() throws Exception { ClientConfigurationData conf = new ClientConfigurationData(); conf.setConnectionsPerBroker(1); - + ScheduledExecutorProvider executorProvider = new ScheduledExecutorProvider(conf.getNumIoThreads(), + "test-pulsar-client-scheduled"); EventLoopGroup eventLoop = EventLoopUtil.newEventLoopGroup(8, false, @@ -240,7 +258,7 @@ protected void doResolveAll(SocketAddress socketAddress, Promise promise) throws ConnectionPool pool = spyWithClassAndConstructorArgs(ConnectionPool.class, InstrumentProvider.NOOP, conf, eventLoop, (Supplier) () -> new ClientCnx(InstrumentProvider.NOOP, conf, eventLoop), - Optional.of(resolver)); + Optional.of(resolver), executorProvider); ClientCnx cnx = pool.getConnection( @@ -268,5 +286,6 @@ protected void doResolveAll(SocketAddress socketAddress, Promise promise) throws pool.closeAllConnections(); pool.close(); eventLoop.shutdownGracefully(); + executorProvider.shutdownNow(); } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/PulsarTestClient.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/PulsarTestClient.java index 8a79eb502439fa..f69cd576f9ac29 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/PulsarTestClient.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/PulsarTestClient.java @@ -81,7 +81,7 @@ public static PulsarTestClient create(ClientBuilder clientBuilder) throws Pulsar AtomicReference> clientCnxSupplierReference = new AtomicReference<>(); ConnectionPool connectionPool = new ConnectionPool(InstrumentProvider.NOOP, clientConfigurationData, eventLoopGroup, - () -> clientCnxSupplierReference.get().get()); + () -> clientCnxSupplierReference.get().get(), null); return new PulsarTestClient(clientConfigurationData, eventLoopGroup, connectionPool, clientCnxSupplierReference); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactorToolTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactorToolTest.java index 72b8628cacaa8c..101d0a10b4fd1b 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactorToolTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/compaction/CompactorToolTest.java @@ -98,6 +98,7 @@ public void testUseTlsUrlWithPEM() throws PulsarClientException { verify(serviceConfiguration, times(1)).getBrokerClientKeyFilePath(); verify(serviceConfiguration, times(1)).getBrokerClientTrustCertsFilePath(); verify(serviceConfiguration, times(1)).getBrokerClientCertificateFilePath(); + serviceConfiguration.setBrokerClientTlsTrustStorePassword(MockedPulsarServiceBaseTest.BROKER_KEYSTORE_PW); } @Test diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/security/MockedPulsarStandalone.java b/pulsar-broker/src/test/java/org/apache/pulsar/security/MockedPulsarStandalone.java index 4a7d71c2b4f3e8..866018b32fb0c0 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/security/MockedPulsarStandalone.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/security/MockedPulsarStandalone.java @@ -69,6 +69,9 @@ public abstract class MockedPulsarStandalone implements AutoCloseable { serviceConfiguration.setWebServicePortTls(Optional.of(0)); serviceConfiguration.setNumExecutorThreadPoolSize(5); serviceConfiguration.setExposeBundlesMetricsInPrometheus(true); + serviceConfiguration.setTlsTrustCertsFilePath(TLS_EC_TRUSTED_CERT_PATH); + serviceConfiguration.setTlsCertificateFilePath(TLS_EC_SERVER_CERT_PATH); + serviceConfiguration.setTlsKeyFilePath(TLS_EC_SERVER_KEY_PATH); } diff --git a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/client/admin/PulsarAdminBuilder.java b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/client/admin/PulsarAdminBuilder.java index b26e5b2cec8020..5c41d98b89dbcf 100644 --- a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/client/admin/PulsarAdminBuilder.java +++ b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/client/admin/PulsarAdminBuilder.java @@ -290,6 +290,20 @@ PulsarAdminBuilder authentication(String authPluginClassName, Map tlsProtocols); + /** + * SSL Factory Plugin used to generate the SSL Context and SSLEngine. + * @param sslFactoryPlugin Name of the SSL Factory Class to be used. + * @return PulsarAdminBuilder + */ + PulsarAdminBuilder sslFactoryPlugin(String sslFactoryPlugin); + + /** + * Parameters used by the SSL Factory Plugin class. + * @param sslFactoryPluginParams String parameters to be used by the SSL Factory Class. + * @return + */ + PulsarAdminBuilder sslFactoryPluginParams(String sslFactoryPluginParams); + /** * This sets the connection time out for the pulsar admin client. * diff --git a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/ClusterData.java b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/ClusterData.java index 1f7126521c6d6f..6aeed746db428f 100644 --- a/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/ClusterData.java +++ b/pulsar-client-admin-api/src/main/java/org/apache/pulsar/common/policies/data/ClusterData.java @@ -65,6 +65,10 @@ public interface ClusterData { String getBrokerClientTlsKeyStore(); + String getBrokerClientSslFactoryPlugin(); + + String getBrokerClientSslFactoryPluginParams(); + String getListenerName(); interface Builder { @@ -112,6 +116,10 @@ interface Builder { Builder listenerName(String listenerName); + Builder brokerClientSslFactoryPlugin(String sslFactoryPlugin); + + Builder brokerClientSslFactoryPluginParams(String sslFactoryPluginParams); + ClusterData build(); } diff --git a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/PulsarAdminBuilderImpl.java b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/PulsarAdminBuilderImpl.java index 9bfb4fc45f3b7e..7f0b3ab9a42184 100644 --- a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/PulsarAdminBuilderImpl.java +++ b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/PulsarAdminBuilderImpl.java @@ -214,6 +214,20 @@ public PulsarAdminBuilder tlsCiphers(Set tlsCiphers) { return this; } + @Override + public PulsarAdminBuilder sslFactoryPlugin(String sslFactoryPlugin) { + if (StringUtils.isNotBlank(sslFactoryPlugin)) { + conf.setSslFactoryPlugin(sslFactoryPlugin); + } + return this; + } + + @Override + public PulsarAdminBuilder sslFactoryPluginParams(String sslFactoryPluginParams) { + conf.setSslFactoryPluginParams(sslFactoryPluginParams); + return this; + } + @Override public PulsarAdminBuilder tlsProtocols(Set tlsProtocols) { conf.setTlsProtocols(tlsProtocols); diff --git a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/http/AsyncHttpConnector.java b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/http/AsyncHttpConnector.java index 1423d52642027f..d9d8b12aa7ea76 100644 --- a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/http/AsyncHttpConnector.java +++ b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/http/AsyncHttpConnector.java @@ -31,8 +31,6 @@ import io.netty.handler.codec.http.DefaultHttpHeaders; import io.netty.handler.codec.http.HttpRequest; import io.netty.handler.codec.http.HttpResponse; -import io.netty.handler.ssl.SslContext; -import io.netty.handler.ssl.SslProvider; import io.netty.util.concurrent.DefaultThreadFactory; import java.io.ByteArrayOutputStream; import java.io.IOException; @@ -48,9 +46,9 @@ import java.util.concurrent.Executors; import java.util.concurrent.Future; import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import java.util.function.Supplier; -import javax.net.ssl.SSLContext; import javax.ws.rs.ProcessingException; import javax.ws.rs.client.Client; import javax.ws.rs.core.HttpHeaders; @@ -61,15 +59,14 @@ import org.apache.commons.lang3.Validate; import org.apache.pulsar.PulsarVersion; import org.apache.pulsar.client.admin.internal.PulsarAdminImpl; -import org.apache.pulsar.client.api.AuthenticationDataProvider; -import org.apache.pulsar.client.api.KeyStoreParams; +import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.client.impl.PulsarServiceNameResolver; import org.apache.pulsar.client.impl.ServiceNameResolver; import org.apache.pulsar.client.impl.conf.ClientConfigurationData; -import org.apache.pulsar.client.util.WithSNISslEngineFactory; +import org.apache.pulsar.client.util.PulsarHttpAsyncSslEngineFactory; import org.apache.pulsar.common.util.FutureUtil; -import org.apache.pulsar.common.util.SecurityUtility; -import org.apache.pulsar.common.util.keystoretls.KeyStoreSSLContext; +import org.apache.pulsar.common.util.PulsarSslConfiguration; +import org.apache.pulsar.common.util.PulsarSslFactory; import org.asynchttpclient.AsyncCompletionHandlerBase; import org.asynchttpclient.AsyncHandler; import org.asynchttpclient.AsyncHttpClient; @@ -82,7 +79,6 @@ import org.asynchttpclient.Response; import org.asynchttpclient.SslEngineFactory; import org.asynchttpclient.channel.DefaultKeepAliveStrategy; -import org.asynchttpclient.netty.ssl.JsseSslEngineFactory; import org.asynchttpclient.uri.Uri; import org.glassfish.jersey.client.ClientProperties; import org.glassfish.jersey.client.ClientRequest; @@ -105,8 +101,10 @@ public class AsyncHttpConnector implements Connector, AsyncHttpRequestExecutor { private final ServiceNameResolver serviceNameResolver; private final ScheduledExecutorService delayer = Executors.newScheduledThreadPool(1, new DefaultThreadFactory("delayer")); + private ScheduledExecutorService sslRefresher; private final boolean acceptGzipCompression; private final Map> concurrencyReducers = new ConcurrentHashMap<>(); + private PulsarSslFactory sslFactory; public AsyncHttpConnector(Client client, ClientConfigurationData conf, int autoCertRefreshTimeSeconds, boolean acceptGzipCompression) { @@ -185,68 +183,26 @@ protected AsyncHttpClient createAsyncHttpClient(AsyncHttpClientConfig asyncHttpC return new DefaultAsyncHttpClient(asyncHttpClientConfig); } + @SneakyThrows private void configureAsyncHttpClientSslEngineFactory(ClientConfigurationData conf, int autoCertRefreshTimeSeconds, DefaultAsyncHttpClientConfig.Builder confBuilder) throws GeneralSecurityException, IOException { // Set client key and certificate if available - AuthenticationDataProvider authData = conf.getAuthentication().getAuthData(); - - SslEngineFactory sslEngineFactory = null; - if (conf.isUseKeyStoreTls()) { - KeyStoreParams params = authData.hasDataForTls() ? authData.getTlsKeyStoreParams() : - new KeyStoreParams(conf.getTlsKeyStoreType(), conf.getTlsKeyStorePath(), - conf.getTlsKeyStorePassword()); - - final SSLContext sslCtx = KeyStoreSSLContext.createClientSslContext( - conf.getSslProvider(), - params.getKeyStoreType(), - params.getKeyStorePath(), - params.getKeyStorePassword(), - conf.isTlsAllowInsecureConnection(), - conf.getTlsTrustStoreType(), - conf.getTlsTrustStorePath(), - conf.getTlsTrustStorePassword(), - conf.getTlsCiphers(), - conf.getTlsProtocols()); - - sslEngineFactory = new JsseSslEngineFactory(sslCtx); - confBuilder.setSslEngineFactory(sslEngineFactory); - } else { - SslProvider sslProvider = null; - if (conf.getSslProvider() != null) { - sslProvider = SslProvider.valueOf(conf.getSslProvider()); - } - SslContext sslCtx = null; - if (authData.hasDataForTls()) { - sslCtx = authData.getTlsTrustStoreStream() == null - ? SecurityUtility.createAutoRefreshSslContextForClient( - sslProvider, - conf.isTlsAllowInsecureConnection(), - conf.getTlsTrustCertsFilePath(), authData.getTlsCertificateFilePath(), - authData.getTlsPrivateKeyFilePath(), null, autoCertRefreshTimeSeconds, delayer) - : SecurityUtility.createNettySslContextForClient( - sslProvider, - conf.isTlsAllowInsecureConnection(), - authData.getTlsTrustStoreStream(), authData.getTlsCertificates(), - authData.getTlsPrivateKey(), - conf.getTlsCiphers(), - conf.getTlsProtocols()); - } else { - sslCtx = SecurityUtility.createNettySslContextForClient( - sslProvider, - conf.isTlsAllowInsecureConnection(), - conf.getTlsTrustCertsFilePath(), - conf.getTlsCertificateFilePath(), - conf.getTlsKeyFilePath(), - conf.getTlsCiphers(), - conf.getTlsProtocols()); - } - confBuilder.setSslContext(sslCtx); - if (!conf.isTlsHostnameVerificationEnable()) { - confBuilder.setSslEngineFactory(new WithSNISslEngineFactory(serviceNameResolver - .resolveHostUri().getHost())); - } - } + sslRefresher = Executors.newScheduledThreadPool(1, + new DefaultThreadFactory("pulsar-admin-ssl-refresher")); + PulsarSslConfiguration sslConfiguration = buildSslConfiguration(conf); + this.sslFactory = (PulsarSslFactory) Class.forName(conf.getSslFactoryPlugin()) + .getConstructor().newInstance(); + this.sslFactory.initialize(sslConfiguration); + this.sslFactory.createInternalSslContext(); + this.sslRefresher.scheduleWithFixedDelay(this::refreshSslContext, conf.getAutoCertRefreshSeconds(), + conf.getAutoCertRefreshSeconds(), TimeUnit.SECONDS); + String hostname = conf.isTlsHostnameVerificationEnable() ? null : serviceNameResolver + .resolveHostUri().getHost(); + SslEngineFactory sslEngineFactory = new PulsarHttpAsyncSslEngineFactory(sslFactory, hostname); + confBuilder.setSslEngineFactory(sslEngineFactory); + confBuilder.setUseInsecureTrustManager(conf.isTlsAllowInsecureConnection()); + confBuilder.setDisableHttpsEndpointIdentificationAlgorithm(!conf.isTlsHostnameVerificationEnable()); } @Override @@ -548,9 +504,45 @@ public void close() { try { httpClient.close(); delayer.shutdownNow(); + if (sslRefresher != null) { + sslRefresher.shutdownNow(); + } } catch (IOException e) { log.warn("Failed to close http client", e); } } + protected PulsarSslConfiguration buildSslConfiguration(ClientConfigurationData conf) + throws PulsarClientException { + return PulsarSslConfiguration.builder() + .tlsProvider(conf.getSslProvider()) + .tlsKeyStoreType(conf.getTlsKeyStoreType()) + .tlsKeyStorePath(conf.getTlsKeyStorePath()) + .tlsKeyStorePassword(conf.getTlsKeyStorePassword()) + .tlsTrustStoreType(conf.getTlsTrustStoreType()) + .tlsTrustStorePath(conf.getTlsTrustStorePath()) + .tlsTrustStorePassword(conf.getTlsTrustStorePassword()) + .tlsCiphers(conf.getTlsCiphers()) + .tlsProtocols(conf.getTlsProtocols()) + .tlsTrustCertsFilePath(conf.getTlsTrustCertsFilePath()) + .tlsCertificateFilePath(conf.getTlsCertificateFilePath()) + .tlsKeyFilePath(conf.getTlsKeyFilePath()) + .allowInsecureConnection(conf.isTlsAllowInsecureConnection()) + .requireTrustedClientCertOnConnect(false) + .tlsEnabledWithKeystore(conf.isUseKeyStoreTls()) + .authData(conf.getAuthentication().getAuthData()) + .tlsCustomParams(conf.getSslFactoryPluginParams()) + .serverMode(false) + .isHttps(true) + .build(); + } + + protected void refreshSslContext() { + try { + this.sslFactory.update(); + } catch (Exception e) { + log.error("Failed to refresh SSL context", e); + } + } + } diff --git a/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/ClientBuilder.java b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/ClientBuilder.java index 7b98fa57bf0dec..4adf7d89b0e335 100644 --- a/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/ClientBuilder.java +++ b/pulsar-client-api/src/main/java/org/apache/pulsar/client/api/ClientBuilder.java @@ -600,7 +600,7 @@ ClientBuilder authentication(String authPluginClassName, Map aut * * @param proxyServiceUrl proxy service url * @param proxyProtocol protocol to decide type of proxy routing eg: SNI-routing - * @return + * @return the client builder instance */ ClientBuilder proxyServiceUrl(String proxyServiceUrl, ProxyProtocol proxyProtocol); @@ -608,7 +608,7 @@ ClientBuilder authentication(String authPluginClassName, Map aut * If enable transaction, start the transactionCoordinatorClient with pulsar client. * * @param enableTransaction whether enable transaction feature - * @return + * @return the client builder instance */ ClientBuilder enableTransaction(boolean enableTransaction); @@ -616,35 +616,56 @@ ClientBuilder authentication(String authPluginClassName, Map aut * Set dns lookup bind address and port. * @param address dnsBindAddress * @param port dnsBindPort - * @return + * @return the client builder instance */ ClientBuilder dnsLookupBind(String address, int port); /** * Set dns lookup server addresses. * @param addresses dnsServerAddresses - * @return + * @return the client builder instance */ ClientBuilder dnsServerAddresses(List addresses); /** * Set socks5 proxy address. * @param socks5ProxyAddress - * @return + * @return the client builder instance */ ClientBuilder socks5ProxyAddress(InetSocketAddress socks5ProxyAddress); /** * Set socks5 proxy username. * @param socks5ProxyUsername - * @return + * @return the client builder instance */ ClientBuilder socks5ProxyUsername(String socks5ProxyUsername); /** * Set socks5 proxy password. * @param socks5ProxyPassword - * @return + * @return the client builder instance */ ClientBuilder socks5ProxyPassword(String socks5ProxyPassword); + + /** + * Set the SSL Factory Plugin for custom implementation to create SSL Context and SSLEngine. + * @param sslFactoryPlugin ssl factory class name + * @return the client builder instance + */ + ClientBuilder sslFactoryPlugin(String sslFactoryPlugin); + + /** + * Set the SSL Factory Plugin params for the ssl factory plugin to use. + * @param sslFactoryPluginParams Params in String format that will be inputted to the SSL Factory Plugin + * @return the client builder instance + */ + ClientBuilder sslFactoryPluginParams(String sslFactoryPluginParams); + + /** + * Set Cert Refresh interval in seconds. + * @param autoCertRefreshSeconds + * @return the client builder instance + */ + ClientBuilder autoCertRefreshSeconds(int autoCertRefreshSeconds); } diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdClusters.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdClusters.java index 14f9eeadbffb53..9f11b48513867d 100644 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdClusters.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdClusters.java @@ -364,6 +364,14 @@ private static class ClusterDetails { description = "path for the TLS certificate file", required = false) protected String brokerClientCertificateFilePath; + @Option(names = "--tls-factory-plugin", + description = "TLS Factory Plugin to be used to generate SSL Context and SSL Engine") + protected String brokerClientSslFactoryPlugin; + + @Option(names = "--tls-factory-plugin-params", + description = "Parameters used by the TLS Factory Plugin") + protected String brokerClientSslFactoryPluginParams; + @Option(names = "--listener-name", description = "listenerName when client would like to connect to cluster", required = false) protected String listenerName; @@ -440,6 +448,12 @@ protected ClusterData getClusterData() throws IOException { if (brokerClientCertificateFilePath != null) { builder.brokerClientCertificateFilePath(brokerClientCertificateFilePath); } + if (StringUtils.isNotBlank(brokerClientSslFactoryPlugin)) { + builder.brokerClientSslFactoryPlugin(brokerClientSslFactoryPlugin); + } + if (StringUtils.isNotBlank(brokerClientSslFactoryPluginParams)) { + builder.brokerClientSslFactoryPluginParams(brokerClientSslFactoryPluginParams); + } if (listenerName != null) { builder.listenerName(listenerName); diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/PulsarAdminTool.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/PulsarAdminTool.java index 2cc74d2f13bacd..cd79098f0c3e99 100644 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/PulsarAdminTool.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/PulsarAdminTool.java @@ -37,6 +37,7 @@ import org.apache.pulsar.client.admin.PulsarAdmin; import org.apache.pulsar.client.admin.PulsarAdminBuilder; import org.apache.pulsar.client.admin.internal.PulsarAdminImpl; +import org.apache.pulsar.common.util.DefaultPulsarSslFactory; import org.apache.pulsar.common.util.ShutdownUtil; import org.apache.pulsar.internal.CommandHook; import org.apache.pulsar.internal.CommanderFactory; @@ -130,6 +131,9 @@ private static PulsarAdminBuilder createAdminBuilderFromProperties(Properties pr boolean tlsEnableHostnameVerification = Boolean.parseBoolean(properties .getProperty("tlsEnableHostnameVerification", "false")); final String tlsTrustCertsFilePath = properties.getProperty("tlsTrustCertsFilePath"); + final String sslFactoryPlugin = properties.getProperty("sslFactoryPlugin", + DefaultPulsarSslFactory.class.getName()); + final String sslFactoryPluginParams = properties.getProperty("sslFactoryPluginParams", ""); return PulsarAdmin.builder().allowTlsInsecureConnection(tlsAllowInsecureConnection) .enableTlsHostnameVerification(tlsEnableHostnameVerification) @@ -142,7 +146,9 @@ private static PulsarAdminBuilder createAdminBuilderFromProperties(Properties pr .tlsKeyStorePath(tlsKeyStorePath) .tlsKeyStorePassword(tlsKeyStorePassword) .tlsKeyFilePath(tlsKeyFilePath) - .tlsCertificateFilePath(tlsCertificateFilePath); + .tlsCertificateFilePath(tlsCertificateFilePath) + .sslFactoryPlugin(sslFactoryPlugin) + .sslFactoryPluginParams(sslFactoryPluginParams); } private void setupCommands(Properties properties) { diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/client/cli/PulsarClientTool.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/client/cli/PulsarClientTool.java index 567c8d201e4ed6..98f129441733d0 100644 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/client/cli/PulsarClientTool.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/client/cli/PulsarClientTool.java @@ -111,6 +111,8 @@ public static class RootParams { String tlsKeyStoreType; String tlsKeyStorePath; String tlsKeyStorePassword; + String sslFactoryPlugin; + String sslFactoryPluginParams; protected final CommandLine commander; protected CmdProduce produceCommand; @@ -152,6 +154,8 @@ protected void initCommander(Properties properties) { this.tlsKeyStorePassword = properties.getProperty("tlsKeyStorePassword"); this.tlsKeyFilePath = properties.getProperty("tlsKeyFilePath"); this.tlsCertificateFilePath = properties.getProperty("tlsCertificateFilePath"); + this.sslFactoryPlugin = properties.getProperty("sslFactoryPlugin"); + this.sslFactoryPluginParams = properties.getProperty("sslFactoryPluginParams"); pulsarClientPropertiesProvider = PulsarClientPropertiesProvider.create(properties); commander.setDefaultValueProvider(pulsarClientPropertiesProvider); @@ -192,6 +196,9 @@ private int updateConfig() throws UnsupportedAuthenticationException { .tlsKeyStorePath(tlsKeyStorePath) .tlsKeyStorePassword(tlsKeyStorePassword); + clientBuilder.sslFactoryPlugin(sslFactoryPlugin) + .sslFactoryPluginParams(sslFactoryPluginParams); + if (isNotBlank(rootParams.proxyServiceURL)) { if (rootParams.proxyProtocol == null) { commander.getErr().println("proxy-protocol must be provided with proxy-url"); diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ClientBuilderImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ClientBuilderImpl.java index 2548a52aa95a85..d9edc53b50e37e 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ClientBuilderImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ClientBuilderImpl.java @@ -39,6 +39,7 @@ import org.apache.pulsar.client.impl.conf.ClientConfigurationData; import org.apache.pulsar.client.impl.conf.ConfigurationDataUtils; import org.apache.pulsar.common.tls.InetAddressUtils; +import org.apache.pulsar.common.util.DefaultPulsarSslFactory; public class ClientBuilderImpl implements ClientBuilder { ClientConfigurationData conf; @@ -431,6 +432,28 @@ public ClientBuilder socks5ProxyPassword(String socks5ProxyPassword) { return this; } + @Override + public ClientBuilder sslFactoryPlugin(String sslFactoryPlugin) { + if (StringUtils.isBlank(sslFactoryPlugin)) { + conf.setSslFactoryPlugin(DefaultPulsarSslFactory.class.getName()); + } else { + conf.setSslFactoryPlugin(sslFactoryPlugin); + } + return this; + } + + @Override + public ClientBuilder sslFactoryPluginParams(String sslFactoryPluginParams) { + conf.setSslFactoryPluginParams(sslFactoryPluginParams); + return this; + } + + @Override + public ClientBuilder autoCertRefreshSeconds(int autoCertRefreshSeconds) { + conf.setAutoCertRefreshSeconds(autoCertRefreshSeconds); + return this; + } + /** * Set the description. * diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConnectionPool.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConnectionPool.java index 21575578e76f20..723911122d954b 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConnectionPool.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConnectionPool.java @@ -58,6 +58,7 @@ import org.apache.pulsar.client.impl.metrics.Counter; import org.apache.pulsar.client.impl.metrics.InstrumentProvider; import org.apache.pulsar.client.impl.metrics.Unit; +import org.apache.pulsar.client.util.ScheduledExecutorProvider; import org.apache.pulsar.common.allocator.PulsarByteBufAllocator; import org.apache.pulsar.common.util.FutureUtil; import org.apache.pulsar.common.util.netty.DnsResolverUtil; @@ -103,20 +104,25 @@ private static class Key { } public ConnectionPool(InstrumentProvider instrumentProvider, - ClientConfigurationData conf, EventLoopGroup eventLoopGroup) throws PulsarClientException { - this(instrumentProvider, conf, eventLoopGroup, () -> new ClientCnx(instrumentProvider, conf, eventLoopGroup)); + ClientConfigurationData conf, EventLoopGroup eventLoopGroup, + ScheduledExecutorProvider scheduledExecutorProvider) throws PulsarClientException { + this(instrumentProvider, conf, eventLoopGroup, () -> new ClientCnx(instrumentProvider, conf, eventLoopGroup), + scheduledExecutorProvider); } public ConnectionPool(InstrumentProvider instrumentProvider, ClientConfigurationData conf, EventLoopGroup eventLoopGroup, - Supplier clientCnxSupplier) throws PulsarClientException { - this(instrumentProvider, conf, eventLoopGroup, clientCnxSupplier, Optional.empty()); + Supplier clientCnxSupplier, + ScheduledExecutorProvider scheduledExecutorProvider) throws PulsarClientException { + this(instrumentProvider, conf, eventLoopGroup, clientCnxSupplier, Optional.empty(), + scheduledExecutorProvider); } public ConnectionPool(InstrumentProvider instrumentProvider, ClientConfigurationData conf, EventLoopGroup eventLoopGroup, Supplier clientCnxSupplier, - Optional> addressResolver) + Optional> addressResolver, + ScheduledExecutorProvider scheduledExecutorProvider) throws PulsarClientException { this.eventLoopGroup = eventLoopGroup; this.clientConfig = conf; @@ -134,7 +140,8 @@ public ConnectionPool(InstrumentProvider instrumentProvider, bootstrap.option(ChannelOption.ALLOCATOR, PulsarByteBufAllocator.DEFAULT); try { - channelInitializerHandler = new PulsarChannelInitializer(conf, clientCnxSupplier); + channelInitializerHandler = new PulsarChannelInitializer(conf, clientCnxSupplier, + scheduledExecutorProvider); bootstrap.handler(channelInitializerHandler); } catch (Exception e) { log.error("Failed to create channel initializer"); diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/HttpClient.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/HttpClient.java index 38b8954377957e..bcf89d9611fe6e 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/HttpClient.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/HttpClient.java @@ -21,40 +21,39 @@ import io.netty.channel.EventLoopGroup; import io.netty.handler.codec.http.HttpRequest; import io.netty.handler.codec.http.HttpResponse; -import io.netty.handler.ssl.SslContext; -import io.netty.handler.ssl.SslProvider; import java.io.Closeable; import java.io.IOException; import java.net.HttpURLConnection; import java.net.InetSocketAddress; import java.net.URI; import java.net.URL; -import java.security.GeneralSecurityException; import java.util.Map; import java.util.Map.Entry; import java.util.Set; import java.util.concurrent.CompletableFuture; -import javax.net.ssl.SSLContext; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; import lombok.extern.slf4j.Slf4j; import org.apache.pulsar.PulsarVersion; import org.apache.pulsar.client.api.Authentication; import org.apache.pulsar.client.api.AuthenticationDataProvider; -import org.apache.pulsar.client.api.KeyStoreParams; import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.client.api.PulsarClientException.NotFoundException; import org.apache.pulsar.client.impl.conf.ClientConfigurationData; -import org.apache.pulsar.client.util.WithSNISslEngineFactory; +import org.apache.pulsar.client.util.ExecutorProvider; +import org.apache.pulsar.client.util.PulsarHttpAsyncSslEngineFactory; import org.apache.pulsar.common.util.ObjectMapperFactory; -import org.apache.pulsar.common.util.SecurityUtility; -import org.apache.pulsar.common.util.keystoretls.KeyStoreSSLContext; +import org.apache.pulsar.common.util.PulsarSslConfiguration; +import org.apache.pulsar.common.util.PulsarSslFactory; import org.asynchttpclient.AsyncHttpClient; import org.asynchttpclient.AsyncHttpClientConfig; import org.asynchttpclient.BoundRequestBuilder; import org.asynchttpclient.DefaultAsyncHttpClient; import org.asynchttpclient.DefaultAsyncHttpClientConfig; import org.asynchttpclient.Request; +import org.asynchttpclient.SslEngineFactory; import org.asynchttpclient.channel.DefaultKeepAliveStrategy; -import org.asynchttpclient.netty.ssl.JsseSslEngineFactory; @Slf4j @@ -66,6 +65,8 @@ public class HttpClient implements Closeable { protected final AsyncHttpClient httpClient; protected final ServiceNameResolver serviceNameResolver; protected final Authentication authentication; + protected ScheduledExecutorService executorService; + protected PulsarSslFactory sslFactory; protected HttpClient(ClientConfigurationData conf, EventLoopGroup eventLoopGroup) throws PulsarClientException { this.authentication = conf.getAuthentication(); @@ -92,65 +93,25 @@ public boolean keepAlive(InetSocketAddress remoteAddress, Request ahcRequest, if ("https".equals(serviceNameResolver.getServiceUri().getServiceName())) { try { // Set client key and certificate if available - AuthenticationDataProvider authData = authentication.getAuthData(); + this.executorService = Executors + .newSingleThreadScheduledExecutor(new ExecutorProvider + .ExtendedThreadFactory("httpclient-ssl-refresh")); + PulsarSslConfiguration sslConfiguration = buildSslConfiguration(conf); + this.sslFactory = (PulsarSslFactory) Class.forName(conf.getSslFactoryPlugin()) + .getConstructor().newInstance(); + this.sslFactory.initialize(sslConfiguration); + this.sslFactory.createInternalSslContext(); + this.executorService.scheduleWithFixedDelay(this::refreshSslContext, conf.getAutoCertRefreshSeconds(), + conf.getAutoCertRefreshSeconds(), TimeUnit.SECONDS); + String hostname = conf.isTlsHostnameVerificationEnable() ? null : serviceNameResolver + .resolveHostUri().getHost(); + SslEngineFactory sslEngineFactory = new PulsarHttpAsyncSslEngineFactory(this.sslFactory, hostname); + confBuilder.setSslEngineFactory(sslEngineFactory); - if (conf.isUseKeyStoreTls()) { - SSLContext sslCtx = null; - KeyStoreParams params = authData.hasDataForTls() ? authData.getTlsKeyStoreParams() : - new KeyStoreParams(conf.getTlsKeyStoreType(), conf.getTlsKeyStorePath(), - conf.getTlsKeyStorePassword()); - sslCtx = KeyStoreSSLContext.createClientSslContext( - conf.getSslProvider(), - params.getKeyStoreType(), - params.getKeyStorePath(), - params.getKeyStorePassword(), - conf.isTlsAllowInsecureConnection(), - conf.getTlsTrustStoreType(), - conf.getTlsTrustStorePath(), - conf.getTlsTrustStorePassword(), - conf.getTlsCiphers(), - conf.getTlsProtocols()); - - JsseSslEngineFactory sslEngineFactory = new JsseSslEngineFactory(sslCtx); - confBuilder.setSslEngineFactory(sslEngineFactory); - } else { - SslProvider sslProvider = null; - if (conf.getSslProvider() != null) { - sslProvider = SslProvider.valueOf(conf.getSslProvider()); - } - SslContext sslCtx = null; - if (authData.hasDataForTls()) { - sslCtx = authData.getTlsTrustStoreStream() == null - ? SecurityUtility.createNettySslContextForClient(sslProvider, - conf.isTlsAllowInsecureConnection(), - conf.getTlsTrustCertsFilePath(), authData.getTlsCertificates(), - authData.getTlsPrivateKey(), conf.getTlsCiphers(), conf.getTlsProtocols()) - : SecurityUtility.createNettySslContextForClient(sslProvider, - conf.isTlsAllowInsecureConnection(), - authData.getTlsTrustStoreStream(), authData.getTlsCertificates(), - authData.getTlsPrivateKey(), conf.getTlsCiphers(), conf.getTlsProtocols()); - } else { - sslCtx = SecurityUtility.createNettySslContextForClient( - sslProvider, - conf.isTlsAllowInsecureConnection(), - conf.getTlsTrustCertsFilePath(), - conf.getTlsCertificateFilePath(), - conf.getTlsKeyFilePath(), - conf.getTlsCiphers(), - conf.getTlsProtocols()); - } - confBuilder.setSslContext(sslCtx); - if (!conf.isTlsHostnameVerificationEnable()) { - confBuilder.setSslEngineFactory(new WithSNISslEngineFactory(serviceNameResolver - .resolveHostUri().getHost())); - } - } confBuilder.setUseInsecureTrustManager(conf.isTlsAllowInsecureConnection()); confBuilder.setDisableHttpsEndpointIdentificationAlgorithm(!conf.isTlsHostnameVerificationEnable()); - } catch (GeneralSecurityException e) { - throw new PulsarClientException.InvalidConfigurationException(e); } catch (Exception e) { throw new PulsarClientException.InvalidConfigurationException(e); } @@ -177,6 +138,9 @@ void setServiceUrl(String serviceUrl) throws PulsarClientException { @Override public void close() throws IOException { httpClient.close(); + if (executorService != null) { + executorService.shutdownNow(); + } } public CompletableFuture get(String path, Class clazz) { @@ -264,4 +228,37 @@ public CompletableFuture get(String path, Class clazz) { return future; } + + protected PulsarSslConfiguration buildSslConfiguration(ClientConfigurationData config) + throws PulsarClientException { + return PulsarSslConfiguration.builder() + .tlsProvider(config.getSslProvider()) + .tlsKeyStoreType(config.getTlsKeyStoreType()) + .tlsKeyStorePath(config.getTlsKeyStorePath()) + .tlsKeyStorePassword(config.getTlsKeyStorePassword()) + .tlsTrustStoreType(config.getTlsTrustStoreType()) + .tlsTrustStorePath(config.getTlsTrustStorePath()) + .tlsTrustStorePassword(config.getTlsTrustStorePassword()) + .tlsCiphers(config.getTlsCiphers()) + .tlsProtocols(config.getTlsProtocols()) + .tlsTrustCertsFilePath(config.getTlsTrustCertsFilePath()) + .tlsCertificateFilePath(config.getTlsCertificateFilePath()) + .tlsKeyFilePath(config.getTlsKeyFilePath()) + .allowInsecureConnection(config.isTlsAllowInsecureConnection()) + .requireTrustedClientCertOnConnect(false) + .tlsEnabledWithKeystore(config.isUseKeyStoreTls()) + .tlsCustomParams(config.getSslFactoryPluginParams()) + .authData(config.getAuthentication().getAuthData()) + .serverMode(false) + .isHttps(true) + .build(); + } + + protected void refreshSslContext() { + try { + this.sslFactory.update(); + } catch (Exception e) { + log.error("Failed to refresh SSL context", e); + } + } } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PulsarChannelInitializer.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PulsarChannelInitializer.java index dff423d19fbefb..ef4976d4f173ef 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PulsarChannelInitializer.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PulsarChannelInitializer.java @@ -25,25 +25,23 @@ import io.netty.handler.codec.LengthFieldBasedFrameDecoder; import io.netty.handler.flush.FlushConsolidationHandler; import io.netty.handler.proxy.Socks5ProxyHandler; -import io.netty.handler.ssl.SslContext; import io.netty.handler.ssl.SslHandler; -import io.netty.handler.ssl.SslProvider; import java.net.InetSocketAddress; import java.util.Objects; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; import java.util.function.Supplier; import lombok.Getter; import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; -import org.apache.pulsar.client.api.AuthenticationDataProvider; import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.client.impl.conf.ClientConfigurationData; -import org.apache.pulsar.client.util.ObjectCache; +import org.apache.pulsar.client.util.ScheduledExecutorProvider; import org.apache.pulsar.common.protocol.ByteBufPair; import org.apache.pulsar.common.protocol.Commands; +import org.apache.pulsar.common.util.PulsarSslConfiguration; +import org.apache.pulsar.common.util.PulsarSslFactory; import org.apache.pulsar.common.util.SecurityUtility; -import org.apache.pulsar.common.util.keystoretls.NettySSLContextAutoRefreshBuilder; import org.apache.pulsar.common.util.netty.NettyFutureUtil; @Slf4j @@ -55,18 +53,16 @@ public class PulsarChannelInitializer extends ChannelInitializer @Getter private final boolean tlsEnabled; private final boolean tlsHostnameVerificationEnabled; - private final boolean tlsEnabledWithKeyStore; private final InetSocketAddress socks5ProxyAddress; private final String socks5ProxyUsername; private final String socks5ProxyPassword; - private final Supplier sslContextSupplier; - private NettySSLContextAutoRefreshBuilder nettySSLContextAutoRefreshBuilder; + private final PulsarSslFactory pulsarSslFactory; private static final long TLS_CERTIFICATE_CACHE_MILLIS = TimeUnit.MINUTES.toMillis(1); - public PulsarChannelInitializer(ClientConfigurationData conf, Supplier clientCnxSupplier) - throws Exception { + public PulsarChannelInitializer(ClientConfigurationData conf, Supplier clientCnxSupplier, + ScheduledExecutorProvider scheduledExecutorProvider) throws Exception { super(); this.clientCnxSupplier = clientCnxSupplier; this.tlsEnabled = conf.isUseTls(); @@ -75,71 +71,26 @@ public PulsarChannelInitializer(ClientConfigurationData conf, Supplier { + this.refreshSslContext(conf); + }, conf.getAutoCertRefreshSeconds(), + conf.getAutoCertRefreshSeconds(), TimeUnit.SECONDS); } - sslContextSupplier = new ObjectCache(() -> { - try { - SslProvider sslProvider = null; - if (conf.getSslProvider() != null) { - sslProvider = SslProvider.valueOf(conf.getSslProvider()); - } - - // Set client certificate if available - AuthenticationDataProvider authData = conf.getAuthentication().getAuthData(); - if (authData.hasDataForTls()) { - return authData.getTlsTrustStoreStream() == null - ? SecurityUtility.createNettySslContextForClient( - sslProvider, - conf.isTlsAllowInsecureConnection(), - conf.getTlsTrustCertsFilePath(), - authData.getTlsCertificates(), - authData.getTlsPrivateKey(), - conf.getTlsCiphers(), - conf.getTlsProtocols()) - : SecurityUtility.createNettySslContextForClient(sslProvider, - conf.isTlsAllowInsecureConnection(), - authData.getTlsTrustStoreStream(), - authData.getTlsCertificates(), authData.getTlsPrivateKey(), - conf.getTlsCiphers(), - conf.getTlsProtocols()); - } else { - return SecurityUtility.createNettySslContextForClient( - sslProvider, - conf.isTlsAllowInsecureConnection(), - conf.getTlsTrustCertsFilePath(), - conf.getTlsCertificateFilePath(), - conf.getTlsKeyFilePath(), - conf.getTlsCiphers(), - conf.getTlsProtocols()); - } - } catch (Exception e) { - throw new RuntimeException("Failed to create TLS context", e); - } - }, TLS_CERTIFICATE_CACHE_MILLIS, TimeUnit.MILLISECONDS); } else { - sslContextSupplier = null; + pulsarSslFactory = null; } } @@ -174,10 +125,8 @@ CompletableFuture initTls(Channel ch, InetSocketAddress sniHost) { CompletableFuture initTlsFuture = new CompletableFuture<>(); ch.eventLoop().execute(() -> { try { - SslHandler handler = tlsEnabledWithKeyStore - ? new SslHandler(nettySSLContextAutoRefreshBuilder.get() - .createSSLEngine(sniHost.getHostString(), sniHost.getPort())) - : sslContextSupplier.get().newHandler(ch.alloc(), sniHost.getHostString(), sniHost.getPort()); + SslHandler handler = new SslHandler(pulsarSslFactory + .createClientSslEngine(ch.alloc(), sniHost.getHostName(), sniHost.getPort())); if (tlsHostnameVerificationEnabled) { SecurityUtility.configureSSLHandler(handler); @@ -234,5 +183,48 @@ CompletableFuture initializeClientCnx(Channel ch, return ch; })); } + protected PulsarSslConfiguration buildSslConfiguration(ClientConfigurationData config) + throws PulsarClientException { + return PulsarSslConfiguration.builder() + .tlsProvider(config.getSslProvider()) + .tlsKeyStoreType(config.getTlsKeyStoreType()) + .tlsKeyStorePath(config.getTlsKeyStorePath()) + .tlsKeyStorePassword(config.getTlsKeyStorePassword()) + .tlsTrustStoreType(config.getTlsTrustStoreType()) + .tlsTrustStorePath(config.getTlsTrustStorePath()) + .tlsTrustStorePassword(config.getTlsTrustStorePassword()) + .tlsCiphers(config.getTlsCiphers()) + .tlsProtocols(config.getTlsProtocols()) + .tlsTrustCertsFilePath(config.getTlsTrustCertsFilePath()) + .tlsCertificateFilePath(config.getTlsCertificateFilePath()) + .tlsKeyFilePath(config.getTlsKeyFilePath()) + .allowInsecureConnection(config.isTlsAllowInsecureConnection()) + .requireTrustedClientCertOnConnect(false) + .tlsEnabledWithKeystore(config.isUseKeyStoreTls()) + .tlsCustomParams(config.getSslFactoryPluginParams()) + .authData(config.getAuthentication().getAuthData()) + .serverMode(false) + .build(); + } + + protected void refreshSslContext(ClientConfigurationData conf) { + try { + try { + if (conf.isUseKeyStoreTls()) { + this.pulsarSslFactory.getInternalSslContext(); + } else { + this.pulsarSslFactory.getInternalNettySslContext(); + } + } catch (Exception e) { + log.error("SSL Context is not initialized", e); + PulsarSslConfiguration sslConfiguration = buildSslConfiguration(conf); + this.pulsarSslFactory.initialize(sslConfiguration); + } + this.pulsarSslFactory.update(); + } catch (Exception e) { + log.error("Failed to refresh SSL context", e); + } + } + } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PulsarClientImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PulsarClientImpl.java index d37c3a10e16074..23c1b722f40070 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PulsarClientImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/PulsarClientImpl.java @@ -206,16 +206,17 @@ private PulsarClientImpl(ClientConfigurationData conf, EventLoopGroup eventLoopG this.instrumentProvider = new InstrumentProvider(conf.getOpenTelemetry()); clientClock = conf.getClock(); conf.getAuthentication().start(); + this.scheduledExecutorProvider = scheduledExecutorProvider != null ? scheduledExecutorProvider : + new ScheduledExecutorProvider(conf.getNumIoThreads(), "pulsar-client-scheduled"); connectionPoolReference = connectionPool != null ? connectionPool : - new ConnectionPool(instrumentProvider, conf, this.eventLoopGroup); + new ConnectionPool(instrumentProvider, conf, this.eventLoopGroup, + this.scheduledExecutorProvider); this.cnxPool = connectionPoolReference; this.externalExecutorProvider = externalExecutorProvider != null ? externalExecutorProvider : new ExecutorProvider(conf.getNumListenerThreads(), "pulsar-external-listener"); this.internalExecutorProvider = internalExecutorProvider != null ? internalExecutorProvider : new ExecutorProvider(conf.getNumIoThreads(), "pulsar-client-internal"); - this.scheduledExecutorProvider = scheduledExecutorProvider != null ? scheduledExecutorProvider : - new ScheduledExecutorProvider(conf.getNumIoThreads(), "pulsar-client-scheduled"); if (conf.getServiceUrl().startsWith("http")) { lookup = new HttpLookupService(instrumentProvider, conf, this.eventLoopGroup); } else { diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/conf/ClientConfigurationData.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/conf/ClientConfigurationData.java index 237c6b5aebc3c0..e2713644af6415 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/conf/ClientConfigurationData.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/conf/ClientConfigurationData.java @@ -43,6 +43,8 @@ import org.apache.pulsar.client.api.ServiceUrlProvider; import org.apache.pulsar.client.impl.auth.AuthenticationDisabled; import org.apache.pulsar.client.util.Secret; +import org.apache.pulsar.common.util.DefaultPulsarSslFactory; + /** * This is a simple holder of the client configuration values. @@ -179,6 +181,18 @@ public class ClientConfigurationData implements Serializable, Cloneable { value = "Whether the hostname is validated when the client creates a TLS connection with brokers." ) private boolean tlsHostnameVerificationEnable = false; + + @ApiModelProperty( + name = "sslFactoryPlugin", + value = "SSL Factory Plugin class to provide SSLEngine and SSLContext objects. The default " + + " class used is DefaultPulsarSslFactory.") + private String sslFactoryPlugin = DefaultPulsarSslFactory.class.getName(); + + @ApiModelProperty( + name = "sslFactoryPluginParams", + value = "SSL Factory plugin configuration parameters.") + private String sslFactoryPluginParams = ""; + @ApiModelProperty( name = "concurrentLookupRequest", value = "The number of concurrent lookup requests that can be sent on each broker connection. " diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/util/WithSNISslEngineFactory.java b/pulsar-client/src/main/java/org/apache/pulsar/client/util/PulsarHttpAsyncSslEngineFactory.java similarity index 53% rename from pulsar-client/src/main/java/org/apache/pulsar/client/util/WithSNISslEngineFactory.java rename to pulsar-client/src/main/java/org/apache/pulsar/client/util/PulsarHttpAsyncSslEngineFactory.java index d950e68271bcd9..ddf034bbb098eb 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/util/WithSNISslEngineFactory.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/util/PulsarHttpAsyncSslEngineFactory.java @@ -20,23 +20,42 @@ import java.util.Collections; import javax.net.ssl.SNIHostName; +import javax.net.ssl.SSLContext; import javax.net.ssl.SSLEngine; import javax.net.ssl.SSLParameters; +import org.apache.commons.lang3.StringUtils; +import org.apache.pulsar.common.util.PulsarSslFactory; import org.asynchttpclient.AsyncHttpClientConfig; import org.asynchttpclient.netty.ssl.DefaultSslEngineFactory; -public class WithSNISslEngineFactory extends DefaultSslEngineFactory { +public class PulsarHttpAsyncSslEngineFactory extends DefaultSslEngineFactory { + + private final PulsarSslFactory pulsarSslFactory; private final String host; - public WithSNISslEngineFactory(String host) { + public PulsarHttpAsyncSslEngineFactory(PulsarSslFactory pulsarSslFactory, String host) { + this.pulsarSslFactory = pulsarSslFactory; this.host = host; } @Override protected void configureSslEngine(SSLEngine sslEngine, AsyncHttpClientConfig config) { super.configureSslEngine(sslEngine, config); - SSLParameters params = sslEngine.getSSLParameters(); - params.setServerNames(Collections.singletonList(new SNIHostName(host))); - sslEngine.setSSLParameters(params); + if (StringUtils.isNotBlank(host)) { + SSLParameters parameters = sslEngine.getSSLParameters(); + parameters.setServerNames(Collections.singletonList(new SNIHostName(host))); + sslEngine.setSSLParameters(parameters); + } } -} + + @Override + public SSLEngine newSslEngine(AsyncHttpClientConfig config, String peerHost, int peerPort) { + SSLContext sslContext = this.pulsarSslFactory.getInternalSslContext(); + SSLEngine sslEngine = config.isDisableHttpsEndpointIdentificationAlgorithm() + ? sslContext.createSSLEngine() : + sslContext.createSSLEngine(domain(peerHost), peerPort); + configureSslEngine(sslEngine, config); + return sslEngine; + } + +} \ No newline at end of file diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ClientInitializationTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ClientInitializationTest.java index 3b92f362ca188e..2682d011cd0c5c 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ClientInitializationTest.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ClientInitializationTest.java @@ -19,8 +19,8 @@ package org.apache.pulsar.client.impl; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.verifyNoMoreInteractions; import lombok.Cleanup; import org.apache.pulsar.client.api.Authentication; import org.apache.pulsar.client.api.PulsarClient; @@ -40,9 +40,7 @@ public void testInitializeAuthWithTls() throws PulsarClientException { .authentication(auth) .build(); - // Auth should only be started, though we shouldn't have tried to get credentials yet (until we first attempt to - // connect). verify(auth).start(); - verifyNoMoreInteractions(auth); + verify(auth, times(1)).getAuthData(); } } diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/PulsarClientImplTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/PulsarClientImplTest.java index 103254a6b90a47..4481de9f1e65fa 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/PulsarClientImplTest.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/PulsarClientImplTest.java @@ -182,7 +182,7 @@ public void testInitializeWithTimer() throws PulsarClientException { ClientConfigurationData conf = new ClientConfigurationData(); @Cleanup("shutdownGracefully") EventLoopGroup eventLoop = EventLoopUtil.newEventLoopGroup(1, false, new DefaultThreadFactory("test")); - ConnectionPool pool = Mockito.spy(new ConnectionPool(InstrumentProvider.NOOP, conf, eventLoop)); + ConnectionPool pool = Mockito.spy(new ConnectionPool(InstrumentProvider.NOOP, conf, eventLoop, null)); conf.setServiceUrl("pulsar://localhost:6650"); HashedWheelTimer timer = new HashedWheelTimer(); @@ -207,7 +207,7 @@ public void testResourceCleanup() throws Exception { ClientConfigurationData conf = new ClientConfigurationData(); conf.setServiceUrl(""); initializeEventLoopGroup(conf); - try (ConnectionPool connectionPool = new ConnectionPool(InstrumentProvider.NOOP, conf, eventLoopGroup)) { + try (ConnectionPool connectionPool = new ConnectionPool(InstrumentProvider.NOOP, conf, eventLoopGroup, null)) { assertThrows(() -> new PulsarClientImpl(conf, eventLoopGroup, connectionPool)); } finally { // Externally passed eventLoopGroup should not be shutdown. diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/ClusterDataImpl.java b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/ClusterDataImpl.java index 6cb5a0034e9383..b887fe0a5861bf 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/ClusterDataImpl.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/ClusterDataImpl.java @@ -28,6 +28,7 @@ import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.apache.pulsar.client.api.ProxyProtocol; +import org.apache.pulsar.common.util.DefaultPulsarSslFactory; import org.apache.pulsar.common.util.URIPreconditions; /** @@ -170,6 +171,16 @@ public final class ClusterDataImpl implements ClusterData, Cloneable { + "used by the internal client to authenticate with Pulsar brokers" ) private String brokerClientCertificateFilePath; + @ApiModelProperty( + name = "brokerClientSslFactoryPlugin", + value = "SSL Factory plugin used by internal client to generate the SSL Context and Engine" + ) + private String brokerClientSslFactoryPlugin; + @ApiModelProperty( + name = "brokerClientSslFactoryPluginParams", + value = "Parameters used by the internal client's SSL factory plugin to generate the SSL Context and Engine" + ) + private String brokerClientSslFactoryPluginParams; @ApiModelProperty( name = "listenerName", value = "listenerName when client would like to connect to cluster", @@ -205,6 +216,8 @@ public ClusterDataImplBuilder clone() { .brokerClientTrustCertsFilePath(brokerClientTrustCertsFilePath) .brokerClientCertificateFilePath(brokerClientCertificateFilePath) .brokerClientKeyFilePath(brokerClientKeyFilePath) + .brokerClientSslFactoryPlugin(brokerClientSslFactoryPlugin) + .brokerClientSslFactoryPluginParams(brokerClientSslFactoryPluginParams) .listenerName(listenerName); } @@ -231,6 +244,8 @@ public static class ClusterDataImplBuilder implements ClusterData.Builder { private String brokerClientCertificateFilePath; private String brokerClientKeyFilePath; private String brokerClientTrustCertsFilePath; + private String brokerClientSslFactoryPlugin = DefaultPulsarSslFactory.class.getName(); + private String brokerClientSslFactoryPluginParams; private String listenerName; ClusterDataImplBuilder() { @@ -346,6 +361,17 @@ public ClusterDataImplBuilder brokerClientKeyFilePath(String keyFilePath) { return this; } + @Override + public ClusterDataImplBuilder brokerClientSslFactoryPlugin(String sslFactoryPlugin) { + this.brokerClientSslFactoryPlugin = sslFactoryPlugin; + return this; + } + + @Override + public ClusterDataImplBuilder brokerClientSslFactoryPluginParams(String sslFactoryPluginParams) { + this.brokerClientSslFactoryPluginParams = sslFactoryPluginParams; + return this; + } public ClusterDataImplBuilder listenerName(String listenerName) { this.listenerName = listenerName; @@ -375,6 +401,8 @@ public ClusterDataImpl build() { brokerClientTrustCertsFilePath, brokerClientKeyFilePath, brokerClientCertificateFilePath, + brokerClientSslFactoryPlugin, + brokerClientSslFactoryPluginParams, listenerName); } } diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/DefaultPulsarSslFactory.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/DefaultPulsarSslFactory.java new file mode 100644 index 00000000000000..9be16f835b28b7 --- /dev/null +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/util/DefaultPulsarSslFactory.java @@ -0,0 +1,366 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.common.util; + +import io.netty.buffer.ByteBufAllocator; +import io.netty.handler.ssl.SslContext; +import io.netty.handler.ssl.SslProvider; +import java.io.IOException; +import java.security.GeneralSecurityException; +import java.util.concurrent.atomic.AtomicReference; +import javax.annotation.concurrent.NotThreadSafe; +import javax.net.ssl.SSLContext; +import javax.net.ssl.SSLEngine; +import javax.net.ssl.SSLParameters; +import org.apache.commons.lang3.StringUtils; +import org.apache.pulsar.client.api.AuthenticationDataProvider; +import org.apache.pulsar.client.api.KeyStoreParams; +import org.apache.pulsar.common.util.keystoretls.KeyStoreSSLContext; + +/** + * Default Implementation of {@link PulsarSslFactory}. This factory loads file based certificates to create SSLContext + * and SSL Engines. This class is not thread safe. It has been integrated into the pulsar code base as a single writer, + * multiple readers pattern. + */ +@NotThreadSafe +public class DefaultPulsarSslFactory implements PulsarSslFactory { + + private PulsarSslConfiguration config; + private final AtomicReference internalSslContext = new AtomicReference<>(); + private final AtomicReference internalNettySslContext = new AtomicReference<>(); + + protected FileModifiedTimeUpdater tlsKeyStore; + protected FileModifiedTimeUpdater tlsTrustStore; + protected FileModifiedTimeUpdater tlsTrustCertsFilePath; + protected FileModifiedTimeUpdater tlsCertificateFilePath; + protected FileModifiedTimeUpdater tlsKeyFilePath; + protected AuthenticationDataProvider authData; + protected boolean isTlsTrustStoreStreamProvided; + protected final String[] defaultSslEnabledProtocols = {"TLSv1.3", "TLSv1.2"}; + protected String tlsKeystoreType; + protected String tlsKeystorePath; + protected String tlsKeystorePassword; + + /** + * Initializes the DefaultPulsarSslFactory. + * + * @param config {@link PulsarSslConfiguration} object required for initialization. + * + */ + @Override + public void initialize(PulsarSslConfiguration config) { + this.config = config; + AuthenticationDataProvider authData = this.config.getAuthData(); + if (this.config.isTlsEnabledWithKeystore()) { + if (authData != null && authData.hasDataForTls()) { + KeyStoreParams authParams = authData.getTlsKeyStoreParams(); + if (authParams != null) { + this.tlsKeystoreType = authParams.getKeyStoreType(); + this.tlsKeystorePath = authParams.getKeyStorePath(); + this.tlsKeystorePassword = authParams.getKeyStorePassword(); + } + } + if (this.tlsKeystoreType == null) { + this.tlsKeystoreType = this.config.getTlsKeyStoreType(); + } + if (this.tlsKeystorePath == null) { + this.tlsKeystorePath = this.config.getTlsKeyStorePath(); + } + if (this.tlsKeystorePassword == null) { + this.tlsKeystorePassword = this.config.getTlsKeyStorePassword(); + } + this.tlsKeyStore = new FileModifiedTimeUpdater(this.tlsKeystorePath); + this.tlsTrustStore = new FileModifiedTimeUpdater(this.config.getTlsTrustStorePath()); + } else { + if (authData != null && authData.hasDataForTls()) { + if (authData.getTlsTrustStoreStream() != null) { + this.isTlsTrustStoreStreamProvided = true; + } else { + this.tlsTrustCertsFilePath = new FileModifiedTimeUpdater(this.config.getTlsTrustCertsFilePath()); + } + this.authData = authData; + } else { + this.tlsCertificateFilePath = new FileModifiedTimeUpdater(this.config.getTlsCertificateFilePath()); + this.tlsTrustCertsFilePath = new FileModifiedTimeUpdater(this.config.getTlsTrustCertsFilePath()); + this.tlsKeyFilePath = new FileModifiedTimeUpdater(this.config.getTlsKeyFilePath()); + } + } + } + + /** + * Creates a Client {@link SSLEngine} utilizing the peer hostname, peer port and {@link PulsarSslConfiguration} + * object provided during initialization. + * + * @param peerHost the name of the peer host + * @param peerPort the port number of the peer + * @return {@link SSLEngine} + */ + @Override + public SSLEngine createClientSslEngine(ByteBufAllocator buf, String peerHost, int peerPort) { + return createSSLEngine(buf, peerHost, peerPort, NetworkMode.CLIENT); + } + + /** + * Creates a Server {@link SSLEngine} utilizing the {@link PulsarSslConfiguration} object provided during + * initialization. + * + * @return {@link SSLEngine} + */ + @Override + public SSLEngine createServerSslEngine(ByteBufAllocator buf) { + return createSSLEngine(buf, "", 0, NetworkMode.SERVER); + } + + /** + * Returns a boolean value based on if the underlying certificate files have been modified since it was last read. + * + * @return {@code true} if the underlying certificates have been modified indicating that + * the SSL Context should be refreshed. + */ + @Override + public boolean needsUpdate() { + if (this.config.isTlsEnabledWithKeystore()) { + return (this.tlsKeyStore != null && this.tlsKeyStore.checkAndRefresh()) + || (this.tlsTrustStore != null && this.tlsTrustStore.checkAndRefresh()); + } else { + if (this.authData != null && this.authData.hasDataForTls()) { + return true; + } else { + return this.tlsTrustCertsFilePath.checkAndRefresh() || this.tlsCertificateFilePath.checkAndRefresh() + || this.tlsKeyFilePath.checkAndRefresh(); + } + } + } + + /** + * Creates a {@link SSLContext} object and saves it internally. + * + * @throws Exception If there were any issues generating the {@link SSLContext} + */ + @Override + public void createInternalSslContext() throws Exception { + if (this.config.isTlsEnabledWithKeystore()) { + this.internalSslContext.set(buildKeystoreSslContext(this.config.isServerMode())); + } else { + if (this.config.isHttps()) { + this.internalSslContext.set(buildSslContext()); + } else { + this.internalNettySslContext.set(buildNettySslContext()); + } + } + } + + + /** + * Get the internally stored {@link SSLContext}. + * + * @return {@link SSLContext} + * @throws RuntimeException if the {@link SSLContext} object has not yet been initialized. + */ + @Override + public SSLContext getInternalSslContext() { + if (this.internalSslContext.get() == null) { + throw new RuntimeException("Internal SSL context is not initialized. " + + "Please call createInternalSslContext() first."); + } + return this.internalSslContext.get(); + } + + /** + * Get the internally stored {@link SslContext}. + * + * @return {@link SslContext} + * @throws RuntimeException if the {@link SslContext} object has not yet been initialized. + */ + public SslContext getInternalNettySslContext() { + if (this.internalNettySslContext.get() == null) { + throw new RuntimeException("Internal SSL context is not initialized. " + + "Please call createInternalSslContext() first."); + } + return this.internalNettySslContext.get(); + } + + private SSLContext buildKeystoreSslContext(boolean isServerMode) throws GeneralSecurityException, IOException { + KeyStoreSSLContext keyStoreSSLContext; + if (isServerMode) { + keyStoreSSLContext = KeyStoreSSLContext.createServerKeyStoreSslContext(this.config.getTlsProvider(), + this.tlsKeystoreType, this.tlsKeyStore.getFileName(), + this.tlsKeystorePassword, this.config.isAllowInsecureConnection(), + this.config.getTlsTrustStoreType(), this.tlsTrustStore.getFileName(), + this.config.getTlsTrustStorePassword(), this.config.isRequireTrustedClientCertOnConnect(), + this.config.getTlsCiphers(), this.config.getTlsProtocols()); + } else { + keyStoreSSLContext = KeyStoreSSLContext.createClientKeyStoreSslContext(this.config.getTlsProvider(), + this.tlsKeystoreType, this.tlsKeyStore.getFileName(), + this.tlsKeystorePassword, this.config.isAllowInsecureConnection(), + this.config.getTlsTrustStoreType(), this.tlsTrustStore.getFileName(), + this.config.getTlsTrustStorePassword(), this.config.getTlsCiphers(), + this.config.getTlsProtocols()); + } + return keyStoreSSLContext.createSSLContext(); + } + + private SSLContext buildSslContext() throws GeneralSecurityException { + if (this.authData != null && this.authData.hasDataForTls()) { + if (this.isTlsTrustStoreStreamProvided) { + return SecurityUtility.createSslContext(this.config.isAllowInsecureConnection(), + SecurityUtility.loadCertificatesFromPemStream(this.authData.getTlsTrustStoreStream()), + this.authData.getTlsCertificates(), + this.authData.getTlsPrivateKey(), + this.config.getTlsProvider()); + } else { + if (this.authData.getTlsCertificates() != null) { + return SecurityUtility.createSslContext(this.config.isAllowInsecureConnection(), + SecurityUtility.loadCertificatesFromPemFile(this.tlsTrustCertsFilePath.getFileName()), + this.authData.getTlsCertificates(), + this.authData.getTlsPrivateKey(), + this.config.getTlsProvider()); + } else { + return SecurityUtility.createSslContext(this.config.isAllowInsecureConnection(), + this.tlsTrustCertsFilePath.getFileName(), + this.authData.getTlsCertificateFilePath(), + this.authData.getTlsPrivateKeyFilePath(), + this.config.getTlsProvider() + ); + } + } + } else { + return SecurityUtility.createSslContext(this.config.isAllowInsecureConnection(), + this.tlsTrustCertsFilePath.getFileName(), + this.tlsCertificateFilePath.getFileName(), + this.tlsKeyFilePath.getFileName(), + this.config.getTlsProvider()); + } + } + + private SslContext buildNettySslContext() throws GeneralSecurityException, IOException { + SslProvider sslProvider = null; + if (StringUtils.isNotBlank(this.config.getTlsProvider())) { + sslProvider = SslProvider.valueOf(this.config.getTlsProvider()); + } + if (this.authData != null && this.authData.hasDataForTls()) { + if (this.isTlsTrustStoreStreamProvided) { + return SecurityUtility.createNettySslContextForClient(sslProvider, + this.config.isAllowInsecureConnection(), + this.authData.getTlsTrustStoreStream(), + this.authData.getTlsCertificates(), + this.authData.getTlsPrivateKey(), + this.config.getTlsCiphers(), + this.config.getTlsProtocols()); + } else { + if (this.authData.getTlsCertificates() != null) { + return SecurityUtility.createNettySslContextForClient(sslProvider, + this.config.isAllowInsecureConnection(), + this.tlsTrustCertsFilePath.getFileName(), + this.authData.getTlsCertificates(), + this.authData.getTlsPrivateKey(), + this.config.getTlsCiphers(), + this.config.getTlsProtocols()); + } else { + return SecurityUtility.createNettySslContextForClient(sslProvider, + this.config.isAllowInsecureConnection(), + this.tlsTrustCertsFilePath.getFileName(), + this.authData.getTlsCertificateFilePath(), + this.authData.getTlsPrivateKeyFilePath(), + this.config.getTlsCiphers(), + this.config.getTlsProtocols()); + } + } + } else { + if (this.config.isServerMode()) { + return SecurityUtility.createNettySslContextForServer(sslProvider, + this.config.isAllowInsecureConnection(), + this.tlsTrustCertsFilePath.getFileName(), + this.tlsCertificateFilePath.getFileName(), + this.tlsKeyFilePath.getFileName(), + this.config.getTlsCiphers(), + this.config.getTlsProtocols(), + this.config.isRequireTrustedClientCertOnConnect()); + } else { + return SecurityUtility.createNettySslContextForClient(sslProvider, + this.config.isAllowInsecureConnection(), + this.tlsTrustCertsFilePath.getFileName(), + this.tlsCertificateFilePath.getFileName(), + this.tlsKeyFilePath.getFileName(), + this.config.getTlsCiphers(), + this.config.getTlsProtocols()); + } + } + } + + private SSLEngine createSSLEngine(ByteBufAllocator buf, String peerHost, int peerPort, NetworkMode mode) { + SSLEngine sslEngine; + SSLParameters sslParams; + SSLContext sslContext = this.internalSslContext.get(); + SslContext nettySslContext = this.internalNettySslContext.get(); + validateSslContext(sslContext, nettySslContext); + if (mode == NetworkMode.CLIENT) { + if (sslContext != null) { + sslEngine = sslContext.createSSLEngine(peerHost, peerPort); + } else { + sslEngine = nettySslContext.newEngine(buf, peerHost, peerPort); + } + sslEngine.setUseClientMode(true); + sslParams = sslEngine.getSSLParameters(); + } else { + if (sslContext != null) { + sslEngine = sslContext.createSSLEngine(); + } else { + sslEngine = nettySslContext.newEngine(buf); + } + sslEngine.setUseClientMode(false); + sslParams = sslEngine.getSSLParameters(); + if (this.config.isRequireTrustedClientCertOnConnect()) { + sslParams.setNeedClientAuth(true); + } else { + sslParams.setWantClientAuth(true); + } + } + if (this.config.getTlsProtocols() != null && !this.config.getTlsProtocols().isEmpty()) { + sslParams.setProtocols(this.config.getTlsProtocols().toArray(new String[0])); + } else { + sslParams.setProtocols(defaultSslEnabledProtocols); + } + if (this.config.getTlsCiphers() != null && !this.config.getTlsCiphers().isEmpty()) { + sslParams.setCipherSuites(this.config.getTlsCiphers().toArray(new String[0])); + } + sslEngine.setSSLParameters(sslParams); + return sslEngine; + } + + private void validateSslContext(SSLContext sslContext, SslContext nettySslContext) { + if (sslContext == null && nettySslContext == null) { + throw new RuntimeException("Internal SSL context is not initialized. " + + "Please call createInternalSslContext() first."); + } + } + + /** + * Clean any resources that may have been created. + * @throws Exception if any resources failed to be cleaned. + */ + @Override + public void close() throws Exception { + // noop + } + + private enum NetworkMode { + CLIENT, SERVER + } +} \ No newline at end of file diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/DefaultSslContextBuilder.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/DefaultSslContextBuilder.java deleted file mode 100644 index ab5f41c6bbf8d7..00000000000000 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/util/DefaultSslContextBuilder.java +++ /dev/null @@ -1,76 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.common.util; - -import java.security.GeneralSecurityException; -import javax.net.ssl.SSLContext; - -@SuppressWarnings("checkstyle:JavadocType") -public class DefaultSslContextBuilder extends SslContextAutoRefreshBuilder { - private volatile SSLContext sslContext; - - protected final boolean tlsAllowInsecureConnection; - protected final FileModifiedTimeUpdater tlsTrustCertsFilePath, tlsCertificateFilePath, tlsKeyFilePath; - protected final boolean tlsRequireTrustedClientCertOnConnect; - private final String providerName; - - public DefaultSslContextBuilder(boolean allowInsecure, String trustCertsFilePath, String certificateFilePath, - String keyFilePath, boolean requireTrustedClientCertOnConnect, - long certRefreshInSec) { - super(certRefreshInSec); - this.tlsAllowInsecureConnection = allowInsecure; - this.tlsTrustCertsFilePath = new FileModifiedTimeUpdater(trustCertsFilePath); - this.tlsCertificateFilePath = new FileModifiedTimeUpdater(certificateFilePath); - this.tlsKeyFilePath = new FileModifiedTimeUpdater(keyFilePath); - this.tlsRequireTrustedClientCertOnConnect = requireTrustedClientCertOnConnect; - this.providerName = null; - } - - public DefaultSslContextBuilder(boolean allowInsecure, String trustCertsFilePath, String certificateFilePath, - String keyFilePath, boolean requireTrustedClientCertOnConnect, - long certRefreshInSec, String providerName) { - super(certRefreshInSec); - this.tlsAllowInsecureConnection = allowInsecure; - this.tlsTrustCertsFilePath = new FileModifiedTimeUpdater(trustCertsFilePath); - this.tlsCertificateFilePath = new FileModifiedTimeUpdater(certificateFilePath); - this.tlsKeyFilePath = new FileModifiedTimeUpdater(keyFilePath); - this.tlsRequireTrustedClientCertOnConnect = requireTrustedClientCertOnConnect; - this.providerName = providerName; - } - - @Override - public synchronized SSLContext update() throws GeneralSecurityException { - this.sslContext = SecurityUtility.createSslContext(tlsAllowInsecureConnection, - tlsTrustCertsFilePath.getFileName(), tlsCertificateFilePath.getFileName(), - tlsKeyFilePath.getFileName(), this.providerName); - return this.sslContext; - } - - @Override - public SSLContext getSslContext() { - return this.sslContext; - } - - @Override - public boolean needUpdate() { - return tlsTrustCertsFilePath.checkAndRefresh() - || tlsCertificateFilePath.checkAndRefresh() - || tlsKeyFilePath.checkAndRefresh(); - } -} diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/NettyClientSslContextRefresher.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/NettyClientSslContextRefresher.java deleted file mode 100644 index 828cf35121d7eb..00000000000000 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/util/NettyClientSslContextRefresher.java +++ /dev/null @@ -1,96 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.common.util; - -import io.netty.handler.ssl.SslContext; -import io.netty.handler.ssl.SslProvider; -import java.io.FileNotFoundException; -import java.io.IOException; -import java.security.GeneralSecurityException; -import java.security.cert.X509Certificate; -import java.util.Set; -import javax.net.ssl.SSLException; -import lombok.extern.slf4j.Slf4j; -import org.apache.pulsar.client.api.AuthenticationDataProvider; - -/** - * SSL context builder for Netty Client side. - */ -@Slf4j -public class NettyClientSslContextRefresher extends SslContextAutoRefreshBuilder { - private volatile SslContext sslNettyContext; - private final boolean tlsAllowInsecureConnection; - protected final FileModifiedTimeUpdater tlsTrustCertsFilePath; - protected final FileModifiedTimeUpdater tlsCertsFilePath; - protected final FileModifiedTimeUpdater tlsPrivateKeyFilePath; - private final AuthenticationDataProvider authData; - private final SslProvider sslProvider; - private final Set ciphers; - private final Set protocols; - - public NettyClientSslContextRefresher(SslProvider sslProvider, boolean allowInsecure, - String trustCertsFilePath, - AuthenticationDataProvider authData, - Set ciphers, - Set protocols, - long delayInSeconds) { - super(delayInSeconds); - this.tlsAllowInsecureConnection = allowInsecure; - this.tlsTrustCertsFilePath = new FileModifiedTimeUpdater(trustCertsFilePath); - this.authData = authData; - this.tlsCertsFilePath = new FileModifiedTimeUpdater( - authData != null ? authData.getTlsCertificateFilePath() : null); - this.tlsPrivateKeyFilePath = new FileModifiedTimeUpdater( - authData != null ? authData.getTlsPrivateKeyFilePath() : null); - this.sslProvider = sslProvider; - this.ciphers = ciphers; - this.protocols = protocols; - } - - @Override - public synchronized SslContext update() - throws SSLException, FileNotFoundException, GeneralSecurityException, IOException { - if (authData != null && authData.hasDataForTls()) { - this.sslNettyContext = authData.getTlsTrustStoreStream() == null - ? SecurityUtility.createNettySslContextForClient(this.sslProvider, this.tlsAllowInsecureConnection, - tlsTrustCertsFilePath.getFileName(), (X509Certificate[]) authData.getTlsCertificates(), - authData.getTlsPrivateKey(), this.ciphers, this.protocols) - : SecurityUtility.createNettySslContextForClient(this.sslProvider, this.tlsAllowInsecureConnection, - authData.getTlsTrustStoreStream(), (X509Certificate[]) authData.getTlsCertificates(), - authData.getTlsPrivateKey(), this.ciphers, this.protocols); - } else { - this.sslNettyContext = - SecurityUtility.createNettySslContextForClient(this.sslProvider, this.tlsAllowInsecureConnection, - this.tlsTrustCertsFilePath.getFileName(), this.ciphers, this.protocols); - } - return this.sslNettyContext; - } - - @Override - public SslContext getSslContext() { - return this.sslNettyContext; - } - - @Override - public boolean needUpdate() { - return tlsTrustCertsFilePath.checkAndRefresh() || tlsCertsFilePath.checkAndRefresh() - || tlsPrivateKeyFilePath.checkAndRefresh(); - - } -} diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/NettyServerSslContextBuilder.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/NettyServerSslContextBuilder.java deleted file mode 100644 index eda61be3f87c2f..00000000000000 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/util/NettyServerSslContextBuilder.java +++ /dev/null @@ -1,89 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.common.util; - -import io.netty.handler.ssl.SslContext; -import io.netty.handler.ssl.SslProvider; -import java.io.FileNotFoundException; -import java.io.IOException; -import java.security.GeneralSecurityException; -import java.util.Set; -import javax.net.ssl.SSLException; - -/** - * SSL context builder for Netty Server side. - */ -public class NettyServerSslContextBuilder extends SslContextAutoRefreshBuilder { - private volatile SslContext sslNettyContext; - - protected final boolean tlsAllowInsecureConnection; - protected final FileModifiedTimeUpdater tlsTrustCertsFilePath, tlsCertificateFilePath, tlsKeyFilePath; - protected final Set tlsCiphers; - protected final Set tlsProtocols; - protected final boolean tlsRequireTrustedClientCertOnConnect; - protected final SslProvider sslProvider; - - public NettyServerSslContextBuilder(boolean allowInsecure, String trustCertsFilePath, - String certificateFilePath, - String keyFilePath, Set ciphers, Set protocols, - boolean requireTrustedClientCertOnConnect, - long delayInSeconds) { - this(null, allowInsecure, trustCertsFilePath, certificateFilePath, keyFilePath, ciphers, protocols, - requireTrustedClientCertOnConnect, delayInSeconds); - } - - public NettyServerSslContextBuilder(SslProvider sslProvider, boolean allowInsecure, String trustCertsFilePath, - String certificateFilePath, - String keyFilePath, Set ciphers, Set protocols, - boolean requireTrustedClientCertOnConnect, - long delayInSeconds) { - super(delayInSeconds); - this.tlsAllowInsecureConnection = allowInsecure; - this.tlsTrustCertsFilePath = new FileModifiedTimeUpdater(trustCertsFilePath); - this.tlsCertificateFilePath = new FileModifiedTimeUpdater(certificateFilePath); - this.tlsKeyFilePath = new FileModifiedTimeUpdater(keyFilePath); - this.tlsCiphers = ciphers; - this.tlsProtocols = protocols; - this.tlsRequireTrustedClientCertOnConnect = requireTrustedClientCertOnConnect; - this.sslProvider = sslProvider; - } - - @Override - public synchronized SslContext update() - throws SSLException, FileNotFoundException, GeneralSecurityException, IOException { - this.sslNettyContext = - SecurityUtility.createNettySslContextForServer(this.sslProvider, tlsAllowInsecureConnection, - tlsTrustCertsFilePath.getFileName(), tlsCertificateFilePath.getFileName(), - tlsKeyFilePath.getFileName(), - tlsCiphers, tlsProtocols, tlsRequireTrustedClientCertOnConnect); - return this.sslNettyContext; - } - - @Override - public SslContext getSslContext() { - return this.sslNettyContext; - } - - @Override - public boolean needUpdate() { - return tlsTrustCertsFilePath.checkAndRefresh() - || tlsCertificateFilePath.checkAndRefresh() - || tlsKeyFilePath.checkAndRefresh(); - } -} diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/PulsarSslConfiguration.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/PulsarSslConfiguration.java new file mode 100644 index 00000000000000..f71888009bf4ca --- /dev/null +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/util/PulsarSslConfiguration.java @@ -0,0 +1,167 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.common.util; + +import io.swagger.annotations.ApiModelProperty; +import java.io.Serializable; +import java.util.Set; +import lombok.Builder; +import lombok.Getter; +import lombok.ToString; +import org.apache.pulsar.client.api.AuthenticationDataProvider; + +/** + * Pulsar SSL Configuration Object to be used by all Pulsar Server and Client Components. + */ +@Builder +@Getter +@ToString +public class PulsarSslConfiguration implements Serializable, Cloneable { + + private static final long serialVersionUID = 1L; + + @ApiModelProperty( + name = "tlsCiphers", + value = "TLS ciphers to be used", + required = true + ) + private Set tlsCiphers; + + @ApiModelProperty( + name = "tlsProtocols", + value = "TLS protocols to be used", + required = true + ) + private Set tlsProtocols; + + @ApiModelProperty( + name = "allowInsecureConnection", + value = "Insecure Connections are allowed", + required = true + ) + private boolean allowInsecureConnection; + + @ApiModelProperty( + name = "requireTrustedClientCertOnConnect", + value = "Require trusted client certificate on connect", + required = true + ) + private boolean requireTrustedClientCertOnConnect; + + @ApiModelProperty( + name = "authData", + value = "Authentication Data Provider utilized by the Client for identification" + ) + private AuthenticationDataProvider authData; + + @ApiModelProperty( + name = "tlsCustomParams", + value = "Custom Parameters required by Pulsar SSL factory plugins" + ) + private String tlsCustomParams; + + @ApiModelProperty( + name = "tlsProvider", + value = "TLS Provider to be used" + ) + private String tlsProvider; + + @ApiModelProperty( + name = "tlsTrustStoreType", + value = "TLS Trust Store Type to be used" + ) + private String tlsTrustStoreType; + + @ApiModelProperty( + name = "tlsTrustStorePath", + value = "TLS Trust Store Path" + ) + private String tlsTrustStorePath; + + @ApiModelProperty( + name = "tlsTrustStorePassword", + value = "TLS Trust Store Password" + ) + private String tlsTrustStorePassword; + + @ApiModelProperty( + name = "tlsTrustCertsFilePath", + value = " TLS Trust certificates file path" + ) + private String tlsTrustCertsFilePath; + + @ApiModelProperty( + name = "tlsCertificateFilePath", + value = "Path for the TLS Certificate file" + ) + private String tlsCertificateFilePath; + + @ApiModelProperty( + name = "tlsKeyFilePath", + value = "Path for TLS Private key file" + ) + private String tlsKeyFilePath; + + @ApiModelProperty( + name = "tlsKeyStoreType", + value = "TLS Key Store Type to be used" + ) + private String tlsKeyStoreType; + + @ApiModelProperty( + name = "tlsKeyStorePath", + value = "TLS Key Store Path" + ) + private String tlsKeyStorePath; + + @ApiModelProperty( + name = "tlsKeyStorePassword", + value = "TLS Key Store Password" + ) + private String tlsKeyStorePassword; + + @ApiModelProperty( + name = "isTlsEnabledWithKeystore", + value = "TLS configuration enabled with key store configs" + ) + private boolean tlsEnabledWithKeystore; + + @ApiModelProperty( + name = "isServerMode", + value = "Is the SSL Configuration for a Server or Client", + required = true + ) + private boolean serverMode; + + @ApiModelProperty( + name = "isHttps", + value = "Is the SSL Configuration for a Http client or Server" + ) + private boolean isHttps; + + @Override + public PulsarSslConfiguration clone() { + try { + return (PulsarSslConfiguration) super.clone(); + } catch (CloneNotSupportedException e) { + throw new RuntimeException("Failed to clone PulsarSslConfiguration", e); + } + } + +} \ No newline at end of file diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/PulsarSslFactory.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/PulsarSslFactory.java new file mode 100644 index 00000000000000..bccbbbe5b25161 --- /dev/null +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/util/PulsarSslFactory.java @@ -0,0 +1,106 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.common.util; + +import io.netty.buffer.ByteBufAllocator; +import io.netty.handler.ssl.SslContext; +import javax.net.ssl.SSLContext; +import javax.net.ssl.SSLEngine; + +/** + * Factory for generating SSL Context and SSL Engine using {@link PulsarSslConfiguration}. + */ +public interface PulsarSslFactory extends AutoCloseable { + + /** + * Initializes the PulsarSslFactory. + * @param config {@link PulsarSslConfiguration} object required for initialization + */ + void initialize(PulsarSslConfiguration config); + + /** + * Creates a Client {@link SSLEngine} utilizing {@link ByteBufAllocator} object, the peer hostname, peer port and + * {@link PulsarSslConfiguration} object provided during initialization. + * + * @param buf The ByteBufAllocator required for netty connections. This can be passed as {@code null} if utilized + * for web connections. + * @param peerHost the name of the peer host + * @param peerPort the port number of the peer + * @return {@link SSLEngine} + */ + SSLEngine createClientSslEngine(ByteBufAllocator buf, String peerHost, int peerPort); + + /** + * Creates a Server {@link SSLEngine} utilizing the {@link ByteBufAllocator} object and + * {@link PulsarSslConfiguration} object provided during initialization. + * + * @param buf The ByteBufAllocator required for netty connections. This can be passed as {@code null} if utilized + * for web connections. + * @return {@link SSLEngine} + */ + SSLEngine createServerSslEngine(ByteBufAllocator buf); + + /** + * Returns a boolean value indicating {@link SSLContext} or {@link SslContext} should be refreshed. + * + * @return {@code true} if {@link SSLContext} or {@link SslContext} should be refreshed. + */ + boolean needsUpdate(); + + /** + * Update the internal {@link SSLContext} or {@link SslContext}. + * @throws Exception if there are any issues generating the new {@link SSLContext} or {@link SslContext} + */ + default void update() throws Exception { + if (this.needsUpdate()) { + this.createInternalSslContext(); + } + } + + /** + * Creates the following: + * 1. {@link SslContext} if netty connections are being created for Non-Keystore based TLS configurations. + * 2. {@link SSLContext} if netty connections are being created for Keystore based TLS configurations. It will + * also create it for all web connections irrespective of it being Keystore or Non-Keystore based TLS + * configurations. + * + * @throws Exception if there are any issues creating the new {@link SSLContext} or {@link SslContext} + */ + void createInternalSslContext() throws Exception; + + /** + * Get the internally stored {@link SSLContext}. It will be used in the following scenarios: + * 1. Netty connection creations for keystore based TLS configurations + * 2. All Web connections + * + * @return {@link SSLContext} + * @throws RuntimeException if the {@link SSLContext} object has not yet been initialized. + */ + SSLContext getInternalSslContext(); + + /** + * Get the internally stored {@link SslContext}. It will be used to create Netty Connections for non-keystore based + * tls configurations. + * + * @return {@link SslContext} + * @throws RuntimeException if the {@link SslContext} object has not yet been initialized. + */ + SslContext getInternalNettySslContext(); + +} \ No newline at end of file diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/SslContextAutoRefreshBuilder.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/SslContextAutoRefreshBuilder.java deleted file mode 100644 index 8c8f5800464489..00000000000000 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/util/SslContextAutoRefreshBuilder.java +++ /dev/null @@ -1,100 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.common.util; - -import java.io.IOException; -import java.security.GeneralSecurityException; -import java.util.concurrent.TimeUnit; -import lombok.extern.slf4j.Slf4j; - -/** - * Auto refresher and builder of SSLContext. - * - * @param - * type of SSLContext - */ -@Slf4j -public abstract class SslContextAutoRefreshBuilder { - protected final long refreshTime; - protected long lastRefreshTime; - - public SslContextAutoRefreshBuilder( - long certRefreshInSec) { - this.refreshTime = TimeUnit.SECONDS.toMillis(certRefreshInSec); - this.lastRefreshTime = -1; - - if (log.isDebugEnabled()) { - log.debug("Certs will be refreshed every {} seconds", certRefreshInSec); - } - } - - /** - * updates and returns cached SSLContext. - * - * @return - * @throws GeneralSecurityException - * @throws IOException - */ - protected abstract T update() throws GeneralSecurityException, IOException; - - /** - * Returns cached SSLContext. - * - * @return - */ - protected abstract T getSslContext(); - - /** - * Returns whether the key files modified after a refresh time, and context need update. - * - * @return true if files modified - */ - protected abstract boolean needUpdate(); - - /** - * It updates SSLContext at every configured refresh time and returns updated SSLContext. - * - * @return - */ - public T get() { - T ctx = getSslContext(); - if (ctx == null) { - try { - update(); - lastRefreshTime = System.currentTimeMillis(); - return getSslContext(); - } catch (GeneralSecurityException | IOException e) { - log.error("Exception while trying to refresh ssl Context {}", e.getMessage(), e); - } - } else { - long now = System.currentTimeMillis(); - if (refreshTime <= 0 || now > (lastRefreshTime + refreshTime)) { - if (needUpdate()) { - try { - ctx = update(); - lastRefreshTime = now; - } catch (GeneralSecurityException | IOException e) { - log.error("Exception while trying to refresh ssl Context {} ", e.getMessage(), e); - } - } - } - } - return ctx; - } -} diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/keystoretls/NetSslContextBuilder.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/keystoretls/NetSslContextBuilder.java deleted file mode 100644 index 3d4d4e72546ea1..00000000000000 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/util/keystoretls/NetSslContextBuilder.java +++ /dev/null @@ -1,90 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.common.util.keystoretls; - -import java.io.IOException; -import java.security.GeneralSecurityException; -import javax.net.ssl.SSLContext; -import org.apache.pulsar.common.util.FileModifiedTimeUpdater; -import org.apache.pulsar.common.util.SslContextAutoRefreshBuilder; - -/** - * Similar to `DefaultSslContextBuilder`, which build `javax.net.ssl.SSLContext` for web service. - */ -public class NetSslContextBuilder extends SslContextAutoRefreshBuilder { - private volatile SSLContext sslContext; - - protected final boolean tlsAllowInsecureConnection; - protected final boolean tlsRequireTrustedClientCertOnConnect; - - protected final String tlsProvider; - protected final String tlsKeyStoreType; - protected final String tlsKeyStorePassword; - protected final FileModifiedTimeUpdater tlsKeyStore; - protected final String tlsTrustStoreType; - protected final String tlsTrustStorePassword; - protected final FileModifiedTimeUpdater tlsTrustStore; - - public NetSslContextBuilder(String sslProviderString, - String keyStoreTypeString, - String keyStore, - String keyStorePasswordPath, - boolean allowInsecureConnection, - String trustStoreTypeString, - String trustStore, - String trustStorePasswordPath, - boolean requireTrustedClientCertOnConnect, - long certRefreshInSec) { - super(certRefreshInSec); - - this.tlsAllowInsecureConnection = allowInsecureConnection; - this.tlsProvider = sslProviderString; - this.tlsKeyStoreType = keyStoreTypeString; - this.tlsKeyStore = new FileModifiedTimeUpdater(keyStore); - this.tlsKeyStorePassword = keyStorePasswordPath; - - this.tlsTrustStoreType = trustStoreTypeString; - this.tlsTrustStore = new FileModifiedTimeUpdater(trustStore); - this.tlsTrustStorePassword = trustStorePasswordPath; - - this.tlsRequireTrustedClientCertOnConnect = requireTrustedClientCertOnConnect; - } - - @Override - public synchronized SSLContext update() - throws GeneralSecurityException, IOException { - this.sslContext = KeyStoreSSLContext.createServerSslContext(tlsProvider, - tlsKeyStoreType, tlsKeyStore.getFileName(), tlsKeyStorePassword, - tlsAllowInsecureConnection, - tlsTrustStoreType, tlsTrustStore.getFileName(), tlsTrustStorePassword, - tlsRequireTrustedClientCertOnConnect); - return this.sslContext; - } - - @Override - public SSLContext getSslContext() { - return this.sslContext; - } - - @Override - public boolean needUpdate() { - return tlsKeyStore.checkAndRefresh() - || tlsTrustStore.checkAndRefresh(); - } -} diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/keystoretls/NettySSLContextAutoRefreshBuilder.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/keystoretls/NettySSLContextAutoRefreshBuilder.java deleted file mode 100644 index 6d0cfb108bd0ec..00000000000000 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/util/keystoretls/NettySSLContextAutoRefreshBuilder.java +++ /dev/null @@ -1,155 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pulsar.common.util.keystoretls; - -import java.io.IOException; -import java.security.GeneralSecurityException; -import java.util.Set; -import org.apache.pulsar.client.api.AuthenticationDataProvider; -import org.apache.pulsar.client.api.KeyStoreParams; -import org.apache.pulsar.common.util.FileModifiedTimeUpdater; -import org.apache.pulsar.common.util.SslContextAutoRefreshBuilder; - -/** - * SSL context builder for Netty. - */ -public class NettySSLContextAutoRefreshBuilder extends SslContextAutoRefreshBuilder { - private volatile KeyStoreSSLContext keyStoreSSLContext; - - protected final boolean tlsAllowInsecureConnection; - protected final Set tlsCiphers; - protected final Set tlsProtocols; - protected boolean tlsRequireTrustedClientCertOnConnect; - - protected final String tlsProvider; - protected final String tlsTrustStoreType; - protected final String tlsTrustStorePassword; - protected final FileModifiedTimeUpdater tlsTrustStore; - - // client context not need keystore at start time, keyStore is passed in by authData. - protected String tlsKeyStoreType; - protected String tlsKeyStorePassword; - protected FileModifiedTimeUpdater tlsKeyStore; - - protected final boolean isServer; - - // for server - public NettySSLContextAutoRefreshBuilder(String sslProviderString, - String keyStoreTypeString, - String keyStore, - String keyStorePassword, - boolean allowInsecureConnection, - String trustStoreTypeString, - String trustStore, - String trustStorePassword, - boolean requireTrustedClientCertOnConnect, - Set ciphers, - Set protocols, - long certRefreshInSec) { - super(certRefreshInSec); - - this.tlsAllowInsecureConnection = allowInsecureConnection; - this.tlsProvider = sslProviderString; - - this.tlsKeyStoreType = keyStoreTypeString; - this.tlsKeyStore = new FileModifiedTimeUpdater(keyStore); - this.tlsKeyStorePassword = keyStorePassword; - - this.tlsTrustStoreType = trustStoreTypeString; - this.tlsTrustStore = new FileModifiedTimeUpdater(trustStore); - this.tlsTrustStorePassword = trustStorePassword; - - this.tlsRequireTrustedClientCertOnConnect = requireTrustedClientCertOnConnect; - this.tlsCiphers = ciphers; - this.tlsProtocols = protocols; - - this.isServer = true; - } - - // for client - public NettySSLContextAutoRefreshBuilder(String sslProviderString, - boolean allowInsecureConnection, - String trustStoreTypeString, - String trustStore, - String trustStorePassword, - String keyStoreTypeString, - String keyStore, - String keyStorePassword, - Set ciphers, - Set protocols, - long certRefreshInSec, - AuthenticationDataProvider authData) { - super(certRefreshInSec); - - this.tlsAllowInsecureConnection = allowInsecureConnection; - this.tlsProvider = sslProviderString; - - if (authData != null) { - KeyStoreParams authParams = authData.getTlsKeyStoreParams(); - if (authParams != null) { - keyStoreTypeString = authParams.getKeyStoreType(); - keyStore = authParams.getKeyStorePath(); - keyStorePassword = authParams.getKeyStorePassword(); - } - } - this.tlsKeyStoreType = keyStoreTypeString; - this.tlsKeyStore = new FileModifiedTimeUpdater(keyStore); - this.tlsKeyStorePassword = keyStorePassword; - - this.tlsTrustStoreType = trustStoreTypeString; - this.tlsTrustStore = new FileModifiedTimeUpdater(trustStore); - this.tlsTrustStorePassword = trustStorePassword; - - this.tlsCiphers = ciphers; - this.tlsProtocols = protocols; - - this.isServer = false; - } - - @Override - public synchronized KeyStoreSSLContext update() throws GeneralSecurityException, IOException { - if (isServer) { - this.keyStoreSSLContext = KeyStoreSSLContext.createServerKeyStoreSslContext(tlsProvider, - tlsKeyStoreType, tlsKeyStore.getFileName(), tlsKeyStorePassword, - tlsAllowInsecureConnection, - tlsTrustStoreType, tlsTrustStore.getFileName(), tlsTrustStorePassword, - tlsRequireTrustedClientCertOnConnect, tlsCiphers, tlsProtocols); - } else { - this.keyStoreSSLContext = KeyStoreSSLContext.createClientKeyStoreSslContext(tlsProvider, - tlsKeyStoreType, - tlsKeyStore.getFileName(), - tlsKeyStorePassword, - tlsAllowInsecureConnection, - tlsTrustStoreType, tlsTrustStore.getFileName(), tlsTrustStorePassword, - tlsCiphers, tlsProtocols); - } - return this.keyStoreSSLContext; - } - - @Override - public KeyStoreSSLContext getSslContext() { - return this.keyStoreSSLContext; - } - - @Override - public boolean needUpdate() { - return (tlsKeyStore != null && tlsKeyStore.checkAndRefresh()) - || (tlsTrustStore != null && tlsTrustStore.checkAndRefresh()); - } -} diff --git a/pulsar-common/src/test/java/org/apache/pulsar/common/util/DefaultPulsarSslFactoryTest.java b/pulsar-common/src/test/java/org/apache/pulsar/common/util/DefaultPulsarSslFactoryTest.java new file mode 100644 index 00000000000000..34cf3a97ce8031 --- /dev/null +++ b/pulsar-common/src/test/java/org/apache/pulsar/common/util/DefaultPulsarSslFactoryTest.java @@ -0,0 +1,282 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.common.util; + +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNotNull; +import static org.testng.Assert.assertThrows; +import com.google.common.io.Resources; +import io.netty.buffer.ByteBufAllocator; +import io.netty.handler.ssl.OpenSslEngine; +import java.io.File; +import java.util.Arrays; +import java.util.HashSet; +import java.util.Set; +import javax.net.ssl.SSLEngine; +import org.testng.annotations.Test; + +public class DefaultPulsarSslFactoryTest { + + public final static String KEYSTORE_FILE_PATH = + getAbsolutePath("certificate-authority/jks/broker.keystore.jks"); + public final static String TRUSTSTORE_FILE_PATH = + getAbsolutePath("certificate-authority/jks/broker.truststore.jks"); + public final static String TRUSTSTORE_NO_PASSWORD_FILE_PATH = + getAbsolutePath("certificate-authority/jks/broker.truststore.nopassword.jks"); + public final static String KEYSTORE_PW = "111111"; + public final static String TRUSTSTORE_PW = "111111"; + public final static String KEYSTORE_TYPE = "JKS"; + + public final static String CA_CERT_FILE_PATH = + getAbsolutePath("certificate-authority/certs/ca.cert.pem"); + public final static String CERT_FILE_PATH = + getAbsolutePath("certificate-authority/server-keys/broker.cert.pem"); + public final static String KEY_FILE_PATH = + getAbsolutePath("certificate-authority/server-keys/broker.key-pk8.pem"); + + @Test + public void sslContextCreationUsingKeystoreTest() throws Exception { + PulsarSslConfiguration pulsarSslConfiguration = PulsarSslConfiguration.builder() + .tlsEnabledWithKeystore(true) + .tlsKeyStoreType(KEYSTORE_TYPE) + .tlsKeyStorePath(KEYSTORE_FILE_PATH) + .tlsKeyStorePassword(KEYSTORE_PW) + .tlsTrustStorePath(TRUSTSTORE_FILE_PATH) + .tlsTrustStorePassword(TRUSTSTORE_PW) + .build(); + PulsarSslFactory pulsarSslFactory = new DefaultPulsarSslFactory(); + pulsarSslFactory.initialize(pulsarSslConfiguration); + pulsarSslFactory.createInternalSslContext(); + assertNotNull(pulsarSslFactory.getInternalSslContext()); + assertThrows(RuntimeException.class, pulsarSslFactory::getInternalNettySslContext); + } + + @Test + public void sslContextCreationUsingPasswordLessTruststoreTest() throws Exception { + PulsarSslConfiguration pulsarSslConfiguration = PulsarSslConfiguration.builder() + .tlsEnabledWithKeystore(true) + .tlsKeyStoreType(KEYSTORE_TYPE) + .tlsKeyStorePath(KEYSTORE_FILE_PATH) + .tlsKeyStorePassword(KEYSTORE_PW) + .tlsTrustStorePath(TRUSTSTORE_NO_PASSWORD_FILE_PATH) + .build(); + PulsarSslFactory pulsarSslFactory = new DefaultPulsarSslFactory(); + pulsarSslFactory.initialize(pulsarSslConfiguration); + pulsarSslFactory.createInternalSslContext(); + assertNotNull(pulsarSslFactory.getInternalSslContext()); + assertThrows(RuntimeException.class, pulsarSslFactory::getInternalNettySslContext); + } + + @Test + public void sslContextCreationUsingTlsCertsTest() throws Exception { + PulsarSslConfiguration pulsarSslConfiguration = PulsarSslConfiguration.builder() + .tlsCertificateFilePath(CERT_FILE_PATH) + .tlsKeyFilePath(KEY_FILE_PATH) + .tlsTrustCertsFilePath(CA_CERT_FILE_PATH) + .build(); + PulsarSslFactory pulsarSslFactory = new DefaultPulsarSslFactory(); + pulsarSslFactory.initialize(pulsarSslConfiguration); + pulsarSslFactory.createInternalSslContext(); + assertNotNull(pulsarSslFactory.getInternalNettySslContext()); + assertThrows(RuntimeException.class, pulsarSslFactory::getInternalSslContext); + } + + @Test + public void sslContextCreationUsingOnlyCACertsTest() throws Exception { + PulsarSslConfiguration pulsarSslConfiguration = PulsarSslConfiguration.builder() + .tlsTrustCertsFilePath(CA_CERT_FILE_PATH) + .build(); + PulsarSslFactory pulsarSslFactory = new DefaultPulsarSslFactory(); + pulsarSslFactory.initialize(pulsarSslConfiguration); + pulsarSslFactory.createInternalSslContext(); + assertNotNull(pulsarSslFactory.getInternalNettySslContext()); + assertThrows(RuntimeException.class, pulsarSslFactory::getInternalSslContext); + } + + @Test + public void sslContextCreationForWebClientConnections() throws Exception { + PulsarSslConfiguration pulsarSslConfiguration = PulsarSslConfiguration.builder() + .tlsCertificateFilePath(CERT_FILE_PATH) + .tlsKeyFilePath(KEY_FILE_PATH) + .tlsTrustCertsFilePath(CA_CERT_FILE_PATH) + .isHttps(true) + .build(); + PulsarSslFactory pulsarSslFactory = new DefaultPulsarSslFactory(); + pulsarSslFactory.initialize(pulsarSslConfiguration); + pulsarSslFactory.createInternalSslContext(); + assertNotNull(pulsarSslFactory.getInternalSslContext()); + assertThrows(RuntimeException.class, pulsarSslFactory::getInternalNettySslContext); + } + + @Test + public void sslContextCreationForWebServerConnectionsTest() throws Exception { + PulsarSslConfiguration pulsarSslConfiguration = PulsarSslConfiguration.builder() + .tlsCertificateFilePath(CERT_FILE_PATH) + .tlsKeyFilePath(KEY_FILE_PATH) + .tlsTrustCertsFilePath(CA_CERT_FILE_PATH) + .isHttps(true) + .serverMode(true) + .build(); + PulsarSslFactory pulsarSslFactory = new DefaultPulsarSslFactory(); + pulsarSslFactory.initialize(pulsarSslConfiguration); + pulsarSslFactory.createInternalSslContext(); + assertNotNull(pulsarSslFactory.getInternalSslContext()); + assertThrows(RuntimeException.class, pulsarSslFactory::getInternalNettySslContext); + } + + @Test + public void sslEngineCreationWithEnabledProtocolsAndCiphersForOpenSSLTest() throws Exception { + Set ciphers = new HashSet<>(); + ciphers.add("TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256"); + ciphers.add("TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256"); + Set protocols = new HashSet<>(); + protocols.add("TLSv1.2"); + protocols.add("TLSv1"); + PulsarSslConfiguration pulsarSslConfiguration = PulsarSslConfiguration.builder() + .tlsCertificateFilePath(CERT_FILE_PATH) + .tlsKeyFilePath(KEY_FILE_PATH) + .tlsTrustCertsFilePath(CA_CERT_FILE_PATH) + .tlsCiphers(ciphers) + .tlsProtocols(protocols) + .serverMode(true) + .build(); + PulsarSslFactory pulsarSslFactory = new DefaultPulsarSslFactory(); + pulsarSslFactory.initialize(pulsarSslConfiguration); + pulsarSslFactory.createInternalSslContext(); + assertNotNull(pulsarSslFactory.getInternalNettySslContext()); + assertThrows(RuntimeException.class, pulsarSslFactory::getInternalSslContext); + SSLEngine sslEngine = pulsarSslFactory.createServerSslEngine(ByteBufAllocator.DEFAULT); + /* Adding SSLv2Hello protocol only during expected checks as Netty adds it as part of the + ReferenceCountedOpenSslEngine's setEnabledProtocols method. The reasoning is that OpenSSL currently has no + way to disable this protocol. + */ + protocols.add("SSLv2Hello"); + assertEquals(new HashSet<>(Arrays.asList(sslEngine.getEnabledProtocols())), protocols); + assertEquals(new HashSet<>(Arrays.asList(sslEngine.getEnabledCipherSuites())), ciphers); + assert(!sslEngine.getUseClientMode()); + } + + @Test + public void sslEngineCreationWithEnabledProtocolsAndCiphersForWebTest() throws Exception { + Set ciphers = new HashSet<>(); + ciphers.add("TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256"); + ciphers.add("TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256"); + Set protocols = new HashSet<>(); + protocols.add("TLSv1.2"); + protocols.add("TLSv1"); + PulsarSslConfiguration pulsarSslConfiguration = PulsarSslConfiguration.builder() + .tlsCertificateFilePath(CERT_FILE_PATH) + .tlsKeyFilePath(KEY_FILE_PATH) + .tlsTrustCertsFilePath(CA_CERT_FILE_PATH) + .tlsCiphers(ciphers) + .tlsProtocols(protocols) + .isHttps(true) + .serverMode(true) + .build(); + PulsarSslFactory pulsarSslFactory = new DefaultPulsarSslFactory(); + pulsarSslFactory.initialize(pulsarSslConfiguration); + pulsarSslFactory.createInternalSslContext(); + assertNotNull(pulsarSslFactory.getInternalSslContext()); + assertThrows(RuntimeException.class, pulsarSslFactory::getInternalNettySslContext); + SSLEngine sslEngine = pulsarSslFactory.createServerSslEngine(ByteBufAllocator.DEFAULT); + assertEquals(new HashSet<>(Arrays.asList(sslEngine.getEnabledProtocols())), protocols); + assertEquals(new HashSet<>(Arrays.asList(sslEngine.getEnabledCipherSuites())), ciphers); + assert(!sslEngine.getUseClientMode()); + } + + @Test + public void sslContextCreationAsOpenSslTlsProvider() throws Exception { + PulsarSslConfiguration pulsarSslConfiguration = PulsarSslConfiguration.builder() + .tlsProvider("OPENSSL") + .tlsCertificateFilePath(CERT_FILE_PATH) + .tlsKeyFilePath(KEY_FILE_PATH) + .tlsTrustCertsFilePath(CA_CERT_FILE_PATH) + .serverMode(true) + .build(); + PulsarSslFactory pulsarSslFactory = new DefaultPulsarSslFactory(); + pulsarSslFactory.initialize(pulsarSslConfiguration); + pulsarSslFactory.createInternalSslContext(); + assertNotNull(pulsarSslFactory.getInternalNettySslContext()); + assertThrows(RuntimeException.class, pulsarSslFactory::getInternalSslContext); + SSLEngine sslEngine = pulsarSslFactory.createServerSslEngine(ByteBufAllocator.DEFAULT); + assert(sslEngine instanceof OpenSslEngine); + } + + @Test + public void sslContextCreationAsJDKTlsProvider() throws Exception { + PulsarSslConfiguration pulsarSslConfiguration = PulsarSslConfiguration.builder() + .tlsProvider("JDK") + .tlsCertificateFilePath(CERT_FILE_PATH) + .tlsKeyFilePath(KEY_FILE_PATH) + .tlsTrustCertsFilePath(CA_CERT_FILE_PATH) + .serverMode(true) + .build(); + PulsarSslFactory pulsarSslFactory = new DefaultPulsarSslFactory(); + pulsarSslFactory.initialize(pulsarSslConfiguration); + pulsarSslFactory.createInternalSslContext(); + assertNotNull(pulsarSslFactory.getInternalNettySslContext()); + assertThrows(RuntimeException.class, pulsarSslFactory::getInternalSslContext); + SSLEngine sslEngine = pulsarSslFactory.createServerSslEngine(ByteBufAllocator.DEFAULT); + assert (!(sslEngine instanceof OpenSslEngine)); + } + + @Test + public void sslEngineMutualAuthEnabledTest() throws Exception { + PulsarSslConfiguration pulsarSslConfiguration = PulsarSslConfiguration.builder() + .tlsProvider("JDK") + .tlsCertificateFilePath(CERT_FILE_PATH) + .tlsKeyFilePath(KEY_FILE_PATH) + .tlsTrustCertsFilePath(CA_CERT_FILE_PATH) + .requireTrustedClientCertOnConnect(true) + .serverMode(true) + .build(); + PulsarSslFactory pulsarSslFactory = new DefaultPulsarSslFactory(); + pulsarSslFactory.initialize(pulsarSslConfiguration); + pulsarSslFactory.createInternalSslContext(); + assertNotNull(pulsarSslFactory.getInternalNettySslContext()); + assertThrows(RuntimeException.class, pulsarSslFactory::getInternalSslContext); + SSLEngine sslEngine = pulsarSslFactory.createServerSslEngine(ByteBufAllocator.DEFAULT); + assert(sslEngine.getNeedClientAuth()); + } + + @Test + public void sslEngineSniClientTest() throws Exception { + PulsarSslConfiguration pulsarSslConfiguration = PulsarSslConfiguration.builder() + .tlsCertificateFilePath(CERT_FILE_PATH) + .tlsKeyFilePath(KEY_FILE_PATH) + .tlsTrustCertsFilePath(CA_CERT_FILE_PATH) + .build(); + PulsarSslFactory pulsarSslFactory = new DefaultPulsarSslFactory(); + pulsarSslFactory.initialize(pulsarSslConfiguration); + pulsarSslFactory.createInternalSslContext(); + assertNotNull(pulsarSslFactory.getInternalNettySslContext()); + assertThrows(RuntimeException.class, pulsarSslFactory::getInternalSslContext); + SSLEngine sslEngine = pulsarSslFactory.createClientSslEngine(ByteBufAllocator.DEFAULT, "localhost", + 1234); + assertEquals(sslEngine.getPeerHost(), "localhost"); + assertEquals(sslEngine.getPeerPort(), 1234); + } + + + + private static String getAbsolutePath(String resourceName) { + return new File(Resources.getResource(resourceName).getPath()).getAbsolutePath(); + } + +} diff --git a/pulsar-common/src/test/java/org/apache/pulsar/common/util/FileModifiedTimeUpdaterTest.java b/pulsar-common/src/test/java/org/apache/pulsar/common/util/FileModifiedTimeUpdaterTest.java index 1d950078d21c52..a41c9ceb2cbb70 100644 --- a/pulsar-common/src/test/java/org/apache/pulsar/common/util/FileModifiedTimeUpdaterTest.java +++ b/pulsar-common/src/test/java/org/apache/pulsar/common/util/FileModifiedTimeUpdaterTest.java @@ -48,6 +48,11 @@ public BasicAuthenticationData(String authParam) { this.authParam = authParam; } + @Override + public boolean hasDataForTls() { + return true; + } + public boolean hasDataFromCommand() { return true; } @@ -107,14 +112,17 @@ public void testNettyClientSslContextRefresher() throws Exception { createFile(Paths.get(certFile)); provider.certFilePath = certFile; provider.keyFilePath = certFile; - NettyClientSslContextRefresher refresher = new NettyClientSslContextRefresher(null, false, certFile, - provider, null, null, 1); - Thread.sleep(5000); - Paths.get(certFile).toFile().delete(); - // update the file - createFile(Paths.get(certFile)); - Awaitility.await().atMost(30, TimeUnit.SECONDS).until(()-> refresher.needUpdate()); - assertTrue(refresher.needUpdate()); + PulsarSslConfiguration pulsarSslConfiguration = PulsarSslConfiguration.builder() + .allowInsecureConnection(false).tlsTrustCertsFilePath(certFile).authData(provider).build(); + try (PulsarSslFactory pulsarSslFactory = new DefaultPulsarSslFactory()) { + pulsarSslFactory.initialize(pulsarSslConfiguration); + Thread.sleep(5000); + Paths.get(certFile).toFile().delete(); + // update the file + createFile(Paths.get(certFile)); + Awaitility.await().atMost(30, TimeUnit.SECONDS).until(pulsarSslFactory::needsUpdate); + assertTrue(pulsarSslFactory.needsUpdate()); + } } } diff --git a/pulsar-common/src/test/java/org/apache/pulsar/common/util/netty/SslContextTest.java b/pulsar-common/src/test/java/org/apache/pulsar/common/util/netty/SslContextTest.java index 303df5a0032781..120fee9319db70 100644 --- a/pulsar-common/src/test/java/org/apache/pulsar/common/util/netty/SslContextTest.java +++ b/pulsar-common/src/test/java/org/apache/pulsar/common/util/netty/SslContextTest.java @@ -21,16 +21,14 @@ import static org.testng.Assert.assertThrows; import com.google.common.io.Resources; import io.netty.handler.ssl.SslProvider; -import java.io.IOException; -import java.security.GeneralSecurityException; import java.util.HashSet; import java.util.Set; import javax.net.ssl.SSLException; import org.apache.pulsar.client.api.AuthenticationDataProvider; import org.apache.pulsar.client.api.KeyStoreParams; -import org.apache.pulsar.common.util.NettyClientSslContextRefresher; -import org.apache.pulsar.common.util.NettyServerSslContextBuilder; -import org.apache.pulsar.common.util.keystoretls.NettySSLContextAutoRefreshBuilder; +import org.apache.pulsar.common.util.DefaultPulsarSslFactory; +import org.apache.pulsar.common.util.PulsarSslConfiguration; +import org.apache.pulsar.common.util.PulsarSslFactory; import org.testng.annotations.DataProvider; import org.testng.annotations.Test; @@ -84,13 +82,22 @@ public static Object[] getCipher() { @Test(dataProvider = "cipherDataProvider") public void testServerKeyStoreSSLContext(Set cipher) throws Exception { - NettySSLContextAutoRefreshBuilder contextAutoRefreshBuilder = new NettySSLContextAutoRefreshBuilder( - null, - keyStoreType, brokerKeyStorePath, keyStorePassword, false, - keyStoreType, brokerTrustStorePath, keyStorePassword, - true, cipher, - null, 600); - contextAutoRefreshBuilder.update(); + PulsarSslConfiguration pulsarSslConfiguration = PulsarSslConfiguration.builder() + .tlsEnabledWithKeystore(true) + .tlsKeyStoreType(keyStoreType) + .tlsKeyStorePath(brokerKeyStorePath) + .tlsKeyStorePassword(keyStorePassword) + .allowInsecureConnection(false) + .tlsTrustStoreType(keyStoreType) + .tlsTrustStorePath(brokerTrustStorePath) + .tlsTrustStorePassword(keyStorePassword) + .requireTrustedClientCertOnConnect(true) + .tlsCiphers(cipher) + .build(); + try (PulsarSslFactory pulsarSslFactory = new DefaultPulsarSslFactory()) { + pulsarSslFactory.initialize(pulsarSslConfiguration); + pulsarSslFactory.createInternalSslContext(); + } } private static class ClientAuthenticationData implements AuthenticationDataProvider { @@ -102,45 +109,67 @@ public KeyStoreParams getTlsKeyStoreParams() { @Test(dataProvider = "cipherDataProvider") public void testClientKeyStoreSSLContext(Set cipher) throws Exception { - NettySSLContextAutoRefreshBuilder contextAutoRefreshBuilder = new NettySSLContextAutoRefreshBuilder( - null, - false, - keyStoreType, brokerTrustStorePath, keyStorePassword, - null, null, null, - cipher, null, 0, new ClientAuthenticationData()); - contextAutoRefreshBuilder.update(); + PulsarSslConfiguration pulsarSslConfiguration = PulsarSslConfiguration.builder() + .allowInsecureConnection(false) + .tlsEnabledWithKeystore(true) + .tlsTrustStoreType(keyStoreType) + .tlsTrustStorePath(brokerTrustStorePath) + .tlsTrustStorePassword(keyStorePassword) + .tlsCiphers(cipher) + .authData(new ClientAuthenticationData()) + .build(); + try (PulsarSslFactory pulsarSslFactory = new DefaultPulsarSslFactory()) { + pulsarSslFactory.initialize(pulsarSslConfiguration); + pulsarSslFactory.createInternalSslContext(); + } } @Test(dataProvider = "caCertSslContextDataProvider") public void testServerCaCertSslContextWithSslProvider(SslProvider sslProvider, Set ciphers) - throws GeneralSecurityException, IOException { - NettyServerSslContextBuilder sslContext = new NettyServerSslContextBuilder(sslProvider, - true, - caCertPath, brokerCertPath, brokerKeyPath, - ciphers, - null, - true, 60); - if (ciphers != null) { - if (sslProvider == null || sslProvider == SslProvider.OPENSSL) { - assertThrows(SSLException.class, sslContext::update); - return; + throws Exception { + try (PulsarSslFactory pulsarSslFactory = new DefaultPulsarSslFactory()) { + PulsarSslConfiguration.PulsarSslConfigurationBuilder builder = PulsarSslConfiguration.builder() + .tlsTrustCertsFilePath(caCertPath) + .tlsCertificateFilePath(brokerCertPath) + .tlsKeyFilePath(brokerKeyPath) + .tlsCiphers(ciphers) + .requireTrustedClientCertOnConnect(true); + if (sslProvider != null) { + builder.tlsProvider(sslProvider.name()); } + PulsarSslConfiguration pulsarSslConfiguration = builder.build(); + pulsarSslFactory.initialize(pulsarSslConfiguration); + + if (ciphers != null) { + if (sslProvider == null || sslProvider == SslProvider.OPENSSL) { + assertThrows(SSLException.class, pulsarSslFactory::createInternalSslContext); + return; + } + } + pulsarSslFactory.createInternalSslContext(); } - sslContext.update(); } @Test(dataProvider = "caCertSslContextDataProvider") public void testClientCaCertSslContextWithSslProvider(SslProvider sslProvider, Set ciphers) - throws GeneralSecurityException, IOException { - NettyClientSslContextRefresher sslContext = new NettyClientSslContextRefresher(sslProvider, - true, caCertPath, - null, ciphers, null, 0); - if (ciphers != null) { - if (sslProvider == null || sslProvider == SslProvider.OPENSSL) { - assertThrows(SSLException.class, sslContext::update); - return; + throws Exception { + try (PulsarSslFactory pulsarSslFactory = new DefaultPulsarSslFactory()) { + PulsarSslConfiguration.PulsarSslConfigurationBuilder builder = PulsarSslConfiguration.builder() + .allowInsecureConnection(true) + .tlsTrustCertsFilePath(caCertPath) + .tlsCiphers(ciphers); + if (sslProvider != null) { + builder.tlsProvider(sslProvider.name()); + } + PulsarSslConfiguration pulsarSslConfiguration = builder.build(); + pulsarSslFactory.initialize(pulsarSslConfiguration); + if (ciphers != null) { + if (sslProvider == null || sslProvider == SslProvider.OPENSSL) { + assertThrows(SSLException.class, pulsarSslFactory::createInternalSslContext); + return; + } } + pulsarSslFactory.createInternalSslContext(); } - sslContext.update(); } } diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/WorkerServer.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/WorkerServer.java index 1d8c66a57df537..4f01f17174e314 100644 --- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/WorkerServer.java +++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/WorkerServer.java @@ -24,6 +24,9 @@ import java.util.EnumSet; import java.util.List; import java.util.Optional; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; import javax.servlet.DispatcherType; import lombok.extern.slf4j.Slf4j; import org.apache.pulsar.broker.authentication.AuthenticationService; @@ -31,6 +34,10 @@ import org.apache.pulsar.broker.web.JettyRequestLogFactory; import org.apache.pulsar.broker.web.RateLimitingFilter; import org.apache.pulsar.broker.web.WebExecutorThreadPool; +import org.apache.pulsar.client.util.ExecutorProvider; +import org.apache.pulsar.common.util.DefaultPulsarSslFactory; +import org.apache.pulsar.common.util.PulsarSslConfiguration; +import org.apache.pulsar.common.util.PulsarSslFactory; import org.apache.pulsar.functions.worker.PulsarWorkerOpenTelemetry; import org.apache.pulsar.functions.worker.WorkerConfig; import org.apache.pulsar.functions.worker.WorkerService; @@ -75,6 +82,8 @@ public class WorkerServer { private ServerConnector httpsConnector; private final FilterInitializer filterInitializer; + private PulsarSslFactory sslFactory; + private ScheduledExecutorService scheduledExecutorService; public WorkerServer(WorkerService workerService, AuthenticationService authenticationService) { this.workerConfig = workerService.getWorkerConfig(); @@ -155,35 +164,22 @@ private void init() { if (this.workerConfig.getTlsEnabled()) { log.info("Configuring https server on port={}", this.workerConfig.getWorkerPortTls()); try { - SslContextFactory sslCtxFactory; - if (workerConfig.isTlsEnabledWithKeyStore()) { - sslCtxFactory = JettySslContextFactory.createServerSslContextWithKeystore( - workerConfig.getTlsProvider(), - workerConfig.getTlsKeyStoreType(), - workerConfig.getTlsKeyStore(), - workerConfig.getTlsKeyStorePassword(), - workerConfig.isTlsAllowInsecureConnection(), - workerConfig.getTlsTrustStoreType(), - workerConfig.getTlsTrustStore(), - workerConfig.getTlsTrustStorePassword(), - workerConfig.isTlsRequireTrustedClientCertOnConnect(), - workerConfig.getWebServiceTlsCiphers(), - workerConfig.getWebServiceTlsProtocols(), - workerConfig.getTlsCertRefreshCheckDurationSec() - ); - } else { - sslCtxFactory = JettySslContextFactory.createServerSslContext( - workerConfig.getTlsProvider(), - workerConfig.isTlsAllowInsecureConnection(), - workerConfig.getTlsTrustCertsFilePath(), - workerConfig.getTlsCertificateFilePath(), - workerConfig.getTlsKeyFilePath(), - workerConfig.isTlsRequireTrustedClientCertOnConnect(), - workerConfig.getWebServiceTlsCiphers(), - workerConfig.getWebServiceTlsProtocols(), - workerConfig.getTlsCertRefreshCheckDurationSec() - ); - } + PulsarSslConfiguration sslConfiguration = buildSslConfiguration(workerConfig); + this.sslFactory = new DefaultPulsarSslFactory(); + this.sslFactory.initialize(sslConfiguration); + this.sslFactory.createInternalSslContext(); + this.scheduledExecutorService = Executors + .newSingleThreadScheduledExecutor(new ExecutorProvider + .ExtendedThreadFactory("functions-worker-web-ssl-refresh")); + this.scheduledExecutorService.scheduleWithFixedDelay(this::refreshSslContext, + workerConfig.getTlsCertRefreshCheckDurationSec(), + workerConfig.getTlsCertRefreshCheckDurationSec(), + TimeUnit.SECONDS); + SslContextFactory sslCtxFactory = + JettySslContextFactory.createSslContextFactory(this.workerConfig.getTlsProvider(), + this.sslFactory, this.workerConfig.isTlsRequireTrustedClientCertOnConnect(), + this.workerConfig.getWebServiceTlsCiphers(), + this.workerConfig.getWebServiceTlsProtocols()); List connectionFactories = new ArrayList<>(); if (workerConfig.isWebServiceHaProxyProtocolEnabled()) { connectionFactories.add(new ProxyConnectionFactory()); @@ -288,6 +284,9 @@ public void stop() { log.warn("Error stopping function web-server executor", e); } } + if (this.scheduledExecutorService != null) { + this.scheduledExecutorService.shutdownNow(); + } } public Optional getListenPortHTTP() { @@ -305,4 +304,33 @@ public Optional getListenPortHTTPS() { return Optional.empty(); } } + + protected void refreshSslContext() { + try { + this.sslFactory.update(); + } catch (Exception e) { + log.error("Failed to refresh SSL context", e); + } + } + + protected PulsarSslConfiguration buildSslConfiguration(WorkerConfig config) { + return PulsarSslConfiguration.builder() + .tlsKeyStoreType(config.getTlsKeyStoreType()) + .tlsKeyStorePath(config.getTlsKeyStore()) + .tlsKeyStorePassword(config.getTlsKeyStorePassword()) + .tlsTrustStoreType(config.getTlsTrustStoreType()) + .tlsTrustStorePath(config.getTlsTrustStore()) + .tlsTrustStorePassword(config.getTlsTrustStorePassword()) + .tlsCiphers(config.getWebServiceTlsCiphers()) + .tlsProtocols(config.getWebServiceTlsProtocols()) + .tlsTrustCertsFilePath(config.getTlsTrustCertsFilePath()) + .tlsCertificateFilePath(config.getTlsCertificateFilePath()) + .tlsKeyFilePath(config.getTlsKeyFilePath()) + .allowInsecureConnection(config.isTlsAllowInsecureConnection()) + .requireTrustedClientCertOnConnect(config.isTlsRequireTrustedClientCertOnConnect()) + .tlsEnabledWithKeystore(config.isTlsEnabledWithKeyStore()) + .serverMode(true) + .isHttps(true) + .build(); + } } diff --git a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/AdminProxyHandler.java b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/AdminProxyHandler.java index 0108b770249a0e..051bbf585c47b6 100644 --- a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/AdminProxyHandler.java +++ b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/AdminProxyHandler.java @@ -24,13 +24,15 @@ import java.io.InputStream; import java.net.URI; import java.nio.ByteBuffer; -import java.security.cert.X509Certificate; import java.util.Arrays; import java.util.Collections; import java.util.HashSet; import java.util.Iterator; import java.util.Set; import java.util.concurrent.Executor; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; import javax.net.ssl.SSLContext; import javax.servlet.ServletConfig; import javax.servlet.ServletException; @@ -39,10 +41,10 @@ import org.apache.pulsar.broker.web.AuthenticationFilter; import org.apache.pulsar.client.api.Authentication; import org.apache.pulsar.client.api.AuthenticationDataProvider; -import org.apache.pulsar.client.api.KeyStoreParams; import org.apache.pulsar.client.api.PulsarClientException; -import org.apache.pulsar.common.util.SecurityUtility; -import org.apache.pulsar.common.util.keystoretls.KeyStoreSSLContext; +import org.apache.pulsar.client.util.ExecutorProvider; +import org.apache.pulsar.common.util.PulsarSslConfiguration; +import org.apache.pulsar.common.util.PulsarSslFactory; import org.apache.pulsar.policies.data.loadbalancer.ServiceLookupData; import org.eclipse.jetty.client.HttpClient; import org.eclipse.jetty.client.HttpRequest; @@ -88,6 +90,8 @@ class AdminProxyHandler extends ProxyServlet { private final Authentication proxyClientAuthentication; private final String brokerWebServiceUrl; private final String functionWorkerWebServiceUrl; + private PulsarSslFactory pulsarSslFactory; + private ScheduledExecutorService scheduledExecutorService; AdminProxyHandler(ProxyConfiguration config, BrokerDiscoveryProvider discoveryProvider, Authentication proxyClientAuthentication) { @@ -98,7 +102,14 @@ class AdminProxyHandler extends ProxyServlet { : config.getBrokerWebServiceURL(); this.functionWorkerWebServiceUrl = config.isTlsEnabledWithBroker() ? config.getFunctionWorkerWebServiceURLTLS() : config.getFunctionWorkerWebServiceURL(); - + if (config.isTlsEnabledWithBroker()) { + this.pulsarSslFactory = createPulsarSslFactory(); + this.scheduledExecutorService = Executors.newSingleThreadScheduledExecutor( + new ExecutorProvider.ExtendedThreadFactory("pulsar-proxy-admin-handler-ssl-refresh")); + this.scheduledExecutorService.scheduleWithFixedDelay(this::refreshSslContext, + config.getTlsCertRefreshCheckDurationSec(), config.getTlsCertRefreshCheckDurationSec(), + TimeUnit.SECONDS); + } super.setTimeout(config.getHttpProxyTimeout()); } @@ -259,44 +270,7 @@ protected HttpClient newHttpClient() { try { if (config.isTlsEnabledWithBroker()) { try { - X509Certificate[] trustCertificates = SecurityUtility - .loadCertificatesFromPemFile(config.getBrokerClientTrustCertsFilePath()); - - SSLContext sslCtx; - AuthenticationDataProvider authData = proxyClientAuthentication.getAuthData(); - if (config.isBrokerClientTlsEnabledWithKeyStore()) { - KeyStoreParams params = authData.hasDataForTls() ? authData.getTlsKeyStoreParams() : null; - sslCtx = KeyStoreSSLContext.createClientSslContext( - config.getBrokerClientSslProvider(), - params != null ? params.getKeyStoreType() : null, - params != null ? params.getKeyStorePath() : null, - params != null ? params.getKeyStorePassword() : null, - config.isTlsAllowInsecureConnection(), - config.getBrokerClientTlsTrustStoreType(), - config.getBrokerClientTlsTrustStore(), - config.getBrokerClientTlsTrustStorePassword(), - config.getBrokerClientTlsCiphers(), - config.getBrokerClientTlsProtocols()); - } else { - if (authData.hasDataForTls()) { - sslCtx = SecurityUtility.createSslContext( - config.isTlsAllowInsecureConnection(), - trustCertificates, - authData.getTlsCertificates(), - authData.getTlsPrivateKey(), - config.getBrokerClientSslProvider() - ); - } else { - sslCtx = SecurityUtility.createSslContext( - config.isTlsAllowInsecureConnection(), - trustCertificates, - config.getBrokerClientSslProvider() - ); - } - } - - SslContextFactory contextFactory = new SslContextFactory.Client(); - contextFactory.setSslContext(sslCtx); + SslContextFactory contextFactory = new Client(this.pulsarSslFactory); if (!config.isTlsHostnameVerificationEnabled()) { contextFactory.setEndpointIdentificationAlgorithm(null); } @@ -379,4 +353,71 @@ protected void addProxyHeaders(HttpServletRequest clientRequest, Request proxyRe proxyRequest.header(ORIGINAL_PRINCIPAL_HEADER, user); } } + + private static class Client extends SslContextFactory.Client { + + private final PulsarSslFactory sslFactory; + + public Client(PulsarSslFactory sslFactory) { + super(); + this.sslFactory = sslFactory; + } + + @Override + public SSLContext getSslContext() { + return this.sslFactory.getInternalSslContext(); + } + } + + protected PulsarSslConfiguration buildSslConfiguration(AuthenticationDataProvider authData) { + return PulsarSslConfiguration.builder() + .tlsProvider(config.getBrokerClientSslProvider()) + .tlsKeyStoreType(config.getBrokerClientTlsKeyStoreType()) + .tlsKeyStorePath(config.getBrokerClientTlsKeyStore()) + .tlsKeyStorePassword(config.getBrokerClientTlsKeyStorePassword()) + .tlsTrustStoreType(config.getBrokerClientTlsTrustStoreType()) + .tlsTrustStorePath(config.getBrokerClientTlsTrustStore()) + .tlsTrustStorePassword(config.getBrokerClientTlsTrustStorePassword()) + .tlsCiphers(config.getBrokerClientTlsCiphers()) + .tlsProtocols(config.getBrokerClientTlsProtocols()) + .tlsTrustCertsFilePath(config.getBrokerClientTrustCertsFilePath()) + .tlsCertificateFilePath(config.getBrokerClientCertificateFilePath()) + .tlsKeyFilePath(config.getBrokerClientKeyFilePath()) + .allowInsecureConnection(config.isTlsAllowInsecureConnection()) + .requireTrustedClientCertOnConnect(false) + .tlsEnabledWithKeystore(config.isBrokerClientTlsEnabledWithKeyStore()) + .tlsCustomParams(config.getBrokerClientSslFactoryPluginParams()) + .authData(authData) + .serverMode(false) + .isHttps(true) + .build(); + } + + protected PulsarSslFactory createPulsarSslFactory() { + try { + try { + AuthenticationDataProvider authData = proxyClientAuthentication.getAuthData(); + PulsarSslConfiguration pulsarSslConfiguration = buildSslConfiguration(authData); + PulsarSslFactory sslFactory = + (PulsarSslFactory) Class.forName(config.getBrokerClientSslFactoryPlugin()) + .getConstructor().newInstance(); + sslFactory.initialize(pulsarSslConfiguration); + sslFactory.createInternalSslContext(); + return sslFactory; + } catch (Exception e) { + LOG.error("Failed to create Pulsar SSLFactory ", e); + throw new PulsarClientException.InvalidConfigurationException(e.getMessage()); + } + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + protected void refreshSslContext() { + try { + this.pulsarSslFactory.update(); + } catch (Exception e) { + LOG.error("Failed to refresh SSL context", e); + } + } } diff --git a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/DirectProxyHandler.java b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/DirectProxyHandler.java index 4678db82c6e55e..407c93074a0fcb 100644 --- a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/DirectProxyHandler.java +++ b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/DirectProxyHandler.java @@ -40,15 +40,14 @@ import io.netty.handler.codec.haproxy.HAProxyProtocolVersion; import io.netty.handler.codec.haproxy.HAProxyProxiedProtocol; import io.netty.handler.flush.FlushConsolidationHandler; -import io.netty.handler.ssl.SslContext; import io.netty.handler.ssl.SslHandler; -import io.netty.handler.ssl.SslProvider; import io.netty.handler.timeout.ReadTimeoutHandler; import io.netty.util.CharsetUtil; import java.net.InetSocketAddress; import java.util.Arrays; import java.util.concurrent.TimeUnit; import lombok.Getter; +import lombok.SneakyThrows; import org.apache.pulsar.PulsarVersion; import org.apache.pulsar.client.api.Authentication; import org.apache.pulsar.client.api.AuthenticationDataProvider; @@ -60,10 +59,9 @@ import org.apache.pulsar.common.protocol.Commands; import org.apache.pulsar.common.protocol.PulsarDecoder; import org.apache.pulsar.common.stats.Rate; -import org.apache.pulsar.common.util.NettyClientSslContextRefresher; +import org.apache.pulsar.common.util.PulsarSslConfiguration; +import org.apache.pulsar.common.util.PulsarSslFactory; import org.apache.pulsar.common.util.SecurityUtility; -import org.apache.pulsar.common.util.SslContextAutoRefreshBuilder; -import org.apache.pulsar.common.util.keystoretls.NettySSLContextAutoRefreshBuilder; import org.apache.pulsar.common.util.netty.NettyChannelUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -88,11 +86,10 @@ public class DirectProxyHandler { private final ProxyService service; private final Runnable onHandshakeCompleteAction; private final boolean tlsHostnameVerificationEnabled; - private final boolean tlsEnabledWithKeyStore; final boolean tlsEnabledWithBroker; - private final SslContextAutoRefreshBuilder clientSslCtxRefresher; - private final NettySSLContextAutoRefreshBuilder clientSSLContextAutoRefreshBuilder; + private PulsarSslFactory sslFactory; + @SneakyThrows public DirectProxyHandler(ProxyService service, ProxyConnection proxyConnection) { this.service = service; this.authentication = proxyConnection.getClientAuthentication(); @@ -104,7 +101,6 @@ public DirectProxyHandler(ProxyService service, ProxyConnection proxyConnection) this.clientAuthMethod = proxyConnection.clientAuthMethod; this.tlsEnabledWithBroker = service.getConfiguration().isTlsEnabledWithBroker(); this.tlsHostnameVerificationEnabled = service.getConfiguration().isTlsHostnameVerificationEnabled(); - this.tlsEnabledWithKeyStore = service.getConfiguration().isTlsEnabledWithKeyStore(); this.onHandshakeCompleteAction = proxyConnection::cancelKeepAliveTask; ProxyConfiguration config = service.getConfiguration(); @@ -118,41 +114,11 @@ public DirectProxyHandler(ProxyService service, ProxyConnection proxyConnection) throw new RuntimeException(e); } } - - if (tlsEnabledWithKeyStore) { - clientSSLContextAutoRefreshBuilder = new NettySSLContextAutoRefreshBuilder( - config.getBrokerClientSslProvider(), - config.isTlsAllowInsecureConnection(), - config.getBrokerClientTlsTrustStoreType(), - config.getBrokerClientTlsTrustStore(), - config.getBrokerClientTlsTrustStorePassword(), - config.getBrokerClientTlsKeyStoreType(), - config.getBrokerClientTlsKeyStore(), - config.getBrokerClientTlsKeyStorePassword(), - config.getBrokerClientTlsCiphers(), - config.getBrokerClientTlsProtocols(), - config.getTlsCertRefreshCheckDurationSec(), - authData); - clientSslCtxRefresher = null; - } else { - SslProvider sslProvider = null; - if (config.getBrokerClientSslProvider() != null) { - sslProvider = SslProvider.valueOf(config.getBrokerClientSslProvider()); - } - clientSslCtxRefresher = new NettyClientSslContextRefresher( - sslProvider, - config.isTlsAllowInsecureConnection(), - config.getBrokerClientTrustCertsFilePath(), - authData, - config.getBrokerClientTlsCiphers(), - config.getBrokerClientTlsProtocols(), - config.getTlsCertRefreshCheckDurationSec() - ); - clientSSLContextAutoRefreshBuilder = null; - } - } else { - clientSSLContextAutoRefreshBuilder = null; - clientSslCtxRefresher = null; + PulsarSslConfiguration sslConfiguration = buildSslConfiguration(config, authData); + this.sslFactory = (PulsarSslFactory) Class.forName(config.getSslFactoryPlugin()) + .getConstructor().newInstance(); + this.sslFactory.initialize(sslConfiguration); + this.sslFactory.createInternalSslContext(); } } @@ -193,9 +159,7 @@ protected void initChannel(SocketChannel ch) { if (tlsEnabledWithBroker) { String host = targetBrokerAddress.getHostString(); int port = targetBrokerAddress.getPort(); - SslHandler handler = tlsEnabledWithKeyStore - ? new SslHandler(clientSSLContextAutoRefreshBuilder.get().createSSLEngine(host, port)) - : clientSslCtxRefresher.get().newHandler(ch.alloc(), host, port); + SslHandler handler = new SslHandler(sslFactory.createClientSslEngine(ch.alloc(), host, port)); if (tlsHostnameVerificationEnabled) { SecurityUtility.configureSSLHandler(handler); } @@ -499,5 +463,29 @@ private void writeAndFlush(ByteBuf cmd) { NettyChannelUtil.writeAndFlushWithVoidPromise(outboundChannel, cmd); } + protected PulsarSslConfiguration buildSslConfiguration(ProxyConfiguration config, + AuthenticationDataProvider authData) { + return PulsarSslConfiguration.builder() + .tlsProvider(config.getBrokerClientSslProvider()) + .tlsKeyStoreType(config.getBrokerClientTlsKeyStoreType()) + .tlsKeyStorePath(config.getBrokerClientTlsKeyStore()) + .tlsKeyStorePassword(config.getBrokerClientTlsKeyStorePassword()) + .tlsTrustStoreType(config.getBrokerClientTlsTrustStoreType()) + .tlsTrustStorePath(config.getBrokerClientTlsTrustStore()) + .tlsTrustStorePassword(config.getBrokerClientTlsTrustStorePassword()) + .tlsCiphers(config.getBrokerClientTlsCiphers()) + .tlsProtocols(config.getBrokerClientTlsProtocols()) + .tlsTrustCertsFilePath(config.getBrokerClientTrustCertsFilePath()) + .tlsCertificateFilePath(config.getBrokerClientCertificateFilePath()) + .tlsKeyFilePath(config.getBrokerClientKeyFilePath()) + .allowInsecureConnection(config.isTlsAllowInsecureConnection()) + .requireTrustedClientCertOnConnect(false) + .tlsEnabledWithKeystore(config.isBrokerClientTlsEnabledWithKeyStore()) + .tlsCustomParams(config.getBrokerClientSslFactoryPluginParams()) + .authData(authData) + .serverMode(false) + .build(); + } + private static final Logger log = LoggerFactory.getLogger(DirectProxyHandler.class); } diff --git a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConfiguration.java b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConfiguration.java index d65408748f4322..b9360e403f6f48 100644 --- a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConfiguration.java +++ b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConfiguration.java @@ -42,6 +42,8 @@ import org.apache.pulsar.common.nar.NarClassLoader; import org.apache.pulsar.common.protocol.Commands; import org.apache.pulsar.common.sasl.SaslConstants; +import org.apache.pulsar.common.util.DefaultPulsarSslFactory; + @Getter @Setter @@ -614,6 +616,16 @@ public class ProxyConfiguration implements PulsarConfiguration { ) private String tlsTrustStorePassword = null; + @FieldContext( + category = CATEGORY_TLS, + doc = "SSL Factory Plugin class to provide SSLEngine and SSLContext objects. The default " + + " class used is DefaultSslFactory.") + private String sslFactoryPlugin = DefaultPulsarSslFactory.class.getName(); + @FieldContext( + category = CATEGORY_TLS, + doc = "SSL Factory plugin configuration parameters.") + private String sslFactoryPluginParams = ""; + /** * KeyStore TLS config variables used for proxy to auth with broker. */ @@ -683,6 +695,16 @@ public class ProxyConfiguration implements PulsarConfiguration { ) private Set brokerClientTlsProtocols = new TreeSet<>(); + @FieldContext( + category = CATEGORY_TLS, + doc = "SSL Factory Plugin class used by internal client to provide SSLEngine and SSLContext objects. " + + "The default class used is DefaultSslFactory.") + private String brokerClientSslFactoryPlugin = DefaultPulsarSslFactory.class.getName(); + @FieldContext( + category = CATEGORY_TLS, + doc = "SSL Factory plugin configuration parameters used by internal client.") + private String brokerClientSslFactoryPluginParams = ""; + // HTTP @FieldContext( diff --git a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConnection.java b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConnection.java index d58fe46e0063ac..f8b5d0844509ef 100644 --- a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConnection.java +++ b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConnection.java @@ -398,7 +398,7 @@ private synchronized void completeConnect() throws PulsarClientException { if (this.connectionPool == null) { this.connectionPool = new ConnectionPool(InstrumentProvider.NOOP, clientConf, service.getWorkerGroup(), clientCnxSupplier, - Optional.of(dnsAddressResolverGroup.getResolver(service.getWorkerGroup().next()))); + Optional.of(dnsAddressResolverGroup.getResolver(service.getWorkerGroup().next())), null); } else { LOG.error("BUG! Connection Pool has already been created for proxy connection to {} state {} role {}", remoteAddress, state, clientAuthRole); diff --git a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ServiceChannelInitializer.java b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ServiceChannelInitializer.java index 19f4002ad52ce6..17babfd78a00b9 100644 --- a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ServiceChannelInitializer.java +++ b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ServiceChannelInitializer.java @@ -22,22 +22,25 @@ import io.netty.channel.socket.SocketChannel; import io.netty.handler.codec.LengthFieldBasedFrameDecoder; import io.netty.handler.flush.FlushConsolidationHandler; -import io.netty.handler.ssl.SslContext; import io.netty.handler.ssl.SslHandler; -import io.netty.handler.ssl.SslProvider; import io.netty.handler.timeout.ReadTimeoutHandler; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; +import org.apache.pulsar.client.util.ExecutorProvider; import org.apache.pulsar.common.protocol.Commands; import org.apache.pulsar.common.protocol.OptionalProxyProtocolDecoder; -import org.apache.pulsar.common.util.NettyServerSslContextBuilder; -import org.apache.pulsar.common.util.SslContextAutoRefreshBuilder; -import org.apache.pulsar.common.util.keystoretls.NettySSLContextAutoRefreshBuilder; +import org.apache.pulsar.common.util.PulsarSslConfiguration; +import org.apache.pulsar.common.util.PulsarSslFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * Initialize service channel handlers. * */ public class ServiceChannelInitializer extends ChannelInitializer { + private static final Logger log = LoggerFactory.getLogger(ServiceChannelInitializer.class); public static final String TLS_HANDLER = "tls"; private final ProxyService proxyService; @@ -46,8 +49,8 @@ public class ServiceChannelInitializer extends ChannelInitializer private final int brokerProxyReadTimeoutMs; private final int maxMessageSize; - private SslContextAutoRefreshBuilder serverSslCtxRefresher; - private NettySSLContextAutoRefreshBuilder serverSSLContextAutoRefreshBuilder; + private PulsarSslFactory sslFactory; + private ScheduledExecutorService scheduledExecutorService; public ServiceChannelInitializer(ProxyService proxyService, ProxyConfiguration serviceConfig, boolean enableTls) throws Exception { @@ -59,36 +62,16 @@ public ServiceChannelInitializer(ProxyService proxyService, ProxyConfiguration s this.maxMessageSize = serviceConfig.getMaxMessageSize(); if (enableTls) { - if (tlsEnabledWithKeyStore) { - serverSSLContextAutoRefreshBuilder = new NettySSLContextAutoRefreshBuilder( - serviceConfig.getTlsProvider(), - serviceConfig.getTlsKeyStoreType(), - serviceConfig.getTlsKeyStore(), - serviceConfig.getTlsKeyStorePassword(), - serviceConfig.isTlsAllowInsecureConnection(), - serviceConfig.getTlsTrustStoreType(), - serviceConfig.getTlsTrustStore(), - serviceConfig.getTlsTrustStorePassword(), - serviceConfig.isTlsRequireTrustedClientCertOnConnect(), - serviceConfig.getTlsCiphers(), - serviceConfig.getTlsProtocols(), - serviceConfig.getTlsCertRefreshCheckDurationSec()); - } else { - SslProvider sslProvider = null; - if (serviceConfig.getTlsProvider() != null) { - sslProvider = SslProvider.valueOf(serviceConfig.getTlsProvider()); - } - serverSslCtxRefresher = new NettyServerSslContextBuilder( - sslProvider, - serviceConfig.isTlsAllowInsecureConnection(), - serviceConfig.getTlsTrustCertsFilePath(), serviceConfig.getTlsCertificateFilePath(), - serviceConfig.getTlsKeyFilePath(), serviceConfig.getTlsCiphers(), - serviceConfig.getTlsProtocols(), - serviceConfig.isTlsRequireTrustedClientCertOnConnect(), - serviceConfig.getTlsCertRefreshCheckDurationSec()); - } - } else { - this.serverSslCtxRefresher = null; + this.scheduledExecutorService = Executors.newSingleThreadScheduledExecutor( + new ExecutorProvider.ExtendedThreadFactory("pulsar-proxy-service-channel-tls-refresh")); + PulsarSslConfiguration sslConfiguration = buildSslConfiguration(serviceConfig); + this.sslFactory = (PulsarSslFactory) Class.forName(serviceConfig.getSslFactoryPlugin()) + .getConstructor().newInstance(); + this.sslFactory.initialize(sslConfiguration); + this.sslFactory.createInternalSslContext(); + scheduledExecutorService.scheduleWithFixedDelay(this::refreshSslContext, + serviceConfig.getTlsCertRefreshCheckDurationSec(), + serviceConfig.getTlsCertRefreshCheckDurationSec(), TimeUnit.SECONDS); } } @@ -96,14 +79,8 @@ public ServiceChannelInitializer(ProxyService proxyService, ProxyConfiguration s protected void initChannel(SocketChannel ch) throws Exception { ch.pipeline().addLast("consolidation", new FlushConsolidationHandler(1024, true)); - if (serverSslCtxRefresher != null && this.enableTls) { - SslContext sslContext = serverSslCtxRefresher.get(); - if (sslContext != null) { - ch.pipeline().addLast(TLS_HANDLER, sslContext.newHandler(ch.alloc())); - } - } else if (this.tlsEnabledWithKeyStore && serverSSLContextAutoRefreshBuilder != null) { - ch.pipeline().addLast(TLS_HANDLER, - new SslHandler(serverSSLContextAutoRefreshBuilder.get().createSSLEngine())); + if (this.enableTls) { + ch.pipeline().addLast(TLS_HANDLER, new SslHandler(this.sslFactory.createServerSslEngine(ch.alloc()))); } if (brokerProxyReadTimeoutMs > 0) { ch.pipeline().addLast("readTimeoutHandler", @@ -117,4 +94,35 @@ protected void initChannel(SocketChannel ch) throws Exception { ch.pipeline().addLast("handler", new ProxyConnection(proxyService, proxyService.getDnsAddressResolverGroup())); } + + protected PulsarSslConfiguration buildSslConfiguration(ProxyConfiguration config) { + return PulsarSslConfiguration.builder() + .tlsProvider(config.getTlsProvider()) + .tlsKeyStoreType(config.getTlsKeyStoreType()) + .tlsKeyStorePath(config.getTlsKeyStore()) + .tlsKeyStorePassword(config.getTlsKeyStorePassword()) + .tlsTrustStoreType(config.getTlsTrustStoreType()) + .tlsTrustStorePath(config.getTlsTrustStore()) + .tlsTrustStorePassword(config.getTlsTrustStorePassword()) + .tlsCiphers(config.getTlsCiphers()) + .tlsProtocols(config.getTlsProtocols()) + .tlsTrustCertsFilePath(config.getTlsTrustCertsFilePath()) + .tlsCertificateFilePath(config.getTlsCertificateFilePath()) + .tlsKeyFilePath(config.getTlsKeyFilePath()) + .allowInsecureConnection(config.isTlsAllowInsecureConnection()) + .requireTrustedClientCertOnConnect(config.isTlsRequireTrustedClientCertOnConnect()) + .tlsEnabledWithKeystore(config.isTlsEnabledWithKeyStore()) + .tlsCustomParams(config.getSslFactoryPluginParams()) + .authData(null) + .serverMode(true) + .build(); + } + + protected void refreshSslContext() { + try { + this.sslFactory.update(); + } catch (Exception e) { + log.error("Failed to refresh SSL context", e); + } + } } diff --git a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/WebServer.java b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/WebServer.java index ad94f1b65a0920..8889c0934f052e 100644 --- a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/WebServer.java +++ b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/WebServer.java @@ -29,6 +29,9 @@ import java.util.EnumSet; import java.util.List; import java.util.Optional; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; import javax.servlet.DispatcherType; import org.apache.commons.lang3.tuple.Pair; import org.apache.pulsar.broker.authentication.AuthenticationService; @@ -37,6 +40,9 @@ import org.apache.pulsar.broker.web.JsonMapperProvider; import org.apache.pulsar.broker.web.RateLimitingFilter; import org.apache.pulsar.broker.web.WebExecutorThreadPool; +import org.apache.pulsar.client.util.ExecutorProvider; +import org.apache.pulsar.common.util.PulsarSslConfiguration; +import org.apache.pulsar.common.util.PulsarSslFactory; import org.apache.pulsar.jetty.tls.JettySslContextFactory; import org.apache.pulsar.proxy.stats.PulsarProxyOpenTelemetry; import org.eclipse.jetty.server.ConnectionFactory; @@ -85,6 +91,9 @@ public class WebServer { private ServerConnector connector; private ServerConnector connectorTls; + private ScheduledExecutorService scheduledExecutorService; + private PulsarSslFactory sslFactory; + private final FilterInitializer filterInitializer; public WebServer(ProxyConfiguration config, AuthenticationService authenticationService) { @@ -121,34 +130,20 @@ public WebServer(ProxyConfiguration config, AuthenticationService authentication } if (config.getWebServicePortTls().isPresent()) { try { - SslContextFactory sslCtxFactory; - if (config.isTlsEnabledWithKeyStore()) { - sslCtxFactory = JettySslContextFactory.createServerSslContextWithKeystore( - config.getWebServiceTlsProvider(), - config.getTlsKeyStoreType(), - config.getTlsKeyStore(), - config.getTlsKeyStorePassword(), - config.isTlsAllowInsecureConnection(), - config.getTlsTrustStoreType(), - config.getTlsTrustStore(), - config.getTlsTrustStorePassword(), - config.isTlsRequireTrustedClientCertOnConnect(), - config.getWebServiceTlsCiphers(), - config.getWebServiceTlsProtocols(), - config.getTlsCertRefreshCheckDurationSec() - ); - } else { - sslCtxFactory = JettySslContextFactory.createServerSslContext( - config.getWebServiceTlsProvider(), - config.isTlsAllowInsecureConnection(), - config.getTlsTrustCertsFilePath(), - config.getTlsCertificateFilePath(), - config.getTlsKeyFilePath(), - config.isTlsRequireTrustedClientCertOnConnect(), - config.getWebServiceTlsCiphers(), - config.getWebServiceTlsProtocols(), - config.getTlsCertRefreshCheckDurationSec()); - } + this.scheduledExecutorService = Executors.newSingleThreadScheduledExecutor( + new ExecutorProvider.ExtendedThreadFactory("pulsar-proxy-web-server-tls-refresh")); + PulsarSslConfiguration sslConfiguration = buildSslConfiguration(config); + this.sslFactory = (PulsarSslFactory) Class.forName(config.getSslFactoryPlugin()) + .getConstructor().newInstance(); + this.sslFactory.initialize(sslConfiguration); + this.sslFactory.createInternalSslContext(); + scheduledExecutorService.scheduleWithFixedDelay(this::refreshSslContext, + config.getTlsCertRefreshCheckDurationSec(), + config.getTlsCertRefreshCheckDurationSec(), TimeUnit.SECONDS); + SslContextFactory sslCtxFactory = + JettySslContextFactory.createSslContextFactory(config.getTlsProvider(), + sslFactory, config.isTlsRequireTrustedClientCertOnConnect(), + config.getWebServiceTlsCiphers(), config.getWebServiceTlsProtocols()); List connectionFactories = new ArrayList<>(); if (config.isWebServiceHaProxyProtocolEnabled()) { connectionFactories.add(new ProxyConnectionFactory()); @@ -388,5 +383,37 @@ public Optional getListenPortHTTPS() { } } + protected PulsarSslConfiguration buildSslConfiguration(ProxyConfiguration config) { + return PulsarSslConfiguration.builder() + .tlsProvider(config.getTlsProvider()) + .tlsKeyStoreType(config.getTlsKeyStoreType()) + .tlsKeyStorePath(config.getTlsKeyStore()) + .tlsKeyStorePassword(config.getTlsKeyStorePassword()) + .tlsTrustStoreType(config.getTlsTrustStoreType()) + .tlsTrustStorePath(config.getTlsTrustStore()) + .tlsTrustStorePassword(config.getTlsTrustStorePassword()) + .tlsCiphers(config.getTlsCiphers()) + .tlsProtocols(config.getTlsProtocols()) + .tlsTrustCertsFilePath(config.getTlsTrustCertsFilePath()) + .tlsCertificateFilePath(config.getTlsCertificateFilePath()) + .tlsKeyFilePath(config.getTlsKeyFilePath()) + .allowInsecureConnection(config.isTlsAllowInsecureConnection()) + .requireTrustedClientCertOnConnect(config.isTlsRequireTrustedClientCertOnConnect()) + .tlsEnabledWithKeystore(config.isTlsEnabledWithKeyStore()) + .tlsCustomParams(config.getSslFactoryPluginParams()) + .authData(null) + .serverMode(true) + .isHttps(true) + .build(); + } + + protected void refreshSslContext() { + try { + this.sslFactory.update(); + } catch (Exception e) { + log.error("Failed to refresh SSL context", e); + } + } + private static final Logger log = LoggerFactory.getLogger(WebServer.class); } diff --git a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/AdminProxyHandlerTest.java b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/AdminProxyHandlerTest.java index 4f925618e8a795..fdf9242c9f3d82 100644 --- a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/AdminProxyHandlerTest.java +++ b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/AdminProxyHandlerTest.java @@ -43,7 +43,7 @@ public class AdminProxyHandlerTest { private AdminProxyHandler adminProxyHandler; @BeforeClass - public void setupMocks() throws ServletException { + public void setupMocks() throws Exception { // given HttpClient httpClient = mock(HttpClient.class); adminProxyHandler = new AdminProxyHandler(mock(ProxyConfiguration.class), diff --git a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyParserTest.java b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyParserTest.java index 583ab7000e54f7..ee0f8010b7d798 100644 --- a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyParserTest.java +++ b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyParserTest.java @@ -268,7 +268,7 @@ protected void handleActiveConsumerChange(CommandActiveConsumerChange change) { throw new UnsupportedOperationException(); } }; - }); + }, null); return new PulsarClientImpl(conf, eventLoopGroup, cnxPool); } diff --git a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyServiceTlsStarterTest.java b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyServiceTlsStarterTest.java index 770424d93747cd..1148234be624ce 100644 --- a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyServiceTlsStarterTest.java +++ b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyServiceTlsStarterTest.java @@ -60,6 +60,8 @@ protected void setup() throws Exception { serviceStarter.getConfig().setBrokerServiceURLTLS(pulsar.getBrokerServiceUrlTls()); serviceStarter.getConfig().setBrokerWebServiceURL(pulsar.getWebServiceAddress()); serviceStarter.getConfig().setBrokerClientTrustCertsFilePath(CA_CERT_FILE_PATH); + serviceStarter.getConfig().setBrokerClientCertificateFilePath(BROKER_CERT_FILE_PATH); + serviceStarter.getConfig().setBrokerClientKeyFilePath(BROKER_KEY_FILE_PATH); serviceStarter.getConfig().setServicePort(Optional.empty()); serviceStarter.getConfig().setServicePortTls(Optional.of(0)); serviceStarter.getConfig().setWebServicePort(Optional.of(0)); diff --git a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyTest.java b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyTest.java index e101eb4ff7a2b7..4c0cd49d74fa75 100644 --- a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyTest.java +++ b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/server/ProxyTest.java @@ -392,7 +392,7 @@ protected void handleActiveConsumerChange(CommandActiveConsumerChange change) { throw new UnsupportedOperationException(); } }; - }); + }, null); registerCloseable(cnxPool); return new PulsarClientImpl(conf, eventLoopGroup, cnxPool); diff --git a/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerfClientUtils.java b/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerfClientUtils.java index b6b3d805edc75b..6bf73e705d16ca 100644 --- a/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerfClientUtils.java +++ b/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerfClientUtils.java @@ -85,6 +85,11 @@ public static ClientBuilder createClientBuilderFromArguments(PerformanceBaseArgu clientBuilder.authentication(arguments.authPluginClassName, arguments.authParams); } + if (isNotBlank(arguments.sslfactoryPlugin)) { + clientBuilder.sslFactoryPlugin(arguments.sslfactoryPlugin) + .sslFactoryPluginParams(arguments.sslFactoryPluginParams); + } + if (arguments.tlsAllowInsecureConnection != null) { clientBuilder.allowTlsInsecureConnection(arguments.tlsAllowInsecureConnection); } @@ -111,6 +116,11 @@ public static PulsarAdminBuilder createAdminBuilderFromArguments(PerformanceBase pulsarAdminBuilder.authentication(arguments.authPluginClassName, arguments.authParams); } + if (isNotBlank(arguments.sslfactoryPlugin)) { + pulsarAdminBuilder.sslFactoryPlugin(arguments.sslfactoryPlugin) + .sslFactoryPluginParams(arguments.sslFactoryPluginParams); + } + if (arguments.tlsAllowInsecureConnection != null) { pulsarAdminBuilder.allowTlsInsecureConnection(arguments.tlsAllowInsecureConnection); } diff --git a/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceBaseArguments.java b/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceBaseArguments.java index 3c4b831332281d..ee79066c32f908 100644 --- a/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceBaseArguments.java +++ b/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceBaseArguments.java @@ -44,6 +44,15 @@ public abstract class PerformanceBaseArguments extends CmdBase{ + "or \"{\"key1\":\"val1\",\"key2\":\"val2\"}\".", descriptionKey = "authParams") public String authParams; + @Option(names = { "--ssl-factory-plugin" }, description = "Pulsar SSL Factory plugin class name", + descriptionKey = "sslFactoryPlugin") + public String sslfactoryPlugin; + + @Option(names = { "--ssl-factory-plugin-params" }, + description = "Pulsar SSL Factory Plugin parameters in the format: " + + "\"{\"key1\":\"val1\",\"key2\":\"val2\"}\".", descriptionKey = "sslFactoryPluginParams") + public String sslFactoryPluginParams; + @Option(names = { "--trust-cert-file" }, description = "Path for the trusted TLS certificate file", descriptionKey = "tlsTrustCertsFilePath") diff --git a/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/service/ProxyServer.java b/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/service/ProxyServer.java index bbb34a3e3f73d0..d81be89e89ff34 100644 --- a/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/service/ProxyServer.java +++ b/pulsar-websocket/src/main/java/org/apache/pulsar/websocket/service/ProxyServer.java @@ -24,6 +24,9 @@ import java.util.EnumSet; import java.util.List; import java.util.Optional; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import javax.servlet.DispatcherType; import javax.servlet.Servlet; @@ -34,6 +37,10 @@ import org.apache.pulsar.broker.web.JsonMapperProvider; import org.apache.pulsar.broker.web.WebExecutorThreadPool; import org.apache.pulsar.client.api.PulsarClientException; +import org.apache.pulsar.client.util.ExecutorProvider; +import org.apache.pulsar.common.util.DefaultPulsarSslFactory; +import org.apache.pulsar.common.util.PulsarSslConfiguration; +import org.apache.pulsar.common.util.PulsarSslFactory; import org.apache.pulsar.jetty.tls.JettySslContextFactory; import org.eclipse.jetty.server.ConnectionFactory; import org.eclipse.jetty.server.ConnectionLimit; @@ -70,6 +77,8 @@ public class ProxyServer { private ServerConnector connector; private ServerConnector connectorTls; + private PulsarSslFactory sslFactory; + private ScheduledExecutorService scheduledExecutorService; public ProxyServer(WebSocketProxyConfiguration config) throws PulsarClientException, MalformedURLException, PulsarServerException { @@ -102,34 +111,21 @@ public ProxyServer(WebSocketProxyConfiguration config) // TLS enabled connector if (config.getWebServicePortTls().isPresent()) { try { - SslContextFactory sslCtxFactory; - if (config.isTlsEnabledWithKeyStore()) { - sslCtxFactory = JettySslContextFactory.createServerSslContextWithKeystore( - config.getTlsProvider(), - config.getTlsKeyStoreType(), - config.getTlsKeyStore(), - config.getTlsKeyStorePassword(), - config.isTlsAllowInsecureConnection(), - config.getTlsTrustStoreType(), - config.getTlsTrustStore(), - config.getTlsTrustStorePassword(), - config.isTlsRequireTrustedClientCertOnConnect(), - config.getWebServiceTlsCiphers(), - config.getWebServiceTlsProtocols(), - config.getTlsCertRefreshCheckDurationSec() - ); - } else { - sslCtxFactory = JettySslContextFactory.createServerSslContext( - config.getTlsProvider(), - config.isTlsAllowInsecureConnection(), - config.getTlsTrustCertsFilePath(), - config.getTlsCertificateFilePath(), - config.getTlsKeyFilePath(), - config.isTlsRequireTrustedClientCertOnConnect(), - config.getWebServiceTlsCiphers(), - config.getWebServiceTlsProtocols(), - config.getTlsCertRefreshCheckDurationSec()); - } + PulsarSslConfiguration sslConfiguration = buildSslConfiguration(config); + this.sslFactory = new DefaultPulsarSslFactory(); + this.sslFactory.initialize(sslConfiguration); + this.sslFactory.createInternalSslContext(); + this.scheduledExecutorService = Executors + .newSingleThreadScheduledExecutor(new ExecutorProvider + .ExtendedThreadFactory("proxy-websocket-ssl-refresh")); + this.scheduledExecutorService.scheduleWithFixedDelay(this::refreshSslContext, + config.getTlsCertRefreshCheckDurationSec(), + config.getTlsCertRefreshCheckDurationSec(), + TimeUnit.SECONDS); + SslContextFactory sslCtxFactory = + JettySslContextFactory.createSslContextFactory(config.getTlsProvider(), + sslFactory, config.isTlsRequireTrustedClientCertOnConnect(), + config.getWebServiceTlsCiphers(), config.getWebServiceTlsProtocols()); List connectionFactories = new ArrayList<>(); if (config.isWebServiceHaProxyProtocolEnabled()) { connectionFactories.add(new ProxyConnectionFactory()); @@ -223,6 +219,9 @@ public void start() throws PulsarServerException { public void stop() throws Exception { server.stop(); executorService.stop(); + if (scheduledExecutorService != null) { + scheduledExecutorService.shutdownNow(); + } } public Optional getListenPortHTTP() { @@ -241,5 +240,34 @@ public Optional getListenPortHTTPS() { } } + protected PulsarSslConfiguration buildSslConfiguration(WebSocketProxyConfiguration config) { + return PulsarSslConfiguration.builder() + .tlsKeyStoreType(config.getTlsKeyStoreType()) + .tlsKeyStorePath(config.getTlsKeyStore()) + .tlsKeyStorePassword(config.getTlsKeyStorePassword()) + .tlsTrustStoreType(config.getTlsTrustStoreType()) + .tlsTrustStorePath(config.getTlsTrustStore()) + .tlsTrustStorePassword(config.getTlsTrustStorePassword()) + .tlsCiphers(config.getWebServiceTlsCiphers()) + .tlsProtocols(config.getWebServiceTlsProtocols()) + .tlsTrustCertsFilePath(config.getTlsTrustCertsFilePath()) + .tlsCertificateFilePath(config.getTlsCertificateFilePath()) + .tlsKeyFilePath(config.getTlsKeyFilePath()) + .allowInsecureConnection(config.isTlsAllowInsecureConnection()) + .requireTrustedClientCertOnConnect(config.isTlsRequireTrustedClientCertOnConnect()) + .tlsEnabledWithKeystore(config.isTlsEnabledWithKeyStore()) + .serverMode(true) + .isHttps(true) + .build(); + } + + protected void refreshSslContext() { + try { + this.sslFactory.update(); + } catch (Exception e) { + log.error("Failed to refresh SSL context", e); + } + } + private static final Logger log = LoggerFactory.getLogger(ProxyServer.class); }