From 01b137d368b37513f366a5c192e298f20a605e9e Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Wed, 16 Feb 2022 14:59:31 +0200 Subject: [PATCH 1/2] [Broker] Increase default numHttpServerThreads value to 200 to prevent Broker Admin API unavailability - numHttpServerThreads is the maximum number of threads. Initial number of threads is 8 when numHttpServerThreads >= 8 - Jetty defaults to 200 maximum threads, to prevent thread pool starvation. - this is the reason to use the value 200 as a default --- conf/broker.conf | 7 ++++--- conf/standalone.conf | 5 +++-- deployment/terraform-ansible/templates/broker.conf | 5 +++-- .../apache/pulsar/broker/ServiceConfiguration.java | 11 ++++------- 4 files changed, 14 insertions(+), 14 deletions(-) diff --git a/conf/broker.conf b/conf/broker.conf index e35642006a39c..90d1a80d59556 100644 --- a/conf/broker.conf +++ b/conf/broker.conf @@ -85,8 +85,9 @@ numIOThreads= # such as init zookeeper client, get namespace policies from zookeeper etc. It also used to split bundle. Default is 8 numOrderedExecutorThreads=8 -# Number of threads to use for HTTP requests processing. Default is set to 2 * Runtime.getRuntime().availableProcessors() -numHttpServerThreads= +# Maximum number of threads to use for HTTP requests processing. +# Default is set to 200. Initial number of threads is 8 when numHttpServerThreads >= 8. +numHttpServerThreads=200 # Number of thread pool size to use for pulsar broker service. # The executor in thread pool will do basic broker operation like load/unload bundle, update managedLedgerConfig, @@ -1430,4 +1431,4 @@ tlsEnabled=false # Enable Key_Shared subscription (default is enabled) # @deprecated since 2.8.0 subscriptionTypesEnabled is preferred over subscriptionKeySharedEnable. -subscriptionKeySharedEnable=true \ No newline at end of file +subscriptionKeySharedEnable=true diff --git a/conf/standalone.conf b/conf/standalone.conf index d774b7b0bf65b..91d555cbc8f91 100644 --- a/conf/standalone.conf +++ b/conf/standalone.conf @@ -49,8 +49,9 @@ numIOThreads= # such as init zookeeper client, get namespace policies from zookeeper etc. It also used to split bundle. Default is 8 numOrderedExecutorThreads=8 -# Number of threads to use for HTTP requests processing. Default is set to 2 * Runtime.getRuntime().availableProcessors() -numHttpServerThreads= +# Maximum number of threads to use for HTTP requests processing. +# Default is set to 200. Initial number of threads is 8 when numHttpServerThreads >= 8. +numHttpServerThreads=200 # Number of thread pool size to use for pulsar broker service. # The executor in thread pool will do basic broker operation like load/unload bundle, update managedLedgerConfig, diff --git a/deployment/terraform-ansible/templates/broker.conf b/deployment/terraform-ansible/templates/broker.conf index 710e218d08c10..1300fe697527c 100644 --- a/deployment/terraform-ansible/templates/broker.conf +++ b/deployment/terraform-ansible/templates/broker.conf @@ -58,8 +58,9 @@ advertisedAddress={{ hostvars[inventory_hostname].private_ip }} # Number of threads to use for Netty IO. Default is set to 2 * Runtime.getRuntime().availableProcessors() numIOThreads= -# Number of threads to use for HTTP requests processing. Default is set to 2 * Runtime.getRuntime().availableProcessors() -numHttpServerThreads= +# Maximum number of threads to use for HTTP requests processing. +# Default is set to 200. Initial number of threads is 8 when numHttpServerThreads >= 8. +numHttpServerThreads=200 # Flag to control features that are meant to be used when running in standalone mode isRunningStandalone= diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java index a622d38d16994..1bf1ba56484a5 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java @@ -258,13 +258,10 @@ public class ServiceConfiguration implements PulsarConfiguration { @FieldContext( category = CATEGORY_SERVER, - doc = "Number of threads to use for HTTP requests processing" - + " Default is set to `2 * Runtime.getRuntime().availableProcessors()`" - ) - // Use at least 8 threads to avoid having Jetty go into threads starving and - // having the possibility of getting into a deadlock where a Jetty thread is - // waiting for another HTTP call to complete in same thread. - private int numHttpServerThreads = Math.max(8, 2 * Runtime.getRuntime().availableProcessors()); + doc = "Maximum number of threads to use for HTTP requests processing" + + " Default is set to 200. Initial number of threads is 8 when numHttpServerThreads >= 8." + ) + private int numHttpServerThreads = 200; @FieldContext( category = CATEGORY_SERVER, From ee8e67f2884840f117b55a066b99b06b3e9a7de8 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Thu, 17 Feb 2022 12:27:40 +0200 Subject: [PATCH 2/2] [Proxy] Set httpNumThreads=200 for proxy.conf --- conf/proxy.conf | 8 ++++---- deployment/terraform-ansible/templates/proxy.conf | 6 +++--- .../apache/pulsar/proxy/server/ProxyConfiguration.java | 5 +++-- pulsar-proxy/src/test/resources/proxy.conf | 6 +++--- 4 files changed, 13 insertions(+), 12 deletions(-) diff --git a/conf/proxy.conf b/conf/proxy.conf index 5d939591ce343..a13f73af224c7 100644 --- a/conf/proxy.conf +++ b/conf/proxy.conf @@ -198,9 +198,9 @@ httpReverseProxyConfigs= # so that clients see the data as soon as possible. httpOutputBufferSize=32768 -# Number of threads to use for HTTP requests processing. Default is -# 2 * Runtime.getRuntime().availableProcessors() -httpNumThreads= +# Maximum number of threads to use for HTTP requests processing. +# Default is set to 200. Initial number of threads is 8 when numHttpServerThreads >= 8. +httpNumThreads=200 # Enable the enforcement of limits on the incoming HTTP requests httpRequestsLimitEnabled=false @@ -269,4 +269,4 @@ zookeeperSessionTimeoutMs=-1 # ZooKeeper cache expiry time in seconds # Deprecated: use metadataStoreCacheExpirySeconds -zooKeeperCacheExpirySeconds=-1 \ No newline at end of file +zooKeeperCacheExpirySeconds=-1 diff --git a/deployment/terraform-ansible/templates/proxy.conf b/deployment/terraform-ansible/templates/proxy.conf index 7d04914fce2d0..f00f179c34aed 100644 --- a/deployment/terraform-ansible/templates/proxy.conf +++ b/deployment/terraform-ansible/templates/proxy.conf @@ -179,9 +179,9 @@ httpReverseProxyConfigs= # so that clients see the data as soon as possible. httpOutputBufferSize=32768 -# Number of threads to use for HTTP requests processing. Default is -# 2 * Runtime.getRuntime().availableProcessors() -httpNumThreads= +# Maximum number of threads to use for HTTP requests processing. +# Default is set to 200. Initial number of threads is 8 when numHttpServerThreads >= 8. +httpNumThreads=200 ### --- Token Authentication Provider --- ### diff --git a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConfiguration.java b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConfiguration.java index 20e4ddecc6f54..de4a8df1e45a2 100644 --- a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConfiguration.java +++ b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConfiguration.java @@ -601,9 +601,10 @@ public class ProxyConfiguration implements PulsarConfiguration { @FieldContext( minValue = 1, category = CATEGORY_HTTP, - doc = "Number of threads to use for HTTP requests processing" + doc = "Maximum number of threads to use for HTTP requests processing" + + " Default is set to 200. Initial number of threads is 8 when numHttpServerThreads >= 8." ) - private int httpNumThreads = Math.max(8, 2 * Runtime.getRuntime().availableProcessors()); + private int httpNumThreads = 200; @FieldContext( category = CATEGORY_SERVER, diff --git a/pulsar-proxy/src/test/resources/proxy.conf b/pulsar-proxy/src/test/resources/proxy.conf index 56c4a7eff4da5..fb6d115b16a40 100644 --- a/pulsar-proxy/src/test/resources/proxy.conf +++ b/pulsar-proxy/src/test/resources/proxy.conf @@ -189,9 +189,9 @@ httpReverseProxyConfigs= # so that clients see the data as soon as possible. httpOutputBufferSize=32768 -# Number of threads to use for HTTP requests processing. Default is -# 2 * Runtime.getRuntime().availableProcessors() -httpNumThreads= +# Maximum number of threads to use for HTTP requests processing. +# Default is set to 200. Initial number of threads is 8 when numHttpServerThreads >= 8. +httpNumThreads=200 # Enable the enforcement of limits on the incoming HTTP requests httpRequestsLimitEnabled=false