diff --git a/build/run_unit_group.sh b/build/run_unit_group.sh
index 351477aed1c92..2694505e0e098 100755
--- a/build/run_unit_group.sh
+++ b/build/run_unit_group.sh
@@ -85,6 +85,8 @@ function test_group_broker_group_2() {
function test_group_broker_group_3() {
mvn_test -pl pulsar-broker -Dgroups='broker-admin'
+ # run AdminApiTransactionMultiBrokerTest independently with a larger heap size
+ mvn_test -pl pulsar-broker -DtestMaxHeapSize=1500M -Dtest=org.apache.pulsar.broker.admin.v3.AdminApiTransactionMultiBrokerTest -DtestForkCount=1 -DtestReuseFork=false
}
function test_group_broker_group_4() {
diff --git a/conf/pulsar_env.sh b/conf/pulsar_env.sh
index c7bba23c234d9..3a069e31fdc90 100755
--- a/conf/pulsar_env.sh
+++ b/conf/pulsar_env.sh
@@ -94,3 +94,7 @@ PULSAR_EXTRA_OPTS="${PULSAR_EXTRA_OPTS:-" -Dpulsar.allocator.exit_on_oom=true -D
#Wait time before forcefully kill the pulsar server instance, if the stop is not successful
#PULSAR_STOP_TIMEOUT=
+# Enable semantically stable telemetry for JVM metrics, unless otherwise overridden by the user.
+if [ -z "$OTEL_SEMCONV_STABILITY_OPT_IN" ]; then
+ export OTEL_SEMCONV_STABILITY_OPT_IN=jvm
+fi
diff --git a/distribution/server/src/assemble/LICENSE.bin.txt b/distribution/server/src/assemble/LICENSE.bin.txt
index aec4df2a93af9..e619dc9800e4a 100644
--- a/distribution/server/src/assemble/LICENSE.bin.txt
+++ b/distribution/server/src/assemble/LICENSE.bin.txt
@@ -542,6 +542,8 @@ The Apache Software License, Version 2.0
- io.opentelemetry.instrumentation-opentelemetry-instrumentation-api-1.33.2.jar
- io.opentelemetry.instrumentation-opentelemetry-instrumentation-api-semconv-1.33.2-alpha.jar
- io.opentelemetry.instrumentation-opentelemetry-resources-1.33.2-alpha.jar
+ - io.opentelemetry.instrumentation-opentelemetry-runtime-telemetry-java17-1.33.2-alpha.jar
+ - io.opentelemetry.instrumentation-opentelemetry-runtime-telemetry-java8-1.33.2-alpha.jar
- io.opentelemetry.semconv-opentelemetry-semconv-1.25.0-alpha.jar
BSD 3-clause "New" or "Revised" License
diff --git a/pom.xml b/pom.xml
index 8f7ae2ed1fc68..33f14ccecd33f 100644
--- a/pom.xml
+++ b/pom.xml
@@ -115,6 +115,7 @@ flexible messaging model and an intuitive client API.
--add-opens jdk.management/com.sun.management.internal=ALL-UNNAMED
--add-opens java.base/jdk.internal.platform=ALL-UNNAMED
+ 1300M
true
4
false
@@ -1652,7 +1653,7 @@ flexible messaging model and an intuitive client API.
org.apache.maven.plugins
maven-surefire-plugin
- ${testJacocoAgentArgument} -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=${testHeapDumpPath} -XX:+ExitOnOutOfMemoryError -Xmx1G -XX:+UseZGC
+ ${testJacocoAgentArgument} -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=${testHeapDumpPath} -XX:+ExitOnOutOfMemoryError -Xmx${testMaxHeapSize} -XX:+UseZGC
-Dpulsar.allocator.pooled=true
-Dpulsar.allocator.leak_detection=Advanced
-Dpulsar.allocator.exit_on_oom=false
diff --git a/pulsar-broker-common/src/test/java/org/apache/pulsar/broker/stats/prometheus/PrometheusMetricsClient.java b/pulsar-broker-common/src/test/java/org/apache/pulsar/broker/stats/prometheus/PrometheusMetricsClient.java
index 6fd509690278d..6d724c289b52c 100644
--- a/pulsar-broker-common/src/test/java/org/apache/pulsar/broker/stats/prometheus/PrometheusMetricsClient.java
+++ b/pulsar-broker-common/src/test/java/org/apache/pulsar/broker/stats/prometheus/PrometheusMetricsClient.java
@@ -59,7 +59,7 @@ public static Multimap parseMetrics(String metrics) {
// or
// pulsar_subscriptions_count{cluster="standalone", namespace="public/default",
// topic="persistent://public/default/test-2"} 0.0
- Pattern pattern = Pattern.compile("^(\\w+)\\{([^}]+)}\\s([+-]?[\\d\\w.-]+)$");
+ Pattern pattern = Pattern.compile("^(\\w+)\\{([^}]+)}\\s([+-]?[\\d\\w.+-]+)$");
Pattern tagsPattern = Pattern.compile("(\\w+)=\"([^\"]+)\"(,\\s?)?");
Splitter.on("\n").split(metrics).forEach(line -> {
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/v3/AdminApiTransactionMultiBrokerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/v3/AdminApiTransactionMultiBrokerTest.java
index e2f4a5abdb9e0..113937c2558d9 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/v3/AdminApiTransactionMultiBrokerTest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/v3/AdminApiTransactionMultiBrokerTest.java
@@ -40,7 +40,7 @@
import org.testng.annotations.Test;
@Slf4j
-@Test(groups = "broker-admin")
+@Test(groups = "broker-admin-isolated")
public class AdminApiTransactionMultiBrokerTest extends TransactionTestBase {
private static final int NUM_BROKERS = 16;
diff --git a/pulsar-opentelemetry/pom.xml b/pulsar-opentelemetry/pom.xml
index 82a9658cc9d31..e32f1b81ff964 100644
--- a/pulsar-opentelemetry/pom.xml
+++ b/pulsar-opentelemetry/pom.xml
@@ -58,6 +58,10 @@
io.opentelemetry.semconv
opentelemetry-semconv
+
+ io.opentelemetry.instrumentation
+ opentelemetry-runtime-telemetry-java17
+
com.google.guava
@@ -130,6 +134,16 @@
+
+
+ org.apache.maven.plugins
+ maven-surefire-plugin
+
+
+ jvm
+
+
+
diff --git a/pulsar-opentelemetry/src/main/java/org/apache/pulsar/opentelemetry/OpenTelemetryService.java b/pulsar-opentelemetry/src/main/java/org/apache/pulsar/opentelemetry/OpenTelemetryService.java
index 16c4264be6d12..4560d3813d6dd 100644
--- a/pulsar-opentelemetry/src/main/java/org/apache/pulsar/opentelemetry/OpenTelemetryService.java
+++ b/pulsar-opentelemetry/src/main/java/org/apache/pulsar/opentelemetry/OpenTelemetryService.java
@@ -21,6 +21,7 @@
import static com.google.common.base.Preconditions.checkArgument;
import com.google.common.annotations.VisibleForTesting;
import io.opentelemetry.api.OpenTelemetry;
+import io.opentelemetry.instrumentation.runtimemetrics.java17.RuntimeMetrics;
import io.opentelemetry.sdk.OpenTelemetrySdk;
import io.opentelemetry.sdk.autoconfigure.AutoConfiguredOpenTelemetrySdk;
import io.opentelemetry.sdk.autoconfigure.AutoConfiguredOpenTelemetrySdkBuilder;
@@ -29,6 +30,7 @@
import java.io.Closeable;
import java.util.Map;
import java.util.Objects;
+import java.util.concurrent.atomic.AtomicReference;
import java.util.function.Consumer;
import lombok.Builder;
import org.apache.commons.lang3.StringUtils;
@@ -42,7 +44,9 @@ public class OpenTelemetryService implements Closeable {
public static final String OTEL_SDK_DISABLED_KEY = "otel.sdk.disabled";
static final int MAX_CARDINALITY_LIMIT = 10000;
- private final OpenTelemetrySdk openTelemetrySdk;
+ private final AtomicReference openTelemetrySdkReference = new AtomicReference<>();
+
+ private final AtomicReference runtimeMetricsReference = new AtomicReference<>();
/**
* Instantiates the OpenTelemetry SDK. All attributes are overridden by system properties or environment
@@ -94,15 +98,28 @@ public OpenTelemetryService(String clusterName,
builderCustomizer.accept(sdkBuilder);
}
- openTelemetrySdk = sdkBuilder.build().getOpenTelemetrySdk();
+ openTelemetrySdkReference.set(sdkBuilder.build().getOpenTelemetrySdk());
+
+ // For a list of exposed metrics, see https://opentelemetry.io/docs/specs/semconv/runtime/jvm-metrics/
+ runtimeMetricsReference.set(RuntimeMetrics.builder(openTelemetrySdkReference.get())
+ .enableAllFeatures()
+ .enableExperimentalJmxTelemetry()
+ .build());
}
public OpenTelemetry getOpenTelemetry() {
- return openTelemetrySdk;
+ return openTelemetrySdkReference.get();
}
@Override
public void close() {
- openTelemetrySdk.close();
+ RuntimeMetrics runtimeMetrics = runtimeMetricsReference.getAndSet(null);
+ if (runtimeMetrics != null) {
+ runtimeMetrics.close();
+ }
+ OpenTelemetrySdk openTelemetrySdk = openTelemetrySdkReference.getAndSet(null);
+ if (openTelemetrySdk != null) {
+ openTelemetrySdk.close();
+ }
}
}
diff --git a/pulsar-opentelemetry/src/test/java/org/apache/pulsar/opentelemetry/OpenTelemetryServiceTest.java b/pulsar-opentelemetry/src/test/java/org/apache/pulsar/opentelemetry/OpenTelemetryServiceTest.java
index bf404496a2eca..31a6c60f83afe 100644
--- a/pulsar-opentelemetry/src/test/java/org/apache/pulsar/opentelemetry/OpenTelemetryServiceTest.java
+++ b/pulsar-opentelemetry/src/test/java/org/apache/pulsar/opentelemetry/OpenTelemetryServiceTest.java
@@ -198,4 +198,52 @@ public void testServiceIsDisabledByDefault() throws Exception {
// Validate that the callback has not being called.
assertThat(callback).isFalse();
}
+
+ @Test
+ public void testJvmRuntimeMetrics() {
+ // Attempt collection of GC metrics. The metrics should be populated regardless if GC is triggered or not.
+ Runtime.getRuntime().gc();
+
+ var metrics = reader.collectAllMetrics();
+
+ // Process Metrics
+ // Replaces process_cpu_seconds_total
+ assertThat(metrics).anySatisfy(metric -> assertThat(metric).hasName("jvm.cpu.time"));
+
+ // Memory Metrics
+ // Replaces jvm_memory_bytes_used
+ assertThat(metrics).anySatisfy(metric -> assertThat(metric).hasName("jvm.memory.used"));
+ // Replaces jvm_memory_bytes_committed
+ assertThat(metrics).anySatisfy(metric -> assertThat(metric).hasName("jvm.memory.committed"));
+ // Replaces jvm_memory_bytes_max
+ assertThat(metrics).anySatisfy(metric -> assertThat(metric).hasName("jvm.memory.limit"));
+ // Replaces jvm_memory_bytes_init
+ assertThat(metrics).anySatisfy(metric -> assertThat(metric).hasName("jvm.memory.init"));
+ // Replaces jvm_memory_pool_allocated_bytes_total
+ assertThat(metrics).anySatisfy(metric -> assertThat(metric).hasName("jvm.memory.used_after_last_gc"));
+
+ // Buffer Pool Metrics
+ // Replaces jvm_buffer_pool_used_bytes
+ assertThat(metrics).anySatisfy(metric -> assertThat(metric).hasName("jvm.buffer.memory.usage"));
+ // Replaces jvm_buffer_pool_capacity_bytes
+ assertThat(metrics).anySatisfy(metric -> assertThat(metric).hasName("jvm.buffer.memory.limit"));
+ // Replaces jvm_buffer_pool_used_buffers
+ assertThat(metrics).anySatisfy(metric -> assertThat(metric).hasName("jvm.buffer.count"));
+
+ // Garbage Collector Metrics
+ // Replaces jvm_gc_collection_seconds
+ assertThat(metrics).anySatisfy(metric -> assertThat(metric).hasName("jvm.gc.duration"));
+
+ // Thread Metrics
+ // Replaces jvm_threads_state, jvm_threads_current and jvm_threads_daemon
+ assertThat(metrics).anySatisfy(metric -> assertThat(metric).hasName("jvm.thread.count"));
+
+ // Class Loading Metrics
+ // Replaces jvm_classes_currently_loaded
+ assertThat(metrics).anySatisfy(metric -> assertThat(metric).hasName("jvm.class.count"));
+ // Replaces jvm_classes_loaded_total
+ assertThat(metrics).anySatisfy(metric -> assertThat(metric).hasName("jvm.class.loaded"));
+ // Replaces jvm_classes_unloaded_total
+ assertThat(metrics).anySatisfy(metric -> assertThat(metric).hasName("jvm.class.unloaded"));
+ }
}
diff --git a/tests/docker-images/latest-version-image/conf/bookie.conf b/tests/docker-images/latest-version-image/conf/bookie.conf
index 07547bcaef6d3..df7501057a58f 100644
--- a/tests/docker-images/latest-version-image/conf/bookie.conf
+++ b/tests/docker-images/latest-version-image/conf/bookie.conf
@@ -22,7 +22,7 @@ autostart=false
redirect_stderr=true
stdout_logfile=/var/log/pulsar/bookie.log
directory=/pulsar
-environment=PULSAR_MEM="-Xmx128M -XX:MaxDirectMemorySize=512M",PULSAR_GC="-XX:+UseZGC"
+environment=PULSAR_MEM="-Xmx128M -XX:MaxDirectMemorySize=512M -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/var/log/pulsar -XX:+ExitOnOutOfMemoryError",PULSAR_GC="-XX:+UseZGC"
command=/pulsar/bin/pulsar bookie
user=pulsar
stopwaitsecs=15
diff --git a/tests/docker-images/latest-version-image/conf/broker.conf b/tests/docker-images/latest-version-image/conf/broker.conf
index 63be36437741b..790dace8d6d85 100644
--- a/tests/docker-images/latest-version-image/conf/broker.conf
+++ b/tests/docker-images/latest-version-image/conf/broker.conf
@@ -22,7 +22,7 @@ autostart=false
redirect_stderr=true
stdout_logfile=/var/log/pulsar/broker.log
directory=/pulsar
-environment=PULSAR_MEM="-Xmx128M",PULSAR_GC="-XX:+UseZGC"
+environment=PULSAR_MEM="-Xmx150M -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/var/log/pulsar -XX:+ExitOnOutOfMemoryError",PULSAR_GC="-XX:+UseZGC"
command=/pulsar/bin/pulsar broker
user=pulsar
stopwaitsecs=15
diff --git a/tests/docker-images/latest-version-image/conf/functions_worker.conf b/tests/docker-images/latest-version-image/conf/functions_worker.conf
index 6feb660231cec..b5d151ce3f9be 100644
--- a/tests/docker-images/latest-version-image/conf/functions_worker.conf
+++ b/tests/docker-images/latest-version-image/conf/functions_worker.conf
@@ -22,7 +22,7 @@ autostart=false
redirect_stderr=true
stdout_logfile=/var/log/pulsar/functions_worker.log
directory=/pulsar
-environment=PULSAR_MEM="-Xmx128M -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/pulsar/logs/functions",PULSAR_GC="-XX:+UseZGC"
+environment=PULSAR_MEM="-Xmx150M -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/var/log/pulsar -XX:+ExitOnOutOfMemoryError",PULSAR_GC="-XX:+UseZGC"
command=/pulsar/bin/pulsar functions-worker
user=pulsar
stopwaitsecs=15
\ No newline at end of file
diff --git a/tests/docker-images/latest-version-image/conf/global-zk.conf b/tests/docker-images/latest-version-image/conf/global-zk.conf
index e5ffd2eb9e769..ef521506846c8 100644
--- a/tests/docker-images/latest-version-image/conf/global-zk.conf
+++ b/tests/docker-images/latest-version-image/conf/global-zk.conf
@@ -22,7 +22,7 @@ autostart=false
redirect_stderr=true
stdout_logfile=/var/log/pulsar/global-zk.log
directory=/pulsar
-environment=PULSAR_MEM="-Xmx128M",PULSAR_GC="-XX:+UseZGC"
+environment=PULSAR_MEM="-Xmx128M -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/var/log/pulsar -XX:+ExitOnOutOfMemoryError",PULSAR_GC="-XX:+UseZGC"
command=/pulsar/bin/pulsar configuration-store
user=pulsar
stopwaitsecs=15
\ No newline at end of file
diff --git a/tests/docker-images/latest-version-image/conf/local-zk.conf b/tests/docker-images/latest-version-image/conf/local-zk.conf
index c96543db8a865..d6bfdcb621b43 100644
--- a/tests/docker-images/latest-version-image/conf/local-zk.conf
+++ b/tests/docker-images/latest-version-image/conf/local-zk.conf
@@ -22,7 +22,7 @@ autostart=false
redirect_stderr=true
stdout_logfile=/var/log/pulsar/local-zk.log
directory=/pulsar
-environment=PULSAR_MEM="-Xmx128M",PULSAR_GC="-XX:+UseZGC"
+environment=PULSAR_MEM="-Xmx128M -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/var/log/pulsar -XX:+ExitOnOutOfMemoryError",PULSAR_GC="-XX:+UseZGC"
command=/pulsar/bin/pulsar zookeeper
user=pulsar
stopwaitsecs=15
\ No newline at end of file
diff --git a/tests/docker-images/latest-version-image/conf/proxy.conf b/tests/docker-images/latest-version-image/conf/proxy.conf
index 343a0f9614e30..17a0a658b4226 100644
--- a/tests/docker-images/latest-version-image/conf/proxy.conf
+++ b/tests/docker-images/latest-version-image/conf/proxy.conf
@@ -22,7 +22,7 @@ autostart=false
redirect_stderr=true
stdout_logfile=/var/log/pulsar/proxy.log
directory=/pulsar
-environment=PULSAR_MEM="-Xmx128M",PULSAR_GC="-XX:+UseZGC"
+environment=PULSAR_MEM="-Xmx150M -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/var/log/pulsar -XX:+ExitOnOutOfMemoryError",PULSAR_GC="-XX:+UseZGC"
command=/pulsar/bin/pulsar proxy
user=pulsar
stopwaitsecs=15
\ No newline at end of file
diff --git a/tests/docker-images/latest-version-image/conf/websocket.conf b/tests/docker-images/latest-version-image/conf/websocket.conf
index 0418c4cbc26a3..7625dba3e030d 100644
--- a/tests/docker-images/latest-version-image/conf/websocket.conf
+++ b/tests/docker-images/latest-version-image/conf/websocket.conf
@@ -22,7 +22,7 @@ autostart=false
redirect_stderr=true
stdout_logfile=/var/log/pulsar/pulsar-websocket.log
directory=/pulsar
-environment=PULSAR_MEM="-Xmx128M",PULSAR_GC="-XX:+UseZGC"
+environment=PULSAR_MEM="-Xmx150M -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/var/log/pulsar -XX:+ExitOnOutOfMemoryError",PULSAR_GC="-XX:+UseZGC"
command=/pulsar/bin/pulsar websocket
user=pulsar
stopwaitsecs=15
\ No newline at end of file