diff --git a/build/run_unit_group.sh b/build/run_unit_group.sh index 351477aed1c92..2694505e0e098 100755 --- a/build/run_unit_group.sh +++ b/build/run_unit_group.sh @@ -85,6 +85,8 @@ function test_group_broker_group_2() { function test_group_broker_group_3() { mvn_test -pl pulsar-broker -Dgroups='broker-admin' + # run AdminApiTransactionMultiBrokerTest independently with a larger heap size + mvn_test -pl pulsar-broker -DtestMaxHeapSize=1500M -Dtest=org.apache.pulsar.broker.admin.v3.AdminApiTransactionMultiBrokerTest -DtestForkCount=1 -DtestReuseFork=false } function test_group_broker_group_4() { diff --git a/conf/pulsar_env.sh b/conf/pulsar_env.sh index c7bba23c234d9..3a069e31fdc90 100755 --- a/conf/pulsar_env.sh +++ b/conf/pulsar_env.sh @@ -94,3 +94,7 @@ PULSAR_EXTRA_OPTS="${PULSAR_EXTRA_OPTS:-" -Dpulsar.allocator.exit_on_oom=true -D #Wait time before forcefully kill the pulsar server instance, if the stop is not successful #PULSAR_STOP_TIMEOUT= +# Enable semantically stable telemetry for JVM metrics, unless otherwise overridden by the user. +if [ -z "$OTEL_SEMCONV_STABILITY_OPT_IN" ]; then + export OTEL_SEMCONV_STABILITY_OPT_IN=jvm +fi diff --git a/distribution/server/src/assemble/LICENSE.bin.txt b/distribution/server/src/assemble/LICENSE.bin.txt index aec4df2a93af9..e619dc9800e4a 100644 --- a/distribution/server/src/assemble/LICENSE.bin.txt +++ b/distribution/server/src/assemble/LICENSE.bin.txt @@ -542,6 +542,8 @@ The Apache Software License, Version 2.0 - io.opentelemetry.instrumentation-opentelemetry-instrumentation-api-1.33.2.jar - io.opentelemetry.instrumentation-opentelemetry-instrumentation-api-semconv-1.33.2-alpha.jar - io.opentelemetry.instrumentation-opentelemetry-resources-1.33.2-alpha.jar + - io.opentelemetry.instrumentation-opentelemetry-runtime-telemetry-java17-1.33.2-alpha.jar + - io.opentelemetry.instrumentation-opentelemetry-runtime-telemetry-java8-1.33.2-alpha.jar - io.opentelemetry.semconv-opentelemetry-semconv-1.25.0-alpha.jar BSD 3-clause "New" or "Revised" License diff --git a/pom.xml b/pom.xml index 8f7ae2ed1fc68..33f14ccecd33f 100644 --- a/pom.xml +++ b/pom.xml @@ -115,6 +115,7 @@ flexible messaging model and an intuitive client API. --add-opens jdk.management/com.sun.management.internal=ALL-UNNAMED --add-opens java.base/jdk.internal.platform=ALL-UNNAMED + 1300M true 4 false @@ -1652,7 +1653,7 @@ flexible messaging model and an intuitive client API. org.apache.maven.plugins maven-surefire-plugin - ${testJacocoAgentArgument} -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=${testHeapDumpPath} -XX:+ExitOnOutOfMemoryError -Xmx1G -XX:+UseZGC + ${testJacocoAgentArgument} -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=${testHeapDumpPath} -XX:+ExitOnOutOfMemoryError -Xmx${testMaxHeapSize} -XX:+UseZGC -Dpulsar.allocator.pooled=true -Dpulsar.allocator.leak_detection=Advanced -Dpulsar.allocator.exit_on_oom=false diff --git a/pulsar-broker-common/src/test/java/org/apache/pulsar/broker/stats/prometheus/PrometheusMetricsClient.java b/pulsar-broker-common/src/test/java/org/apache/pulsar/broker/stats/prometheus/PrometheusMetricsClient.java index 6fd509690278d..6d724c289b52c 100644 --- a/pulsar-broker-common/src/test/java/org/apache/pulsar/broker/stats/prometheus/PrometheusMetricsClient.java +++ b/pulsar-broker-common/src/test/java/org/apache/pulsar/broker/stats/prometheus/PrometheusMetricsClient.java @@ -59,7 +59,7 @@ public static Multimap parseMetrics(String metrics) { // or // pulsar_subscriptions_count{cluster="standalone", namespace="public/default", // topic="persistent://public/default/test-2"} 0.0 - Pattern pattern = Pattern.compile("^(\\w+)\\{([^}]+)}\\s([+-]?[\\d\\w.-]+)$"); + Pattern pattern = Pattern.compile("^(\\w+)\\{([^}]+)}\\s([+-]?[\\d\\w.+-]+)$"); Pattern tagsPattern = Pattern.compile("(\\w+)=\"([^\"]+)\"(,\\s?)?"); Splitter.on("\n").split(metrics).forEach(line -> { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/v3/AdminApiTransactionMultiBrokerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/v3/AdminApiTransactionMultiBrokerTest.java index e2f4a5abdb9e0..113937c2558d9 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/v3/AdminApiTransactionMultiBrokerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/v3/AdminApiTransactionMultiBrokerTest.java @@ -40,7 +40,7 @@ import org.testng.annotations.Test; @Slf4j -@Test(groups = "broker-admin") +@Test(groups = "broker-admin-isolated") public class AdminApiTransactionMultiBrokerTest extends TransactionTestBase { private static final int NUM_BROKERS = 16; diff --git a/pulsar-opentelemetry/pom.xml b/pulsar-opentelemetry/pom.xml index 82a9658cc9d31..e32f1b81ff964 100644 --- a/pulsar-opentelemetry/pom.xml +++ b/pulsar-opentelemetry/pom.xml @@ -58,6 +58,10 @@ io.opentelemetry.semconv opentelemetry-semconv + + io.opentelemetry.instrumentation + opentelemetry-runtime-telemetry-java17 + com.google.guava @@ -130,6 +134,16 @@ + + + org.apache.maven.plugins + maven-surefire-plugin + + + jvm + + + diff --git a/pulsar-opentelemetry/src/main/java/org/apache/pulsar/opentelemetry/OpenTelemetryService.java b/pulsar-opentelemetry/src/main/java/org/apache/pulsar/opentelemetry/OpenTelemetryService.java index 16c4264be6d12..4560d3813d6dd 100644 --- a/pulsar-opentelemetry/src/main/java/org/apache/pulsar/opentelemetry/OpenTelemetryService.java +++ b/pulsar-opentelemetry/src/main/java/org/apache/pulsar/opentelemetry/OpenTelemetryService.java @@ -21,6 +21,7 @@ import static com.google.common.base.Preconditions.checkArgument; import com.google.common.annotations.VisibleForTesting; import io.opentelemetry.api.OpenTelemetry; +import io.opentelemetry.instrumentation.runtimemetrics.java17.RuntimeMetrics; import io.opentelemetry.sdk.OpenTelemetrySdk; import io.opentelemetry.sdk.autoconfigure.AutoConfiguredOpenTelemetrySdk; import io.opentelemetry.sdk.autoconfigure.AutoConfiguredOpenTelemetrySdkBuilder; @@ -29,6 +30,7 @@ import java.io.Closeable; import java.util.Map; import java.util.Objects; +import java.util.concurrent.atomic.AtomicReference; import java.util.function.Consumer; import lombok.Builder; import org.apache.commons.lang3.StringUtils; @@ -42,7 +44,9 @@ public class OpenTelemetryService implements Closeable { public static final String OTEL_SDK_DISABLED_KEY = "otel.sdk.disabled"; static final int MAX_CARDINALITY_LIMIT = 10000; - private final OpenTelemetrySdk openTelemetrySdk; + private final AtomicReference openTelemetrySdkReference = new AtomicReference<>(); + + private final AtomicReference runtimeMetricsReference = new AtomicReference<>(); /** * Instantiates the OpenTelemetry SDK. All attributes are overridden by system properties or environment @@ -94,15 +98,28 @@ public OpenTelemetryService(String clusterName, builderCustomizer.accept(sdkBuilder); } - openTelemetrySdk = sdkBuilder.build().getOpenTelemetrySdk(); + openTelemetrySdkReference.set(sdkBuilder.build().getOpenTelemetrySdk()); + + // For a list of exposed metrics, see https://opentelemetry.io/docs/specs/semconv/runtime/jvm-metrics/ + runtimeMetricsReference.set(RuntimeMetrics.builder(openTelemetrySdkReference.get()) + .enableAllFeatures() + .enableExperimentalJmxTelemetry() + .build()); } public OpenTelemetry getOpenTelemetry() { - return openTelemetrySdk; + return openTelemetrySdkReference.get(); } @Override public void close() { - openTelemetrySdk.close(); + RuntimeMetrics runtimeMetrics = runtimeMetricsReference.getAndSet(null); + if (runtimeMetrics != null) { + runtimeMetrics.close(); + } + OpenTelemetrySdk openTelemetrySdk = openTelemetrySdkReference.getAndSet(null); + if (openTelemetrySdk != null) { + openTelemetrySdk.close(); + } } } diff --git a/pulsar-opentelemetry/src/test/java/org/apache/pulsar/opentelemetry/OpenTelemetryServiceTest.java b/pulsar-opentelemetry/src/test/java/org/apache/pulsar/opentelemetry/OpenTelemetryServiceTest.java index bf404496a2eca..31a6c60f83afe 100644 --- a/pulsar-opentelemetry/src/test/java/org/apache/pulsar/opentelemetry/OpenTelemetryServiceTest.java +++ b/pulsar-opentelemetry/src/test/java/org/apache/pulsar/opentelemetry/OpenTelemetryServiceTest.java @@ -198,4 +198,52 @@ public void testServiceIsDisabledByDefault() throws Exception { // Validate that the callback has not being called. assertThat(callback).isFalse(); } + + @Test + public void testJvmRuntimeMetrics() { + // Attempt collection of GC metrics. The metrics should be populated regardless if GC is triggered or not. + Runtime.getRuntime().gc(); + + var metrics = reader.collectAllMetrics(); + + // Process Metrics + // Replaces process_cpu_seconds_total + assertThat(metrics).anySatisfy(metric -> assertThat(metric).hasName("jvm.cpu.time")); + + // Memory Metrics + // Replaces jvm_memory_bytes_used + assertThat(metrics).anySatisfy(metric -> assertThat(metric).hasName("jvm.memory.used")); + // Replaces jvm_memory_bytes_committed + assertThat(metrics).anySatisfy(metric -> assertThat(metric).hasName("jvm.memory.committed")); + // Replaces jvm_memory_bytes_max + assertThat(metrics).anySatisfy(metric -> assertThat(metric).hasName("jvm.memory.limit")); + // Replaces jvm_memory_bytes_init + assertThat(metrics).anySatisfy(metric -> assertThat(metric).hasName("jvm.memory.init")); + // Replaces jvm_memory_pool_allocated_bytes_total + assertThat(metrics).anySatisfy(metric -> assertThat(metric).hasName("jvm.memory.used_after_last_gc")); + + // Buffer Pool Metrics + // Replaces jvm_buffer_pool_used_bytes + assertThat(metrics).anySatisfy(metric -> assertThat(metric).hasName("jvm.buffer.memory.usage")); + // Replaces jvm_buffer_pool_capacity_bytes + assertThat(metrics).anySatisfy(metric -> assertThat(metric).hasName("jvm.buffer.memory.limit")); + // Replaces jvm_buffer_pool_used_buffers + assertThat(metrics).anySatisfy(metric -> assertThat(metric).hasName("jvm.buffer.count")); + + // Garbage Collector Metrics + // Replaces jvm_gc_collection_seconds + assertThat(metrics).anySatisfy(metric -> assertThat(metric).hasName("jvm.gc.duration")); + + // Thread Metrics + // Replaces jvm_threads_state, jvm_threads_current and jvm_threads_daemon + assertThat(metrics).anySatisfy(metric -> assertThat(metric).hasName("jvm.thread.count")); + + // Class Loading Metrics + // Replaces jvm_classes_currently_loaded + assertThat(metrics).anySatisfy(metric -> assertThat(metric).hasName("jvm.class.count")); + // Replaces jvm_classes_loaded_total + assertThat(metrics).anySatisfy(metric -> assertThat(metric).hasName("jvm.class.loaded")); + // Replaces jvm_classes_unloaded_total + assertThat(metrics).anySatisfy(metric -> assertThat(metric).hasName("jvm.class.unloaded")); + } } diff --git a/tests/docker-images/latest-version-image/conf/bookie.conf b/tests/docker-images/latest-version-image/conf/bookie.conf index 07547bcaef6d3..df7501057a58f 100644 --- a/tests/docker-images/latest-version-image/conf/bookie.conf +++ b/tests/docker-images/latest-version-image/conf/bookie.conf @@ -22,7 +22,7 @@ autostart=false redirect_stderr=true stdout_logfile=/var/log/pulsar/bookie.log directory=/pulsar -environment=PULSAR_MEM="-Xmx128M -XX:MaxDirectMemorySize=512M",PULSAR_GC="-XX:+UseZGC" +environment=PULSAR_MEM="-Xmx128M -XX:MaxDirectMemorySize=512M -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/var/log/pulsar -XX:+ExitOnOutOfMemoryError",PULSAR_GC="-XX:+UseZGC" command=/pulsar/bin/pulsar bookie user=pulsar stopwaitsecs=15 diff --git a/tests/docker-images/latest-version-image/conf/broker.conf b/tests/docker-images/latest-version-image/conf/broker.conf index 63be36437741b..790dace8d6d85 100644 --- a/tests/docker-images/latest-version-image/conf/broker.conf +++ b/tests/docker-images/latest-version-image/conf/broker.conf @@ -22,7 +22,7 @@ autostart=false redirect_stderr=true stdout_logfile=/var/log/pulsar/broker.log directory=/pulsar -environment=PULSAR_MEM="-Xmx128M",PULSAR_GC="-XX:+UseZGC" +environment=PULSAR_MEM="-Xmx150M -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/var/log/pulsar -XX:+ExitOnOutOfMemoryError",PULSAR_GC="-XX:+UseZGC" command=/pulsar/bin/pulsar broker user=pulsar stopwaitsecs=15 diff --git a/tests/docker-images/latest-version-image/conf/functions_worker.conf b/tests/docker-images/latest-version-image/conf/functions_worker.conf index 6feb660231cec..b5d151ce3f9be 100644 --- a/tests/docker-images/latest-version-image/conf/functions_worker.conf +++ b/tests/docker-images/latest-version-image/conf/functions_worker.conf @@ -22,7 +22,7 @@ autostart=false redirect_stderr=true stdout_logfile=/var/log/pulsar/functions_worker.log directory=/pulsar -environment=PULSAR_MEM="-Xmx128M -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/pulsar/logs/functions",PULSAR_GC="-XX:+UseZGC" +environment=PULSAR_MEM="-Xmx150M -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/var/log/pulsar -XX:+ExitOnOutOfMemoryError",PULSAR_GC="-XX:+UseZGC" command=/pulsar/bin/pulsar functions-worker user=pulsar stopwaitsecs=15 \ No newline at end of file diff --git a/tests/docker-images/latest-version-image/conf/global-zk.conf b/tests/docker-images/latest-version-image/conf/global-zk.conf index e5ffd2eb9e769..ef521506846c8 100644 --- a/tests/docker-images/latest-version-image/conf/global-zk.conf +++ b/tests/docker-images/latest-version-image/conf/global-zk.conf @@ -22,7 +22,7 @@ autostart=false redirect_stderr=true stdout_logfile=/var/log/pulsar/global-zk.log directory=/pulsar -environment=PULSAR_MEM="-Xmx128M",PULSAR_GC="-XX:+UseZGC" +environment=PULSAR_MEM="-Xmx128M -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/var/log/pulsar -XX:+ExitOnOutOfMemoryError",PULSAR_GC="-XX:+UseZGC" command=/pulsar/bin/pulsar configuration-store user=pulsar stopwaitsecs=15 \ No newline at end of file diff --git a/tests/docker-images/latest-version-image/conf/local-zk.conf b/tests/docker-images/latest-version-image/conf/local-zk.conf index c96543db8a865..d6bfdcb621b43 100644 --- a/tests/docker-images/latest-version-image/conf/local-zk.conf +++ b/tests/docker-images/latest-version-image/conf/local-zk.conf @@ -22,7 +22,7 @@ autostart=false redirect_stderr=true stdout_logfile=/var/log/pulsar/local-zk.log directory=/pulsar -environment=PULSAR_MEM="-Xmx128M",PULSAR_GC="-XX:+UseZGC" +environment=PULSAR_MEM="-Xmx128M -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/var/log/pulsar -XX:+ExitOnOutOfMemoryError",PULSAR_GC="-XX:+UseZGC" command=/pulsar/bin/pulsar zookeeper user=pulsar stopwaitsecs=15 \ No newline at end of file diff --git a/tests/docker-images/latest-version-image/conf/proxy.conf b/tests/docker-images/latest-version-image/conf/proxy.conf index 343a0f9614e30..17a0a658b4226 100644 --- a/tests/docker-images/latest-version-image/conf/proxy.conf +++ b/tests/docker-images/latest-version-image/conf/proxy.conf @@ -22,7 +22,7 @@ autostart=false redirect_stderr=true stdout_logfile=/var/log/pulsar/proxy.log directory=/pulsar -environment=PULSAR_MEM="-Xmx128M",PULSAR_GC="-XX:+UseZGC" +environment=PULSAR_MEM="-Xmx150M -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/var/log/pulsar -XX:+ExitOnOutOfMemoryError",PULSAR_GC="-XX:+UseZGC" command=/pulsar/bin/pulsar proxy user=pulsar stopwaitsecs=15 \ No newline at end of file diff --git a/tests/docker-images/latest-version-image/conf/websocket.conf b/tests/docker-images/latest-version-image/conf/websocket.conf index 0418c4cbc26a3..7625dba3e030d 100644 --- a/tests/docker-images/latest-version-image/conf/websocket.conf +++ b/tests/docker-images/latest-version-image/conf/websocket.conf @@ -22,7 +22,7 @@ autostart=false redirect_stderr=true stdout_logfile=/var/log/pulsar/pulsar-websocket.log directory=/pulsar -environment=PULSAR_MEM="-Xmx128M",PULSAR_GC="-XX:+UseZGC" +environment=PULSAR_MEM="-Xmx150M -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/var/log/pulsar -XX:+ExitOnOutOfMemoryError",PULSAR_GC="-XX:+UseZGC" command=/pulsar/bin/pulsar websocket user=pulsar stopwaitsecs=15 \ No newline at end of file