From a721ed8cf71ccb3978943c259a03c604a68b8ce3 Mon Sep 17 00:00:00 2001 From: Daniel Mitterdorfer Date: Thu, 7 Dec 2023 16:45:49 +0100 Subject: [PATCH 1/7] [Profiling] Query in parallel only if beneficial (#103061) With this commit we check index allocation before we do key-value lookups. To reduce latency, key-value lookups are done in parallel for multiple slices of data. However, on nodes with spinning disks, parallel accesses are harmful. Therefore, we check whether any index is allocated either to the warm or cold tier (which are usually on spinning disks) and disable parallel key-value lookups. This has improved latency on the warm tier by about 10% in our experiments. --- docs/changelog/103061.yaml | 5 + .../xpack/profiling/IndexAllocation.java | 60 +++++++++ .../TransportGetStackTracesAction.java | 22 +++- .../xpack/profiling/IndexAllocationTests.java | 122 ++++++++++++++++++ .../TransportGetStackTracesActionTests.java | 7 + 5 files changed, 211 insertions(+), 5 deletions(-) create mode 100644 docs/changelog/103061.yaml create mode 100644 x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/IndexAllocation.java create mode 100644 x-pack/plugin/profiling/src/test/java/org/elasticsearch/xpack/profiling/IndexAllocationTests.java diff --git a/docs/changelog/103061.yaml b/docs/changelog/103061.yaml new file mode 100644 index 0000000000000..558429493ac6f --- /dev/null +++ b/docs/changelog/103061.yaml @@ -0,0 +1,5 @@ +pr: 103061 +summary: "[Profiling] Query in parallel only if beneficial" +area: Application +type: bug +issues: [] diff --git a/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/IndexAllocation.java b/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/IndexAllocation.java new file mode 100644 index 0000000000000..701b2d8d8728d --- /dev/null +++ b/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/IndexAllocation.java @@ -0,0 +1,60 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.profiling; + +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.routing.IndexRoutingTable; +import org.elasticsearch.cluster.routing.ShardRouting; +import org.elasticsearch.cluster.routing.allocation.DataTier; +import org.elasticsearch.index.Index; + +import java.util.List; +import java.util.function.Predicate; + +final class IndexAllocation { + private IndexAllocation() { + // no instances intended + } + + static boolean isAnyAssignedToNode(ClusterState state, List indices, Predicate nodePredicate) { + for (Index index : indices) { + IndexMetadata metadata = state.getMetadata().index(index); + if (metadata == null) { + continue; + } + IndexRoutingTable routingTable = state.routingTable().index(index); + if (routingTable == null) { + continue; + } + for (ShardRouting shardRouting : routingTable.randomAllActiveShardsIt()) { + if (shardRouting.assignedToNode() == false) { + continue; + } + DiscoveryNode assignedNode = state.nodes().get(shardRouting.currentNodeId()); + if (nodePredicate.test(assignedNode)) { + return true; + } + } + } + return false; + } + + /** + * Determines whether any of the provided indices is allocated to the warm or cold tier. Machines on these + * tiers usually use spinning disks. + * + * @param state Current cluster state. + * @param indices A list of indices to check. + * @return true iff at least one index is allocated to either a warm or cold data node. + */ + static boolean isAnyOnWarmOrColdTier(ClusterState state, List indices) { + return isAnyAssignedToNode(state, indices, n -> DataTier.isWarmNode(n) || DataTier.isColdNode(n)); + } +} diff --git a/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/TransportGetStackTracesAction.java b/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/TransportGetStackTracesAction.java index 27feb8cc9e22a..000b448696985 100644 --- a/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/TransportGetStackTracesAction.java +++ b/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/TransportGetStackTracesAction.java @@ -401,9 +401,12 @@ private void retrieveStackTraces( return; } List eventIds = new ArrayList<>(responseBuilder.getStackTraceEvents().keySet()); - List> slicedEventIds = sliced(eventIds, desiredSlices); ClusterState clusterState = clusterService.state(); List indices = resolver.resolve(clusterState, "profiling-stacktraces", responseBuilder.getStart(), responseBuilder.getEnd()); + // Avoid parallelism if there is potential we are on spinning disks (frozen tier uses searchable snapshots) + int sliceCount = IndexAllocation.isAnyOnWarmOrColdTier(clusterState, indices) ? 1 : desiredSlices; + log.trace("Using [{}] slice(s) to lookup stacktraces.", sliceCount); + List> slicedEventIds = sliced(eventIds, sliceCount); // Build a set of unique host IDs. Set uniqueHostIDs = new HashSet<>(responseBuilder.getHostEventCounts().size()); @@ -457,7 +460,7 @@ private void retrieveStackTraces( // package private for testing static List> sliced(List c, int slices) { - if (c.size() <= slices) { + if (c.size() <= slices || slices == 1) { return List.of(c); } List> slicedList = new ArrayList<>(); @@ -621,9 +624,6 @@ private void retrieveStackTraceDetails( if (mayNotifyOfCancellation(submitTask, submitListener)) { return; } - - List> slicedStackFrameIds = sliced(stackFrameIds, desiredDetailSlices); - List> slicedExecutableIds = sliced(executableIds, desiredDetailSlices); List stackFrameIndices = resolver.resolve( clusterState, "profiling-stackframes", @@ -636,6 +636,18 @@ private void retrieveStackTraceDetails( responseBuilder.getStart(), responseBuilder.getEnd() ); + // Avoid parallelism if there is potential we are on spinning disks (frozen tier uses searchable snapshots) + int stackFrameSliceCount = IndexAllocation.isAnyOnWarmOrColdTier(clusterState, stackFrameIndices) ? 1 : desiredDetailSlices; + int executableSliceCount = IndexAllocation.isAnyOnWarmOrColdTier(clusterState, executableIndices) ? 1 : desiredDetailSlices; + log.trace( + "Using [{}] slice(s) to lookup stack frames and [{}] slice(s) to lookup executables.", + stackFrameSliceCount, + executableSliceCount + ); + + List> slicedStackFrameIds = sliced(stackFrameIds, stackFrameSliceCount); + List> slicedExecutableIds = sliced(executableIds, executableSliceCount); + DetailsHandler handler = new DetailsHandler( responseBuilder, submitListener, diff --git a/x-pack/plugin/profiling/src/test/java/org/elasticsearch/xpack/profiling/IndexAllocationTests.java b/x-pack/plugin/profiling/src/test/java/org/elasticsearch/xpack/profiling/IndexAllocationTests.java new file mode 100644 index 0000000000000..852790e219a2d --- /dev/null +++ b/x-pack/plugin/profiling/src/test/java/org/elasticsearch/xpack/profiling/IndexAllocationTests.java @@ -0,0 +1,122 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.profiling; + +import org.elasticsearch.cluster.ClusterName; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.block.ClusterBlocks; +import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.cluster.metadata.Metadata; +import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.node.DiscoveryNodeRole; +import org.elasticsearch.cluster.node.DiscoveryNodeUtils; +import org.elasticsearch.cluster.node.DiscoveryNodes; +import org.elasticsearch.cluster.routing.IndexRoutingTable; +import org.elasticsearch.cluster.routing.IndexShardRoutingTable; +import org.elasticsearch.cluster.routing.RecoverySource; +import org.elasticsearch.cluster.routing.RoutingTable; +import org.elasticsearch.cluster.routing.ShardRouting; +import org.elasticsearch.cluster.routing.UnassignedInfo; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.Index; +import org.elasticsearch.index.IndexVersion; +import org.elasticsearch.index.shard.ShardId; +import org.elasticsearch.test.ESTestCase; + +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.UUID; + +public class IndexAllocationTests extends ESTestCase { + private final Index hot = idx("hot"); + private final Index warm = idx("warm"); + private final Index cold = idx("cold"); + private final Index frozen = idx("frozen"); + + public void testEmptyIndicesNotOnWarmColdTier() { + assertFalse(IndexAllocation.isAnyOnWarmOrColdTier(clusterState(), Collections.emptyList())); + } + + public void testOtherIndicesNotOnWarmColdTier() { + assertFalse(IndexAllocation.isAnyOnWarmOrColdTier(clusterState(), List.of(hot, frozen))); + } + + public void testIndicesOnWarmColdTier() { + assertTrue(IndexAllocation.isAnyOnWarmOrColdTier(clusterState(), List.of(warm))); + assertTrue(IndexAllocation.isAnyOnWarmOrColdTier(clusterState(), List.of(cold))); + } + + public void testMixedIndicesOnWarmColdTier() { + assertTrue(IndexAllocation.isAnyOnWarmOrColdTier(clusterState(), List.of(hot, warm))); + assertTrue(IndexAllocation.isAnyOnWarmOrColdTier(clusterState(), List.of(frozen, cold))); + } + + /** + * Creates a cluster state that represents several indices: + * + *
    + *
  • hot assigned to a hot-tier node named n-hot
    • + *
  • warm assigned to a warm-tier node named n-warm
    • + *
  • cold assigned to a cold-tier node named n-cold
    • + *
  • frozen assigned to a frozen-tier node named n-frozen
    • + *
+ */ + private ClusterState clusterState() { + DiscoveryNode node = DiscoveryNodeUtils.create("node"); + DiscoveryNodes.Builder nodesBuilder = DiscoveryNodes.builder().localNodeId("node").masterNodeId("node").add(node); + + nodesBuilder.add(DiscoveryNodeUtils.builder("n-" + hot.getName()).roles(Set.of(DiscoveryNodeRole.DATA_HOT_NODE_ROLE)).build()); + nodesBuilder.add(DiscoveryNodeUtils.builder("n-" + warm.getName()).roles(Set.of(DiscoveryNodeRole.DATA_WARM_NODE_ROLE)).build()); + nodesBuilder.add(DiscoveryNodeUtils.builder("n-" + cold.getName()).roles(Set.of(DiscoveryNodeRole.DATA_COLD_NODE_ROLE)).build()); + nodesBuilder.add( + DiscoveryNodeUtils.builder("n-" + frozen.getName()).roles(Set.of(DiscoveryNodeRole.DATA_FROZEN_NODE_ROLE)).build() + ); + + RoutingTable.Builder routingTableBuilder = RoutingTable.builder(); + Map indices = new HashMap<>(); + for (Index index : List.of(hot, warm, cold, frozen)) { + indices.put(index.getName(), metadata(index)); + ShardRouting shardRouting = ShardRouting.newUnassigned( + new ShardId(index, 0), + true, + RecoverySource.ExistingStoreRecoverySource.INSTANCE, + new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, ""), + ShardRouting.Role.DEFAULT + ); + + shardRouting = shardRouting.initialize("n-" + index.getName(), null, 0).moveToStarted(0); + routingTableBuilder.add( + IndexRoutingTable.builder(index) + .addIndexShard(IndexShardRoutingTable.builder(shardRouting.shardId()).addShard(shardRouting)) + ); + } + + return ClusterState.builder(new ClusterName("test")) + .metadata(Metadata.builder().indices(indices).build()) + .blocks(new ClusterBlocks.Builder().build()) + .nodes(nodesBuilder) + .routingTable(routingTableBuilder) + .build(); + } + + private IndexMetadata metadata(Index index) { + final Settings settings = Settings.builder() + .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current()) + .put(IndexMetadata.SETTING_INDEX_UUID, index.getUUID()) + .build(); + return IndexMetadata.builder(index.getName()).settings(settings).numberOfShards(1).numberOfReplicas(0).build(); + } + + private Index idx(String name) { + return new Index(name, UUID.randomUUID().toString()); + } + +} diff --git a/x-pack/plugin/profiling/src/test/java/org/elasticsearch/xpack/profiling/TransportGetStackTracesActionTests.java b/x-pack/plugin/profiling/src/test/java/org/elasticsearch/xpack/profiling/TransportGetStackTracesActionTests.java index bf4c15d8d1ed5..2eccfb45f5958 100644 --- a/x-pack/plugin/profiling/src/test/java/org/elasticsearch/xpack/profiling/TransportGetStackTracesActionTests.java +++ b/x-pack/plugin/profiling/src/test/java/org/elasticsearch/xpack/profiling/TransportGetStackTracesActionTests.java @@ -17,6 +17,13 @@ public void testSliceEmptyList() { assertEquals(List.of(List.of()), TransportGetStackTracesAction.sliced(Collections.emptyList(), 4)); } + public void testSingleSlice() { + List input = randomList(2, 5, () -> randomAlphaOfLength(3)); + List> sliced = TransportGetStackTracesAction.sliced(input, 1); + assertEquals(1, sliced.size()); + assertEquals(input, sliced.get(0)); + } + public void testSliceListSmallerOrEqualToSliceCount() { int slices = 7; List input = randomList(0, slices, () -> randomAlphaOfLength(3)); From 47625de9e60b21246390dcb2fdb42e95b9fe514c Mon Sep 17 00:00:00 2001 From: Lee Hinman Date: Thu, 7 Dec 2023 08:56:24 -0700 Subject: [PATCH 2/7] Use latest version of entsearch ingestion pipeline (#103087) * Use latest version of entsearch ingestion pipeline This pipeline accidentally had a hardcoded version of "1", instead of using the registry version (like the other assets for this plugin). This led to lots of lines such as: ``` [2023-12-06T14:16:00,593][INFO ][o.e.x.c.t.IndexTemplateRegistry] [runTask-0] adding ingest pipeline ent-search-generic-ingestion [2023-12-06T14:16:00,633][INFO ][o.e.x.c.t.IndexTemplateRegistry] [runTask-0] upgrading ingest pipeline [ent-search-generic-ingestion] for [enterprise_search] from version [1] to version [3] [2023-12-06T14:16:00,634][INFO ][o.e.x.c.t.IndexTemplateRegistry] [runTask-0] adding ingest pipeline ent-search-generic-ingestion [2023-12-06T14:16:00,680][INFO ][o.e.x.c.t.IndexTemplateRegistry] [runTask-0] upgrading ingest pipeline [ent-search-generic-ingestion] for [enterprise_search] from version [1] to version [3] [2023-12-06T14:16:00,681][INFO ][o.e.x.c.t.IndexTemplateRegistry] [runTask-0] adding ingest pipeline ent-search-generic-ingestion [2023-12-06T14:16:00,706][INFO ][o.e.x.c.t.IndexTemplateRegistry] [runTask-0] upgrading ingest pipeline [ent-search-generic-ingestion] for [enterprise_search] from version [1] to version [3] [2023-12-06T14:16:00,707][INFO ][o.e.x.c.t.IndexTemplateRegistry] [runTask-0] adding ingest pipeline ent-search-generic-ingestion [2023-12-06T14:16:00,731][INFO ][o.e.x.c.t.IndexTemplateRegistry] [runTask-0] upgrading ingest pipeline [ent-search-generic-ingestion] for [enterprise_search] from version [1] to version [3] [2023-12-06T14:16:00,732][INFO ][o.e.x.c.t.IndexTemplateRegistry] [runTask-0] adding ingest pipeline ent-search-generic-ingestion etc etc etc ``` Because the pipeline was installed at version 1, and then immediately "upgraded" to version 3, despite no changes. Relates to #97463 --- docs/changelog/103087.yaml | 5 +++++ .../main/resources/entsearch/generic_ingestion_pipeline.json | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) create mode 100644 docs/changelog/103087.yaml diff --git a/docs/changelog/103087.yaml b/docs/changelog/103087.yaml new file mode 100644 index 0000000000000..5824bc53edb8d --- /dev/null +++ b/docs/changelog/103087.yaml @@ -0,0 +1,5 @@ +pr: 103087 +summary: Use latest version of entsearch ingestion pipeline +area: Application +type: bug +issues: [] diff --git a/x-pack/plugin/core/template-resources/src/main/resources/entsearch/generic_ingestion_pipeline.json b/x-pack/plugin/core/template-resources/src/main/resources/entsearch/generic_ingestion_pipeline.json index f66789d25e5f5..e2a2cbd460117 100644 --- a/x-pack/plugin/core/template-resources/src/main/resources/entsearch/generic_ingestion_pipeline.json +++ b/x-pack/plugin/core/template-resources/src/main/resources/entsearch/generic_ingestion_pipeline.json @@ -1,5 +1,5 @@ { - "version": 1, + "version": ${xpack.application.connector.template.version}, "description": "Generic Enterprise Search ingest pipeline", "_meta": { "managed_by": "Enterprise Search", From b510e59c67ba8ddb07100419fccb86d6da2dc141 Mon Sep 17 00:00:00 2001 From: Chris Hegarty <62058229+ChrisHegarty@users.noreply.github.com> Date: Thu, 7 Dec 2023 16:02:53 +0000 Subject: [PATCH 3/7] Add JIT compiler excludes for computeCommonPrefixLengthAndBuildHistogram (#103112) This commit adds a temporary JIT compile command that excludes the compilation of a couple of Lucene methods which, if compiled, crash the JVM. --- .../gradle/internal/ElasticsearchTestBasePlugin.java | 5 ++++- distribution/src/config/jvm.options | 4 ++++ docs/changelog/103112.yaml | 5 +++++ 3 files changed, 13 insertions(+), 1 deletion(-) create mode 100644 docs/changelog/103112.yaml diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchTestBasePlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchTestBasePlugin.java index 31b62c4ac700f..f1804064b7e07 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchTestBasePlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchTestBasePlugin.java @@ -108,7 +108,10 @@ public void execute(Task t) { "--add-opens=java.base/java.nio.file=ALL-UNNAMED", "--add-opens=java.base/java.time=ALL-UNNAMED", "--add-opens=java.management/java.lang.management=ALL-UNNAMED", - "-XX:+HeapDumpOnOutOfMemoryError" + "-XX:+HeapDumpOnOutOfMemoryError", + // REMOVE once bumped to a JDK greater than 21.0.1, https://github.com/elastic/elasticsearch/issues/103004 + "-XX:CompileCommand=exclude,org.apache.lucene.util.MSBRadixSorter::computeCommonPrefixLengthAndBuildHistogram", + "-XX:CompileCommand=exclude,org.apache.lucene.util.RadixSelector::computeCommonPrefixLengthAndBuildHistogram" ); test.getJvmArgumentProviders().add(new SimpleCommandLineArgumentProvider("-XX:HeapDumpPath=" + heapdumpDir)); diff --git a/distribution/src/config/jvm.options b/distribution/src/config/jvm.options index c5e905f461f45..9e26582d58439 100644 --- a/distribution/src/config/jvm.options +++ b/distribution/src/config/jvm.options @@ -58,6 +58,10 @@ # result in less optimal vector performance 20-:--add-modules=jdk.incubator.vector +# REMOVE once bumped to a JDK greater than 21.0.1, https://github.com/elastic/elasticsearch/issues/103004 +19-21:-XX:CompileCommand=exclude,org.apache.lucene.util.MSBRadixSorter::computeCommonPrefixLengthAndBuildHistogram +19-21:-XX:CompileCommand=exclude,org.apache.lucene.util.RadixSelector::computeCommonPrefixLengthAndBuildHistogram + ## heap dumps # generate a heap dump when an allocation from the Java heap fails; heap dumps diff --git a/docs/changelog/103112.yaml b/docs/changelog/103112.yaml new file mode 100644 index 0000000000000..dcb4cf604c179 --- /dev/null +++ b/docs/changelog/103112.yaml @@ -0,0 +1,5 @@ +pr: 103112 +summary: Add JIT compiler excludes for `computeCommonPrefixLengthAndBuildHistogram` +area: Search +type: bug +issues: [] From 5b0b74108a06666471b873cebb40e044818e01f2 Mon Sep 17 00:00:00 2001 From: David Turner Date: Thu, 7 Dec 2023 16:09:50 +0000 Subject: [PATCH 4/7] AwaitsFix for #102987 --- .../esql/expression/function/scalar/convert/ToDegreesTests.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDegreesTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDegreesTests.java index a1c3c1f38aac5..b3e0c65f0c8f8 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDegreesTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDegreesTests.java @@ -10,6 +10,7 @@ import com.carrotsearch.randomizedtesting.annotations.Name; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; +import org.apache.lucene.tests.util.LuceneTestCase; import org.elasticsearch.xpack.esql.expression.function.AbstractFunctionTestCase; import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; import org.elasticsearch.xpack.ql.expression.Expression; @@ -22,6 +23,7 @@ import java.util.function.Function; import java.util.function.Supplier; +@LuceneTestCase.AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/102987") public class ToDegreesTests extends AbstractFunctionTestCase { public ToDegreesTests(@Name("TestCase") Supplier testCaseSupplier) { this.testCase = testCaseSupplier.get(); From 3807cb81340d803884db03ce7a737bd343649353 Mon Sep 17 00:00:00 2001 From: Brian Seeders Date: Thu, 7 Dec 2023 11:10:13 -0500 Subject: [PATCH 5/7] Bump versions after 8.11.2 release --- .buildkite/pipelines/intake.yml | 2 +- .buildkite/pipelines/periodic-packaging.yml | 16 ++++++++++++++++ .buildkite/pipelines/periodic.yml | 10 ++++++++++ .ci/bwcVersions | 1 + .ci/snapshotBwcVersions | 2 +- .../src/main/java/org/elasticsearch/Version.java | 1 + 6 files changed, 30 insertions(+), 2 deletions(-) diff --git a/.buildkite/pipelines/intake.yml b/.buildkite/pipelines/intake.yml index b6bbc62e6bc0e..a200e871ec8e6 100644 --- a/.buildkite/pipelines/intake.yml +++ b/.buildkite/pipelines/intake.yml @@ -48,7 +48,7 @@ steps: timeout_in_minutes: 300 matrix: setup: - BWC_VERSION: ["7.17.16", "8.11.2", "8.12.0", "8.13.0"] + BWC_VERSION: ["7.17.16", "8.11.3", "8.12.0", "8.13.0"] agents: provider: gcp image: family/elasticsearch-ubuntu-2004 diff --git a/.buildkite/pipelines/periodic-packaging.yml b/.buildkite/pipelines/periodic-packaging.yml index c0e51b609faee..d397039128457 100644 --- a/.buildkite/pipelines/periodic-packaging.yml +++ b/.buildkite/pipelines/periodic-packaging.yml @@ -1745,6 +1745,22 @@ steps: env: BWC_VERSION: 8.11.2 + - label: "{{matrix.image}} / 8.11.3 / packaging-tests-upgrade" + command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.11.3 + timeout_in_minutes: 300 + matrix: + setup: + image: + - rocky-8 + - ubuntu-2004 + agents: + provider: gcp + image: family/elasticsearch-{{matrix.image}} + machineType: custom-16-32768 + buildDirectory: /dev/shm/bk + env: + BWC_VERSION: 8.11.3 + - label: "{{matrix.image}} / 8.12.0 / packaging-tests-upgrade" command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.12.0 timeout_in_minutes: 300 diff --git a/.buildkite/pipelines/periodic.yml b/.buildkite/pipelines/periodic.yml index 9fb66a8062ab2..248bfd52742d7 100644 --- a/.buildkite/pipelines/periodic.yml +++ b/.buildkite/pipelines/periodic.yml @@ -1072,6 +1072,16 @@ steps: buildDirectory: /dev/shm/bk env: BWC_VERSION: 8.11.2 + - label: 8.11.3 / bwc + command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.11.3#bwcTest + timeout_in_minutes: 300 + agents: + provider: gcp + image: family/elasticsearch-ubuntu-2004 + machineType: n1-standard-32 + buildDirectory: /dev/shm/bk + env: + BWC_VERSION: 8.11.3 - label: 8.12.0 / bwc command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.12.0#bwcTest timeout_in_minutes: 300 diff --git a/.ci/bwcVersions b/.ci/bwcVersions index 0c29d210149bc..de951a9e6fc24 100644 --- a/.ci/bwcVersions +++ b/.ci/bwcVersions @@ -106,5 +106,6 @@ BWC_VERSION: - "8.11.0" - "8.11.1" - "8.11.2" + - "8.11.3" - "8.12.0" - "8.13.0" diff --git a/.ci/snapshotBwcVersions b/.ci/snapshotBwcVersions index 6fbe04325c898..9df1c097ac941 100644 --- a/.ci/snapshotBwcVersions +++ b/.ci/snapshotBwcVersions @@ -1,5 +1,5 @@ BWC_VERSION: - "7.17.16" - - "8.11.2" + - "8.11.3" - "8.12.0" - "8.13.0" diff --git a/server/src/main/java/org/elasticsearch/Version.java b/server/src/main/java/org/elasticsearch/Version.java index 2ecc9703b2398..3d6995bd9e90f 100644 --- a/server/src/main/java/org/elasticsearch/Version.java +++ b/server/src/main/java/org/elasticsearch/Version.java @@ -157,6 +157,7 @@ public class Version implements VersionId, ToXContentFragment { public static final Version V_8_11_0 = new Version(8_11_00_99); public static final Version V_8_11_1 = new Version(8_11_01_99); public static final Version V_8_11_2 = new Version(8_11_02_99); + public static final Version V_8_11_3 = new Version(8_11_03_99); public static final Version V_8_12_0 = new Version(8_12_00_99); public static final Version V_8_13_0 = new Version(8_13_00_99); public static final Version CURRENT = V_8_13_0; From c7406ed09314f7fc90f22d40c175381c6ff42064 Mon Sep 17 00:00:00 2001 From: Brian Seeders Date: Thu, 7 Dec 2023 11:13:07 -0500 Subject: [PATCH 6/7] Prune changelogs after 8.11.2 release --- docs/changelog/100986.yaml | 6 ------ docs/changelog/101915.yaml | 5 ----- docs/changelog/102057.yaml | 6 ------ docs/changelog/102114.yaml | 6 ------ docs/changelog/102151.yaml | 5 ----- docs/changelog/102220.yaml | 5 ----- docs/changelog/102230.yaml | 6 ------ docs/changelog/102240.yaml | 5 ----- docs/changelog/102250.yaml | 6 ------ docs/changelog/102259.yaml | 5 ----- docs/changelog/102281.yaml | 5 ----- docs/changelog/102282.yaml | 6 ------ docs/changelog/102311.yaml | 5 ----- docs/changelog/102396.yaml | 5 ----- docs/changelog/102399.yaml | 6 ------ docs/changelog/102467.yaml | 6 ------ docs/changelog/102492.yaml | 5 ----- docs/changelog/102580.yaml | 6 ------ docs/changelog/102599.yaml | 5 ----- docs/changelog/102715.yaml | 6 ------ docs/changelog/102716.yaml | 5 ----- docs/changelog/102779.yaml | 5 ----- docs/changelog/102821.yaml | 5 ----- docs/changelog/102831.yaml | 9 --------- docs/changelog/102891.yaml | 7 ------- docs/changelog/102934.yaml | 6 ------ 26 files changed, 147 deletions(-) delete mode 100644 docs/changelog/100986.yaml delete mode 100644 docs/changelog/101915.yaml delete mode 100644 docs/changelog/102057.yaml delete mode 100644 docs/changelog/102114.yaml delete mode 100644 docs/changelog/102151.yaml delete mode 100644 docs/changelog/102220.yaml delete mode 100644 docs/changelog/102230.yaml delete mode 100644 docs/changelog/102240.yaml delete mode 100644 docs/changelog/102250.yaml delete mode 100644 docs/changelog/102259.yaml delete mode 100644 docs/changelog/102281.yaml delete mode 100644 docs/changelog/102282.yaml delete mode 100644 docs/changelog/102311.yaml delete mode 100644 docs/changelog/102396.yaml delete mode 100644 docs/changelog/102399.yaml delete mode 100644 docs/changelog/102467.yaml delete mode 100644 docs/changelog/102492.yaml delete mode 100644 docs/changelog/102580.yaml delete mode 100644 docs/changelog/102599.yaml delete mode 100644 docs/changelog/102715.yaml delete mode 100644 docs/changelog/102716.yaml delete mode 100644 docs/changelog/102779.yaml delete mode 100644 docs/changelog/102821.yaml delete mode 100644 docs/changelog/102831.yaml delete mode 100644 docs/changelog/102891.yaml delete mode 100644 docs/changelog/102934.yaml diff --git a/docs/changelog/100986.yaml b/docs/changelog/100986.yaml deleted file mode 100644 index 3920e2ef78d6a..0000000000000 --- a/docs/changelog/100986.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 100986 -summary: Synchronize Coordinator#onClusterStateApplied -area: Cluster Coordination -type: bug -issues: - - 99023 diff --git a/docs/changelog/101915.yaml b/docs/changelog/101915.yaml deleted file mode 100644 index aed7ca62021a5..0000000000000 --- a/docs/changelog/101915.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 101915 -summary: Add inference counts by model to the machine learning usage stats -area: Machine Learning -type: enhancement -issues: [] diff --git a/docs/changelog/102057.yaml b/docs/changelog/102057.yaml deleted file mode 100644 index d5b664ba14c29..0000000000000 --- a/docs/changelog/102057.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 102057 -summary: Simplify `BlobStoreRepository` idle check -area: Snapshot/Restore -type: bug -issues: - - 101948 diff --git a/docs/changelog/102114.yaml b/docs/changelog/102114.yaml deleted file mode 100644 index a08389da0351b..0000000000000 --- a/docs/changelog/102114.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 102114 -summary: Fix double-completion in `SecurityUsageTransportAction` -area: Security -type: bug -issues: - - 102111 diff --git a/docs/changelog/102151.yaml b/docs/changelog/102151.yaml deleted file mode 100644 index 652ae555af97d..0000000000000 --- a/docs/changelog/102151.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 102151 -summary: Default `run_ml_inference` should be true -area: Application -type: bug -issues: [] diff --git a/docs/changelog/102220.yaml b/docs/changelog/102220.yaml deleted file mode 100644 index d24dab1f91b31..0000000000000 --- a/docs/changelog/102220.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 102220 -summary: Upgrade xmlsec to 2.3.4 -area: Security -type: enhancement -issues: [] diff --git a/docs/changelog/102230.yaml b/docs/changelog/102230.yaml deleted file mode 100644 index 20e8d8d1f10a6..0000000000000 --- a/docs/changelog/102230.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 102230 -summary: Set region for the STS client via privileged calls in AWS SDK -area: Snapshot/Restore -type: bug -issues: - - 102173 diff --git a/docs/changelog/102240.yaml b/docs/changelog/102240.yaml deleted file mode 100644 index 5df0046ee92fc..0000000000000 --- a/docs/changelog/102240.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 102240 -summary: Exclude stack traces from transform audit messages and health -area: Transform -type: bug -issues: [] diff --git a/docs/changelog/102250.yaml b/docs/changelog/102250.yaml deleted file mode 100644 index 755341d9a3a64..0000000000000 --- a/docs/changelog/102250.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 102250 -summary: "[ILM] Fix downsample to skip already downsampled indices" -area: ILM+SLM -type: bug -issues: - - 102249 diff --git a/docs/changelog/102259.yaml b/docs/changelog/102259.yaml deleted file mode 100644 index 3d8a1c6381f6d..0000000000000 --- a/docs/changelog/102259.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 102259 -summary: "[Usage API] Count all the data streams that have lifecycle" -area: Data streams -type: bug -issues: [] diff --git a/docs/changelog/102281.yaml b/docs/changelog/102281.yaml deleted file mode 100644 index ac6c17591e013..0000000000000 --- a/docs/changelog/102281.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 102281 -summary: Improve failure handling in `ContinuousComputation` -area: Allocation -type: bug -issues: [] diff --git a/docs/changelog/102282.yaml b/docs/changelog/102282.yaml deleted file mode 100644 index 4860d70f99ccc..0000000000000 --- a/docs/changelog/102282.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 102282 -summary: "ES|QL: Fix drop of renamed grouping" -area: ES|QL -type: bug -issues: - - 102121 diff --git a/docs/changelog/102311.yaml b/docs/changelog/102311.yaml deleted file mode 100644 index bb1769527fdd4..0000000000000 --- a/docs/changelog/102311.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 102311 -summary: Upgrade reactor netty http version -area: Snapshot/Restore -type: upgrade -issues: [] diff --git a/docs/changelog/102396.yaml b/docs/changelog/102396.yaml deleted file mode 100644 index 9ea53ca5b6840..0000000000000 --- a/docs/changelog/102396.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 102396 -summary: Add more logging to the real memory circuit breaker and lower minimum interval -area: "Infra/Circuit Breakers" -type: bug -issues: [] diff --git a/docs/changelog/102399.yaml b/docs/changelog/102399.yaml deleted file mode 100644 index 7a4e1ff7ddab6..0000000000000 --- a/docs/changelog/102399.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 102399 -summary: "ES|QL: Fix layout management for Project" -area: ES|QL -type: bug -issues: - - 102120 diff --git a/docs/changelog/102467.yaml b/docs/changelog/102467.yaml deleted file mode 100644 index 580788a5aa2f9..0000000000000 --- a/docs/changelog/102467.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 102467 -summary: Fix dense_vector cluster stats indexed_vector_dim_min/max values -area: Mapping -type: bug -issues: - - 102416 diff --git a/docs/changelog/102492.yaml b/docs/changelog/102492.yaml deleted file mode 100644 index 943d82873e0b6..0000000000000 --- a/docs/changelog/102492.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 102492 -summary: Ensure datafeed previews with no start or end time don't search the cold or frozen tiers -area: Machine Learning -type: bug -issues: [] diff --git a/docs/changelog/102580.yaml b/docs/changelog/102580.yaml deleted file mode 100644 index 50d315efd7071..0000000000000 --- a/docs/changelog/102580.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 102580 -summary: Fix DISSECT with empty patterns -area: ES|QL -type: bug -issues: - - 102577 diff --git a/docs/changelog/102599.yaml b/docs/changelog/102599.yaml deleted file mode 100644 index 74e3d89421463..0000000000000 --- a/docs/changelog/102599.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 102599 -summary: "Recreate the Elasticsearch private temporary directory if it doesn't exist when an ML job is opened" -area: Machine Learning -type: bug -issues: [] diff --git a/docs/changelog/102715.yaml b/docs/changelog/102715.yaml deleted file mode 100644 index 7311db66ce151..0000000000000 --- a/docs/changelog/102715.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 102715 -summary: Fix leaking blocks in TopN -area: ES|QL -type: bug -issues: - - 102646 diff --git a/docs/changelog/102716.yaml b/docs/changelog/102716.yaml deleted file mode 100644 index 39317fdb38415..0000000000000 --- a/docs/changelog/102716.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 102716 -summary: Fix leaking blocks in `BlockUtils` -area: ES|QL -type: bug -issues: [] diff --git a/docs/changelog/102779.yaml b/docs/changelog/102779.yaml deleted file mode 100644 index 7bbecb29665bd..0000000000000 --- a/docs/changelog/102779.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 102779 -summary: Allow mismatched sort-by field types if there are no docs to sort -area: Search -type: bug -issues: [] diff --git a/docs/changelog/102821.yaml b/docs/changelog/102821.yaml deleted file mode 100644 index dcd6721621878..0000000000000 --- a/docs/changelog/102821.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 102821 -summary: Better processor stat merge -area: Ingest Node -type: bug -issues: [] diff --git a/docs/changelog/102831.yaml b/docs/changelog/102831.yaml deleted file mode 100644 index fb99b0c7f732b..0000000000000 --- a/docs/changelog/102831.yaml +++ /dev/null @@ -1,9 +0,0 @@ -pr: 102831 -summary: Fix memory tracking in TopN.Row -area: ES|QL -type: bug -issues: - - 100640 - - 102784 - - 102790 - - 102683 diff --git a/docs/changelog/102891.yaml b/docs/changelog/102891.yaml deleted file mode 100644 index c5d5ed8c6758e..0000000000000 --- a/docs/changelog/102891.yaml +++ /dev/null @@ -1,7 +0,0 @@ -pr: 102891 -summary: "[Query Rules] Fix bug where combining the same metadata with text/numeric\ - \ values leads to error" -area: Application -type: bug -issues: - - 102827 diff --git a/docs/changelog/102934.yaml b/docs/changelog/102934.yaml deleted file mode 100644 index 4f61427506cf3..0000000000000 --- a/docs/changelog/102934.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 102934 -summary: Ensure transform updates only modify the expected transform task -area: Transform -type: bug -issues: - - 102933 From c6eff199ada799ed716fa0d5a97bb71647382f12 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim=20R=C3=BChsen?= Date: Thu, 7 Dec 2023 17:16:20 +0100 Subject: [PATCH 7/7] [Profiling] Fix stacktraces `total_frames` response value and improve tests (#103062) * Fix flamegraph total_frame value and improve tests * Make inline aggregation more expressive --- .../profiling/GetFlameGraphActionIT.java | 10 +++++++--- .../profiling/GetStackTracesActionIT.java | 14 +++++++------- .../data/profiling-events-all.ndjson | 2 ++ .../profiling/GetFlamegraphResponse.java | 8 ++++++++ .../GetStackTracesResponseBuilder.java | 4 ++++ .../xpack/profiling/StackFrame.java | 19 +++++++++++++++---- .../TransportGetStackTracesAction.java | 6 +++++- 7 files changed, 48 insertions(+), 15 deletions(-) diff --git a/x-pack/plugin/profiling/src/internalClusterTest/java/org/elasticsearch/xpack/profiling/GetFlameGraphActionIT.java b/x-pack/plugin/profiling/src/internalClusterTest/java/org/elasticsearch/xpack/profiling/GetFlameGraphActionIT.java index 4ce441285d432..8553574d39646 100644 --- a/x-pack/plugin/profiling/src/internalClusterTest/java/org/elasticsearch/xpack/profiling/GetFlameGraphActionIT.java +++ b/x-pack/plugin/profiling/src/internalClusterTest/java/org/elasticsearch/xpack/profiling/GetFlameGraphActionIT.java @@ -14,8 +14,12 @@ public void testGetStackTracesUnfiltered() throws Exception { // only spot-check top level properties - detailed tests are done in unit tests assertEquals(994, response.getSize()); assertEquals(1.0d, response.getSamplingRate(), 0.001d); - assertEquals(44, response.getSelfCPU()); - assertEquals(1865, response.getTotalCPU()); - assertEquals(44, response.getTotalSamples()); + assertEquals(46, response.getSelfCPU()); + assertEquals(1903, response.getTotalCPU()); + assertEquals(46, response.getTotalSamples()); + + // The root node's values are the same as the top-level values. + assertEquals("", response.getFileIds().get(0)); + assertEquals(response.getSelfCPU(), response.getCountInclusive().get(0).longValue()); } } diff --git a/x-pack/plugin/profiling/src/internalClusterTest/java/org/elasticsearch/xpack/profiling/GetStackTracesActionIT.java b/x-pack/plugin/profiling/src/internalClusterTest/java/org/elasticsearch/xpack/profiling/GetStackTracesActionIT.java index a5efa24da5397..289f6896ed698 100644 --- a/x-pack/plugin/profiling/src/internalClusterTest/java/org/elasticsearch/xpack/profiling/GetStackTracesActionIT.java +++ b/x-pack/plugin/profiling/src/internalClusterTest/java/org/elasticsearch/xpack/profiling/GetStackTracesActionIT.java @@ -14,14 +14,14 @@ public class GetStackTracesActionIT extends ProfilingTestCase { public void testGetStackTracesUnfiltered() throws Exception { - GetStackTracesRequest request = new GetStackTracesRequest(10, 1.0d, 1.0d, null, null, null, null, null, null, null, null); + GetStackTracesRequest request = new GetStackTracesRequest(1000, 600.0d, 1.0d, null, null, null, null, null, null, null, null); request.setAdjustSampleCount(true); GetStackTracesResponse response = client().execute(GetStackTracesAction.INSTANCE, request).get(); - assertEquals(40, response.getTotalSamples()); - assertEquals(473, response.getTotalFrames()); + assertEquals(46, response.getTotalSamples()); + assertEquals(1821, response.getTotalFrames()); assertNotNull(response.getStackTraceEvents()); - assertEquals(4L, response.getStackTraceEvents().get("L7kj7UvlKbT-vN73el4faQ").count); + assertEquals(3L, response.getStackTraceEvents().get("L7kj7UvlKbT-vN73el4faQ").count); assertNotNull(response.getStackTraces()); // just do a high-level spot check. Decoding is tested in unit-tests @@ -30,8 +30,8 @@ public void testGetStackTracesUnfiltered() throws Exception { assertEquals(18, stackTrace.fileIds.size()); assertEquals(18, stackTrace.frameIds.size()); assertEquals(18, stackTrace.typeIds.size()); - assertEquals(0.007903d, stackTrace.annualCO2Tons, 0.000001d); - assertEquals(74.46d, stackTrace.annualCostsUSD, 0.01d); + assertEquals(0.0000098789d, stackTrace.annualCO2Tons, 0.0000000001d); + assertEquals(0.093075d, stackTrace.annualCostsUSD, 0.000001d); assertNotNull(response.getStackFrames()); StackFrame stackFrame = response.getStackFrames().get("8NlMClggx8jaziUTJXlmWAAAAAAAAIYI"); @@ -58,7 +58,7 @@ public void testGetStackTracesFromAPMWithMatch() throws Exception { null ); GetStackTracesResponse response = client().execute(GetStackTracesAction.INSTANCE, request).get(); - assertEquals(43, response.getTotalFrames()); + assertEquals(49, response.getTotalFrames()); assertNotNull(response.getStackTraceEvents()); assertEquals(3L, response.getStackTraceEvents().get("Ce77w10WeIDow3kd1jowlA").count); diff --git a/x-pack/plugin/profiling/src/internalClusterTest/resources/data/profiling-events-all.ndjson b/x-pack/plugin/profiling/src/internalClusterTest/resources/data/profiling-events-all.ndjson index 6964368e534c7..502494f05c62c 100644 --- a/x-pack/plugin/profiling/src/internalClusterTest/resources/data/profiling-events-all.ndjson +++ b/x-pack/plugin/profiling/src/internalClusterTest/resources/data/profiling-events-all.ndjson @@ -71,6 +71,8 @@ {"create": {"_index": "profiling-events-all"}} {"Stacktrace.count": [1], "profiling.project.id": ["100"], "os.kernel": ["9.9.9-0"], "tags": ["environment:qa", "region:eu-west-1"], "host.ip": ["192.168.1.2"], "@timestamp": ["1698624000"], "container.name": ["instance-0000000010"], "ecs.version": ["1.12.0"], "Stacktrace.id": ["XF9MchOwpePfa6_hYy-vZQ"], "agent.version": ["head-be593ef3-1688111067"], "host.name": ["ip-192-168-1-2"], "host.id": ["8457605156473051743"], "process.thread.name": ["497295213074376"]} {"create": {"_index": "profiling-events-all"}} +{"Stacktrace.count": [2], "profiling.project.id": ["100"], "os.kernel": ["9.9.9-0"], "tags": ["environment:qa", "region:eu-west-1"], "host.ip": ["192.168.1.2"], "@timestamp": ["1698624000"], "container.name": ["instance-0000000010"], "ecs.version": ["1.12.0"], "Stacktrace.id": ["L7kj7UvlKbT-vN73el4faQ"], "agent.version": ["head-be593ef3-1688111067"], "host.name": ["ip-192-168-1-2"], "host.id": ["8457605156473051743"], "process.thread.name": ["497295213074376"]} +{"create": {"_index": "profiling-events-all"}} {"Stacktrace.count": [1], "profiling.project.id": ["100"], "os.kernel": ["9.9.9-0"], "tags": ["environment:qa", "region:eu-west-1"], "host.ip": ["192.168.1.2"], "@timestamp": ["1698624000"], "container.name": ["instance-0000000010"], "ecs.version": ["1.12.0"], "Stacktrace.id": ["L7kj7UvlKbT-vN73el4faQ"], "agent.version": ["head-be593ef3-1688111067"], "host.name": ["ip-192-168-1-2"], "host.id": ["8457605156473051743"], "process.thread.name": ["497295213074376"]} {"create": {"_index": "profiling-events-all"}} {"Stacktrace.count": [1], "profiling.project.id": ["100"], "os.kernel": ["9.9.9-0"], "tags": ["environment:qa", "region:eu-west-1"], "host.ip": ["192.168.1.2"], "@timestamp": ["1698624000"], "container.name": ["instance-0000000010"], "ecs.version": ["1.12.0"], "Stacktrace.id": ["hRqQI2CBPiapzgFG9jrmDA"], "agent.version": ["head-be593ef3-1688111067"], "host.name": ["ip-192-168-1-2"], "host.id": ["8457605156473051743"], "process.thread.name": ["599103450330106"]} diff --git a/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/GetFlamegraphResponse.java b/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/GetFlamegraphResponse.java index 211b6d76f8caa..457faecf4ad54 100644 --- a/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/GetFlamegraphResponse.java +++ b/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/GetFlamegraphResponse.java @@ -194,6 +194,14 @@ public List getSourceLines() { return sourceLines; } + public List getAnnualCO2TonsInclusive() { + return annualCO2TonsInclusive; + } + + public List getAnnualCostsUSDInclusive() { + return annualCostsUSDInclusive; + } + public long getSelfCPU() { return selfCPU; } diff --git a/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/GetStackTracesResponseBuilder.java b/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/GetStackTracesResponseBuilder.java index ccafe99c31d9a..ab98b1d4daa2a 100644 --- a/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/GetStackTracesResponseBuilder.java +++ b/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/GetStackTracesResponseBuilder.java @@ -54,6 +54,10 @@ public void setTotalFrames(int totalFrames) { this.totalFrames = totalFrames; } + public void addTotalFrames(int numFrames) { + this.totalFrames += numFrames; + } + public void setStackFrames(Map stackFrames) { this.stackFrames = stackFrames; } diff --git a/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/StackFrame.java b/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/StackFrame.java index 455b150b6ee76..35f5899536745 100644 --- a/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/StackFrame.java +++ b/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/StackFrame.java @@ -30,9 +30,20 @@ final class StackFrame implements ToXContentObject { this.lineNumber = listOf(lineNumber); } + public int size() { + return this.functionName.size(); // functionName is the only array that is always set + } + + /** + * Returns the number of inlined frames in this stack frame. + * @return the number of inlined frames in this stack frame. + */ + public int inlineFrameCount() { + return size() > 0 ? size() - 1 : 0; + } + public void forEach(Consumer action) { - int size = this.functionName.size(); // functionName is the only array that is always set - for (int i = 0; i < size; i++) { + for (int i = 0; i < size(); i++) { action.accept( new Frame( fileName.size() > i ? fileName.get(i) : "", @@ -40,7 +51,7 @@ public void forEach(Consumer action) { functionOffset.size() > i ? functionOffset.get(i) : 0, lineNumber.size() > i ? lineNumber.get(i) : 0, i > 0, - i == size - 1 + i == size() - 1 ) ); } @@ -67,7 +78,7 @@ public static StackFrame fromSource(Map source) { } public boolean isEmpty() { - return fileName.isEmpty() && functionName.isEmpty() && functionOffset.isEmpty() && lineNumber.isEmpty(); + return size() == 0; } @Override diff --git a/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/TransportGetStackTracesAction.java b/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/TransportGetStackTracesAction.java index 000b448696985..3fa47beebd70a 100644 --- a/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/TransportGetStackTracesAction.java +++ b/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/TransportGetStackTracesAction.java @@ -688,6 +688,7 @@ private static class DetailsHandler { private final Map executables; private final Map stackFrames; private final AtomicInteger expectedSlices; + private final AtomicInteger totalInlineFrames = new AtomicInteger(); private final StopWatch watch = new StopWatch("retrieveStackTraceDetails"); private DetailsHandler( @@ -718,7 +719,9 @@ public void onStackFramesResponse(MultiGetResponse multiGetItemResponses) { if (stackFrames.containsKey(frame.getId()) == false) { StackFrame stackFrame = StackFrame.fromSource(frame.getResponse().getSource()); if (stackFrame.isEmpty() == false) { - stackFrames.putIfAbsent(frame.getId(), stackFrame); + if (stackFrames.putIfAbsent(frame.getId(), stackFrame) == null) { + totalInlineFrames.addAndGet(stackFrame.inlineFrameCount()); + } } else { log.trace("Stack frame with id [{}] has no properties.", frame.getId()); } @@ -757,6 +760,7 @@ public void mayFinish() { if (expectedSlices.decrementAndGet() == 0) { builder.setExecutables(executables); builder.setStackFrames(stackFrames); + builder.addTotalFrames(totalInlineFrames.get()); log.debug("retrieveStackTraceDetails found [{}] stack frames, [{}] executables.", stackFrames.size(), executables.size()); log.debug(watch::report); submitListener.onResponse(builder.build());