Merge branch 'main' into rcs2-reload

.buildkite/pipelines/intake.yml

@@ -48,7 +48,7 @@ steps:
         timeout_in_minutes: 300
         matrix:
           setup:
-            BWC_VERSION: ["7.17.16", "8.11.2", "8.12.0", "8.13.0"]
+            BWC_VERSION: ["7.17.16", "8.11.3", "8.12.0", "8.13.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004

.buildkite/pipelines/periodic-packaging.yml

@@ -1745,6 +1745,22 @@ steps:
         env:
           BWC_VERSION: 8.11.2
 
+      - label: "{{matrix.image}} / 8.11.3 / packaging-tests-upgrade"
+        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.11.3
+        timeout_in_minutes: 300
+        matrix:
+          setup:
+            image:
+              - rocky-8
+              - ubuntu-2004
+        agents:
+          provider: gcp
+          image: family/elasticsearch-{{matrix.image}}
+          machineType: custom-16-32768
+          buildDirectory: /dev/shm/bk
+        env:
+          BWC_VERSION: 8.11.3
+
       - label: "{{matrix.image}} / 8.12.0 / packaging-tests-upgrade"
         command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.12.0
         timeout_in_minutes: 300

.buildkite/pipelines/periodic.yml

@@ -1072,6 +1072,16 @@ steps:
           buildDirectory: /dev/shm/bk
         env:
           BWC_VERSION: 8.11.2
+      - label: 8.11.3 / bwc
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.11.3#bwcTest
+        timeout_in_minutes: 300
+        agents:
+          provider: gcp
+          image: family/elasticsearch-ubuntu-2004
+          machineType: n1-standard-32
+          buildDirectory: /dev/shm/bk
+        env:
+          BWC_VERSION: 8.11.3
       - label: 8.12.0 / bwc
         command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.12.0#bwcTest
         timeout_in_minutes: 300

.ci/bwcVersions

@@ -106,5 +106,6 @@ BWC_VERSION:
   - "8.11.0"
   - "8.11.1"
   - "8.11.2"
+  - "8.11.3"
   - "8.12.0"
   - "8.13.0"

.ci/snapshotBwcVersions

@@ -1,5 +1,5 @@
 BWC_VERSION:
   - "7.17.16"
-  - "8.11.2"
+  - "8.11.3"
   - "8.12.0"
   - "8.13.0"

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchTestBasePlugin.java

@@ -108,7 +108,10 @@ public void execute(Task t) {
                 "--add-opens=java.base/java.nio.file=ALL-UNNAMED",
                 "--add-opens=java.base/java.time=ALL-UNNAMED",
                 "--add-opens=java.management/java.lang.management=ALL-UNNAMED",
-                "-XX:+HeapDumpOnOutOfMemoryError"
+                "-XX:+HeapDumpOnOutOfMemoryError",
+                // REMOVE once bumped to a JDK greater than 21.0.1, https://github.com/elastic/elasticsearch/issues/103004
+                "-XX:CompileCommand=exclude,org.apache.lucene.util.MSBRadixSorter::computeCommonPrefixLengthAndBuildHistogram",
+                "-XX:CompileCommand=exclude,org.apache.lucene.util.RadixSelector::computeCommonPrefixLengthAndBuildHistogram"
             );
 
             test.getJvmArgumentProviders().add(new SimpleCommandLineArgumentProvider("-XX:HeapDumpPath=" + heapdumpDir));

distribution/src/config/jvm.options

@@ -58,6 +58,10 @@
 # result in less optimal vector performance
 20-:--add-modules=jdk.incubator.vector
 
+# REMOVE once bumped to a JDK greater than 21.0.1, https://github.com/elastic/elasticsearch/issues/103004
+19-21:-XX:CompileCommand=exclude,org.apache.lucene.util.MSBRadixSorter::computeCommonPrefixLengthAndBuildHistogram
+19-21:-XX:CompileCommand=exclude,org.apache.lucene.util.RadixSelector::computeCommonPrefixLengthAndBuildHistogram
+
 ## heap dumps
 
 # generate a heap dump when an allocation from the Java heap fails; heap dumps

docs/changelog/103061.yaml

@@ -0,0 +1,5 @@
+pr: 103061
+summary: "[Profiling] Query in parallel only if beneficial"
+area: Application
+type: bug
+issues: []

docs/changelog/103087.yaml

@@ -0,0 +1,5 @@
+pr: 103087
+summary: Use latest version of entsearch ingestion pipeline
+area: Application
+type: bug
+issues: []

docs/changelog/103112.yaml

@@ -0,0 +1,5 @@
+pr: 103112
+summary: Add JIT compiler excludes for `computeCommonPrefixLengthAndBuildHistogram`
+area: Search
+type: bug
+issues: []

server/src/main/java/org/elasticsearch/Version.java

@@ -157,6 +157,7 @@ public class Version implements VersionId<Version>, ToXContentFragment {
     public static final Version V_8_11_0 = new Version(8_11_00_99);
     public static final Version V_8_11_1 = new Version(8_11_01_99);
     public static final Version V_8_11_2 = new Version(8_11_02_99);
+    public static final Version V_8_11_3 = new Version(8_11_03_99);
     public static final Version V_8_12_0 = new Version(8_12_00_99);
     public static final Version V_8_13_0 = new Version(8_13_00_99);
     public static final Version CURRENT = V_8_13_0;

x-pack/plugin/core/template-resources/src/main/resources/entsearch/generic_ingestion_pipeline.json

@@ -1,5 +1,5 @@
 {
-  "version": 1,
+  "version": ${xpack.application.connector.template.version},
   "description": "Generic Enterprise Search ingest pipeline",
   "_meta": {
     "managed_by": "Enterprise Search",

x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDegreesTests.java

@@ -10,6 +10,7 @@
 import com.carrotsearch.randomizedtesting.annotations.Name;
 import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 
+import org.apache.lucene.tests.util.LuceneTestCase;
 import org.elasticsearch.xpack.esql.expression.function.AbstractFunctionTestCase;
 import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
 import org.elasticsearch.xpack.ql.expression.Expression;
@@ -22,6 +23,7 @@
 import java.util.function.Function;
 import java.util.function.Supplier;
 
+@LuceneTestCase.AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/102987")
 public class ToDegreesTests extends AbstractFunctionTestCase {
     public ToDegreesTests(@Name("TestCase") Supplier<TestCaseSupplier.TestCase> testCaseSupplier) {
         this.testCase = testCaseSupplier.get();

x-pack/plugin/profiling/src/internalClusterTest/java/org/elasticsearch/xpack/profiling/GetFlameGraphActionIT.java

@@ -14,8 +14,12 @@ public void testGetStackTracesUnfiltered() throws Exception {
         // only spot-check top level properties - detailed tests are done in unit tests
         assertEquals(994, response.getSize());
         assertEquals(1.0d, response.getSamplingRate(), 0.001d);
-        assertEquals(44, response.getSelfCPU());
-        assertEquals(1865, response.getTotalCPU());
-        assertEquals(44, response.getTotalSamples());
+        assertEquals(46, response.getSelfCPU());
+        assertEquals(1903, response.getTotalCPU());
+        assertEquals(46, response.getTotalSamples());
+
+        // The root node's values are the same as the top-level values.
+        assertEquals("", response.getFileIds().get(0));
+        assertEquals(response.getSelfCPU(), response.getCountInclusive().get(0).longValue());
     }
 }

x-pack/plugin/profiling/src/internalClusterTest/java/org/elasticsearch/xpack/profiling/GetStackTracesActionIT.java

@@ -14,14 +14,14 @@
 
 public class GetStackTracesActionIT extends ProfilingTestCase {
     public void testGetStackTracesUnfiltered() throws Exception {
-        GetStackTracesRequest request = new GetStackTracesRequest(10, 1.0d, 1.0d, null, null, null, null, null, null, null, null);
+        GetStackTracesRequest request = new GetStackTracesRequest(1000, 600.0d, 1.0d, null, null, null, null, null, null, null, null);
         request.setAdjustSampleCount(true);
         GetStackTracesResponse response = client().execute(GetStackTracesAction.INSTANCE, request).get();
-        assertEquals(40, response.getTotalSamples());
-        assertEquals(473, response.getTotalFrames());
+        assertEquals(46, response.getTotalSamples());
+        assertEquals(1821, response.getTotalFrames());
 
         assertNotNull(response.getStackTraceEvents());
-        assertEquals(4L, response.getStackTraceEvents().get("L7kj7UvlKbT-vN73el4faQ").count);
+        assertEquals(3L, response.getStackTraceEvents().get("L7kj7UvlKbT-vN73el4faQ").count);
 
         assertNotNull(response.getStackTraces());
         // just do a high-level spot check. Decoding is tested in unit-tests
@@ -30,8 +30,8 @@ public void testGetStackTracesUnfiltered() throws Exception {
         assertEquals(18, stackTrace.fileIds.size());
         assertEquals(18, stackTrace.frameIds.size());
         assertEquals(18, stackTrace.typeIds.size());
-        assertEquals(0.007903d, stackTrace.annualCO2Tons, 0.000001d);
-        assertEquals(74.46d, stackTrace.annualCostsUSD, 0.01d);
+        assertEquals(0.0000098789d, stackTrace.annualCO2Tons, 0.0000000001d);
+        assertEquals(0.093075d, stackTrace.annualCostsUSD, 0.000001d);
 
         assertNotNull(response.getStackFrames());
         StackFrame stackFrame = response.getStackFrames().get("8NlMClggx8jaziUTJXlmWAAAAAAAAIYI");
@@ -58,7 +58,7 @@ public void testGetStackTracesFromAPMWithMatch() throws Exception {
             null
         );
         GetStackTracesResponse response = client().execute(GetStackTracesAction.INSTANCE, request).get();
-        assertEquals(43, response.getTotalFrames());
+        assertEquals(49, response.getTotalFrames());
 
         assertNotNull(response.getStackTraceEvents());
         assertEquals(3L, response.getStackTraceEvents().get("Ce77w10WeIDow3kd1jowlA").count);

x-pack/plugin/profiling/src/internalClusterTest/resources/data/profiling-events-all.ndjson

@@ -71,6 +71,8 @@
 {"create": {"_index": "profiling-events-all"}}
 {"Stacktrace.count": [1], "profiling.project.id": ["100"], "os.kernel": ["9.9.9-0"], "tags": ["environment:qa", "region:eu-west-1"], "host.ip": ["192.168.1.2"], "@timestamp": ["1698624000"], "container.name": ["instance-0000000010"], "ecs.version": ["1.12.0"], "Stacktrace.id": ["XF9MchOwpePfa6_hYy-vZQ"], "agent.version": ["head-be593ef3-1688111067"], "host.name": ["ip-192-168-1-2"], "host.id": ["8457605156473051743"], "process.thread.name": ["497295213074376"]}
 {"create": {"_index": "profiling-events-all"}}
+{"Stacktrace.count": [2], "profiling.project.id": ["100"], "os.kernel": ["9.9.9-0"], "tags": ["environment:qa", "region:eu-west-1"], "host.ip": ["192.168.1.2"], "@timestamp": ["1698624000"], "container.name": ["instance-0000000010"], "ecs.version": ["1.12.0"], "Stacktrace.id": ["L7kj7UvlKbT-vN73el4faQ"], "agent.version": ["head-be593ef3-1688111067"], "host.name": ["ip-192-168-1-2"], "host.id": ["8457605156473051743"], "process.thread.name": ["497295213074376"]}
+{"create": {"_index": "profiling-events-all"}}
 {"Stacktrace.count": [1], "profiling.project.id": ["100"], "os.kernel": ["9.9.9-0"], "tags": ["environment:qa", "region:eu-west-1"], "host.ip": ["192.168.1.2"], "@timestamp": ["1698624000"], "container.name": ["instance-0000000010"], "ecs.version": ["1.12.0"], "Stacktrace.id": ["L7kj7UvlKbT-vN73el4faQ"], "agent.version": ["head-be593ef3-1688111067"], "host.name": ["ip-192-168-1-2"], "host.id": ["8457605156473051743"], "process.thread.name": ["497295213074376"]}
 {"create": {"_index": "profiling-events-all"}}
 {"Stacktrace.count": [1], "profiling.project.id": ["100"], "os.kernel": ["9.9.9-0"], "tags": ["environment:qa", "region:eu-west-1"], "host.ip": ["192.168.1.2"], "@timestamp": ["1698624000"], "container.name": ["instance-0000000010"], "ecs.version": ["1.12.0"], "Stacktrace.id": ["hRqQI2CBPiapzgFG9jrmDA"], "agent.version": ["head-be593ef3-1688111067"], "host.name": ["ip-192-168-1-2"], "host.id": ["8457605156473051743"], "process.thread.name": ["599103450330106"]}

x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/GetFlamegraphResponse.java

@@ -194,6 +194,14 @@ public List<Integer> getSourceLines() {
         return sourceLines;
     }
 
+    public List<Double> getAnnualCO2TonsInclusive() {
+        return annualCO2TonsInclusive;
+    }
+
+    public List<Double> getAnnualCostsUSDInclusive() {
+        return annualCostsUSDInclusive;
+    }
+
     public long getSelfCPU() {
         return selfCPU;
     }

x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/GetStackTracesResponseBuilder.java

@@ -54,6 +54,10 @@ public void setTotalFrames(int totalFrames) {
         this.totalFrames = totalFrames;
     }
 
+    public void addTotalFrames(int numFrames) {
+        this.totalFrames += numFrames;
+    }
+
     public void setStackFrames(Map<String, StackFrame> stackFrames) {
         this.stackFrames = stackFrames;
     }