Merge branch '8.16' into backport/8.16/pr-117491

.buildkite/hooks/pre-command

@@ -47,6 +47,8 @@ export GRADLE_BUILD_CACHE_PASSWORD
 BUILDKITE_API_TOKEN=$(vault read -field=token secret/ci/elastic-elasticsearch/buildkite-api-token)
 export BUILDKITE_API_TOKEN
 
+export GH_TOKEN="$VAULT_GITHUB_TOKEN"
+
 if [[ "${USE_LUCENE_SNAPSHOT_CREDS:-}" == "true" ]]; then
   data=$(.buildkite/scripts/get-legacy-secret.sh aws-elastic/creds/lucene-snapshots)
 
@@ -117,3 +119,5 @@ if [[ -f /etc/os-release ]] && grep -q '"Amazon Linux 2"' /etc/os-release; then
   echo "$(hostname -i | cut -d' ' -f 2)  $(hostname -f)." | sudo tee /etc/dnsmasq.hosts
   sudo systemctl restart dnsmasq.service
 fi
+
+.buildkite/scripts/get-latest-test-mutes.sh

.buildkite/hooks/pre-command.bat

@@ -15,9 +15,12 @@ set BUILD_NUMBER=%BUILDKITE_BUILD_NUMBER%
 set COMPOSE_HTTP_TIMEOUT=120
 set JOB_BRANCH=%BUILDKITE_BRANCH%
 
+set GH_TOKEN=%VAULT_GITHUB_TOKEN%
+
 set GRADLE_BUILD_CACHE_USERNAME=vault read -field=username secret/ci/elastic-elasticsearch/migrated/gradle-build-cache
 set GRADLE_BUILD_CACHE_PASSWORD=vault read -field=password secret/ci/elastic-elasticsearch/migrated/gradle-build-cache
 
 bash.exe -c "nohup bash .buildkite/scripts/setup-monitoring.sh </dev/null >/dev/null 2>&1 &"
+bash.exe -c "bash .buildkite/scripts/get-latest-test-mutes.sh"
 
 exit /b 0

.buildkite/pipelines/periodic-packaging.yml

@@ -559,8 +559,8 @@ steps:
         env:
           BWC_VERSION: 8.14.3
 
-      - label: "{{matrix.image}} / 8.15.4 / packaging-tests-upgrade"
-        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.15.4
+      - label: "{{matrix.image}} / 8.15.6 / packaging-tests-upgrade"
+        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.15.6
         timeout_in_minutes: 300
         matrix:
           setup:
@@ -573,7 +573,7 @@ steps:
           machineType: custom-16-32768
           buildDirectory: /dev/shm/bk
         env:
-          BWC_VERSION: 8.15.4
+          BWC_VERSION: 8.15.6
 
       - label: "{{matrix.image}} / 8.16.2 / packaging-tests-upgrade"
         command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.16.2

.buildkite/pipelines/periodic.yml

@@ -629,8 +629,8 @@ steps:
             - signal_reason: agent_stop
               limit: 3
 
-      - label: 8.15.4 / bwc
-        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.15.4#bwcTest
+      - label: 8.15.6 / bwc
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.15.6#bwcTest
         timeout_in_minutes: 300
         agents:
           provider: gcp
@@ -639,7 +639,7 @@ steps:
           buildDirectory: /dev/shm/bk
           preemptible: true
         env:
-          BWC_VERSION: 8.15.4
+          BWC_VERSION: 8.15.6
         retry:
           automatic:
             - exit_status: "-1"

.buildkite/scripts/get-latest-test-mutes.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+if [[ ! "${BUILDKITE_PULL_REQUEST:-}" || "${BUILDKITE_AGENT_META_DATA_PROVIDER:-}" == "k8s" ]]; then
+  exit 0
+fi
+
+testMuteBranch="${BUILDKITE_PULL_REQUEST_BASE_BRANCH:-main}"
+testMuteFile="$(mktemp)"
+
+# If this PR contains changes to muted-tests.yml, we disable this functionality
+# Otherwise, we wouldn't be able to test unmutes
+if [[ ! $(gh pr diff "$BUILDKITE_PULL_REQUEST" --name-only | grep 'muted-tests.yml') ]]; then
+  gh api -H 'Accept: application/vnd.github.v3.raw' "repos/elastic/elasticsearch/contents/muted-tests.yml?ref=$testMuteBranch" > "$testMuteFile"
+
+  if [[ -s "$testMuteFile" ]]; then
+    mkdir -p ~/.gradle
+    # This is using gradle.properties instead of an env var so that it's easily compatible with the Windows pre-command hook
+    echo "org.gradle.project.org.elasticsearch.additional.muted.tests=$testMuteFile" >> ~/.gradle/gradle.properties
+  fi
+fi

.ci/bwcVersions

@@ -32,5 +32,5 @@ BWC_VERSION:
   - "8.12.2"
   - "8.13.4"
   - "8.14.3"
-  - "8.15.4"
+  - "8.15.6"
   - "8.16.2"

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/MutedTestsBuildService.java

@@ -28,10 +28,12 @@
 import java.io.UncheckedIOException;
 import java.util.ArrayList;
 import java.util.Collections;
+import java.util.LinkedHashSet;
 import java.util.List;
+import java.util.Set;
 
 public abstract class MutedTestsBuildService implements BuildService<MutedTestsBuildService.Params> {
-    private final List<String> excludePatterns = new ArrayList<>();
+    private final Set<String> excludePatterns = new LinkedHashSet<>();
     private final ObjectMapper objectMapper = new ObjectMapper(new YAMLFactory());
 
     public MutedTestsBuildService() {
@@ -43,23 +45,23 @@ public MutedTestsBuildService() {
         }
     }
 
-    public List<String> getExcludePatterns() {
+    public Set<String> getExcludePatterns() {
         return excludePatterns;
     }
 
-    private List<String> buildExcludePatterns(File file) {
+    private Set<String> buildExcludePatterns(File file) {
         List<MutedTest> mutedTests;
 
         try (InputStream is = new BufferedInputStream(new FileInputStream(file))) {
             mutedTests = objectMapper.readValue(is, MutedTests.class).getTests();
             if (mutedTests == null) {
-                return Collections.emptyList();
+                return Collections.emptySet();
             }
         } catch (IOException e) {
             throw new UncheckedIOException(e);
         }
 
-        List<String> excludes = new ArrayList<>();
+        Set<String> excludes = new LinkedHashSet<>();
         if (mutedTests.isEmpty() == false) {
             for (MutedTestsBuildService.MutedTest mutedTest : mutedTests) {
                 if (mutedTest.getClassName() != null && mutedTest.getMethods().isEmpty() == false) {

docs/changelog/117503.yaml

@@ -0,0 +1,6 @@
+pr: 117503
+summary: Fix COUNT filter pushdown
+area: ES|QL
+type: bug
+issues:
+ - 115522

docs/reference/connector/docs/connectors-sharepoint-online.asciidoc

@@ -87,14 +87,16 @@ Select an expiration date. (At this expiration date, you will need to generate a
 +
 ```
 Graph API
-- Sites.Read.All
+- Sites.Selected
 - Files.Read.All
 - Group.Read.All
 - User.Read.All
 
 Sharepoint
-- Sites.Read.All
+- Sites.Selected
 ```
+NOTE: If the `Comma-separated list of sites` configuration is set to `*` or if a user enables the toggle button `Enumerate all sites`, the connector requires `Sites.Read.All` permission.
+
 * **Grant admin consent**, using the `Grant Admin Consent` link from the permissions screen.
 * Save the tenant name (i.e. Domain name) of Azure platform.
 
@@ -138,7 +140,7 @@ Refer to https://learn.microsoft.com/en-us/sharepoint/dev/general-development/ho
 
 Here's a summary of why we use these Graph API permissions:
 
-* *Sites.Read.All* is used to fetch the sites and their metadata
+* *Sites.Selected* is used to fetch the sites and their metadata
 * *Files.Read.All* is used to fetch Site Drives and files in these drives
 * *Groups.Read.All* is used to fetch groups for document-level permissions
 * *User.Read.All* is used to fetch user information for document-level permissions
@@ -546,14 +548,16 @@ Select an expiration date. (At this expiration date, you will need to generate a
 +
 ```
 Graph API
-- Sites.Read.All
+- Sites.Selected
 - Files.Read.All
 - Group.Read.All
 - User.Read.All
 
 Sharepoint
-- Sites.Read.All
+- Sites.Selected
 ```
+NOTE: If the `Comma-separated list of sites` configuration is set to `*` or if a user enables the toggle button `Enumerate all sites`, the connector requires `Sites.Read.All` permission.
+
 * **Grant admin consent**, using the `Grant Admin Consent` link from the permissions screen.
 * Save the tenant name (i.e. Domain name) of Azure platform.
 
@@ -597,7 +601,7 @@ Refer to https://learn.microsoft.com/en-us/sharepoint/dev/general-development/ho
 
 Here's a summary of why we use these Graph API permissions:
 
-* *Sites.Read.All* is used to fetch the sites and their metadata
+* *Sites.Selected* is used to fetch the sites and their metadata
 * *Files.Read.All* is used to fetch Site Drives and files in these drives
 * *Groups.Read.All* is used to fetch groups for document-level permissions
 * *User.Read.All* is used to fetch user information for document-level permissions

docs/reference/ml/trained-models/apis/get-trained-models-stats.asciidoc

@@ -235,7 +235,7 @@ The reason for the current state. Usually only populated when the `routing_state
 (string)
 The current routing state.
 --
-* `starting`: The model is attempting to allocate on this model, inference calls are not yet accepted.
+* `starting`: The model is attempting to allocate on this node, inference calls are not yet accepted.
 * `started`: The model is allocated and ready to accept inference requests.
 * `stopping`: The model is being deallocated from this node.
 * `stopped`: The model is fully deallocated from this node.

server/src/main/java/org/elasticsearch/Version.java

@@ -188,6 +188,7 @@ public class Version implements VersionId<Version>, ToXContentFragment {
     public static final Version V_8_15_2 = new Version(8_15_02_99);
     public static final Version V_8_15_3 = new Version(8_15_03_99);
     public static final Version V_8_15_4 = new Version(8_15_04_99);
+    public static final Version V_8_15_6 = new Version(8_15_06_99);
     public static final Version V_8_16_0 = new Version(8_16_00_99);
     public static final Version V_8_16_1 = new Version(8_16_01_99);
     public static final Version V_8_16_2 = new Version(8_16_02_99);

server/src/main/resources/org/elasticsearch/TransportVersions.csv

@@ -133,5 +133,6 @@
 8.15.2,8702003
 8.15.3,8702003
 8.15.4,8702003
+8.15.5,8702003
 8.16.0,8772001
 8.16.1,8772004

server/src/main/resources/org/elasticsearch/index/IndexVersions.csv

@@ -133,5 +133,6 @@
 8.15.2,8512000
 8.15.3,8512000
 8.15.4,8512000
+8.15.5,8512000
 8.16.0,8518000
 8.16.1,8518000

x-pack/plugin/esql/qa/testFixtures/src/main/resources/stats.csv-spec

@@ -2537,6 +2537,57 @@ c2:l |c2_f:l |m2:i |m2_f:i |c:l
 1    |1      |5    |5      |21
 ;
 
+simpleCountOnFieldWithFilteringAndNoGrouping
+required_capability: per_agg_filtering
+from employees
+| stats c1 = count(emp_no) where emp_no < 10042
+;
+
+c1:long
+41
+;
+
+simpleCountOnFieldWithFilteringOnDifferentFieldAndNoGrouping
+required_capability: per_agg_filtering
+from employees
+| stats c1 = count(hire_date) where emp_no < 10042
+;
+
+c1:long
+41
+;
+
+simpleCountOnStarWithFilteringAndNoGrouping
+required_capability: per_agg_filtering
+from employees
+| stats c1 = count(*) where emp_no < 10042
+;
+
+c1:long
+41
+;
+
+simpleCountWithFilteringAndNoGroupingOnFieldWithNulls
+required_capability: per_agg_filtering
+from employees
+| stats c1 = count(birth_date) where emp_no <= 10050
+;
+
+c1:long
+40
+;
+
+
+simpleCountWithFilteringAndNoGroupingOnFieldWithMultivalues
+required_capability: per_agg_filtering
+from employees
+| stats c1 = count(job_positions) where emp_no <= 10003
+;
+
+c1:long
+3
+;
+
 filterIsAlwaysTrue
 required_capability: per_agg_filtering
 FROM employees