Merge branch 'main' into custom-roles-get-roles-api

.backportrc.json

@@ -1,9 +1,9 @@
 {
   "upstream" : "elastic/elasticsearch",
-  "targetBranchChoices" : [ "main", "8.12", "8.11", "8.10", "8.9", "8.8", "8.7", "8.6", "8.5", "8.4", "8.3", "8.2", "8.1", "8.0", "7.17", "6.8" ],
+  "targetBranchChoices" : [ "main", "8.13", "8.12", "8.11", "8.10", "8.9", "8.8", "8.7", "8.6", "8.5", "8.4", "8.3", "8.2", "8.1", "8.0", "7.17", "6.8" ],
   "targetPRLabels" : [ "backport" ],
   "branchLabelMapping" : {
-    "^v8.13.0$" : "main",
+    "^v8.14.0$" : "main",
     "^v(\\d+).(\\d+).\\d+(?:-(?:alpha|beta|rc)\\d+)?$" : "$1.$2"
   }
 }

.buildkite/pipelines/intake.yml

@@ -48,7 +48,7 @@ steps:
         timeout_in_minutes: 300
         matrix:
           setup:
-            BWC_VERSION: ["7.17.19", "8.12.2", "8.13.0"]
+            BWC_VERSION: ["7.17.19", "8.12.2", "8.13.0", "8.14.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004

.buildkite/pipelines/periodic-packaging.yml

@@ -1889,6 +1889,22 @@ steps:
         env:
           BWC_VERSION: 8.13.0
 
+      - label: "{{matrix.image}} / 8.14.0 / packaging-tests-upgrade"
+        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.14.0
+        timeout_in_minutes: 300
+        matrix:
+          setup:
+            image:
+              - rocky-8
+              - ubuntu-2004
+        agents:
+          provider: gcp
+          image: family/elasticsearch-{{matrix.image}}
+          machineType: custom-16-32768
+          buildDirectory: /dev/shm/bk
+        env:
+          BWC_VERSION: 8.14.0
+
   - group: packaging-tests-windows
     steps:
       - label: "{{matrix.image}} / packaging-tests-windows"

.buildkite/pipelines/periodic.yml

@@ -1162,6 +1162,16 @@ steps:
           buildDirectory: /dev/shm/bk
         env:
           BWC_VERSION: 8.13.0
+      - label: 8.14.0 / bwc
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.14.0#bwcTest
+        timeout_in_minutes: 300
+        agents:
+          provider: gcp
+          image: family/elasticsearch-ubuntu-2004
+          machineType: n1-standard-32
+          buildDirectory: /dev/shm/bk
+        env:
+          BWC_VERSION: 8.14.0
   - label: concurrent-search-tests
     command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dtests.jvm.argline=-Des.concurrent_search=true -Des.concurrent_search=true functionalTests
     timeout_in_minutes: 420

.ci/bwcVersions

@@ -115,3 +115,4 @@ BWC_VERSION:
   - "8.12.1"
   - "8.12.2"
   - "8.13.0"
+  - "8.14.0"

.ci/snapshotBwcVersions

@@ -2,3 +2,4 @@ BWC_VERSION:
   - "7.17.19"
   - "8.12.2"
   - "8.13.0"
+  - "8.14.0"

.github/workflows/docs-preview-links.yml

@@ -18,8 +18,8 @@ jobs:
           script: |
             const pr = context.payload.pull_request;
             const comment = `Documentation preview:
-              - ✨ [Changed pages](https://${context.repo.repo}_${pr.number}.docs-preview.app.elstc.co/diff)`;
-            
+              - ✨ [Changed pages](https://${context.repo.repo}_bk_${pr.number}.docs-preview.app.elstc.co/diff)`;
+
             github.rest.issues.createComment({
               issue_number: context.issue.number,
               owner: context.repo.owner,

branches.json

@@ -4,6 +4,9 @@
     {
       "branch": "main"
     },
+    {
+      "branch": "8.13"
+    },
     {
       "branch": "8.12"
     },

build-tools-internal/version.properties

@@ -1,4 +1,4 @@
-elasticsearch     = 8.13.0
+elasticsearch     = 8.14.0
 lucene            = 9.9.2
 
 bundled_jdk_vendor = openjdk

docs/changelog/105476.yaml

@@ -0,0 +1,6 @@
+pr: 105476
+summary: "ESQL: Fix bug in grammar that allowed spaces inside id pattern"
+area: ES|QL
+type: bug
+issues:
+ - 105441

docs/changelog/105477.yaml

@@ -0,0 +1,6 @@
+pr: 105477
+summary: "ESQL: Introduce expression validation phase"
+area: ES|QL
+type: enhancement
+issues:
+ - 105425

docs/changelog/105499.yaml

@@ -0,0 +1,5 @@
+pr: 105499
+summary: Fix a bug where destination index aliases are not set up for an unattended transform
+area: Transform
+type: bug
+issues: []

docs/reference/inference/put-inference.asciidoc

@@ -6,10 +6,12 @@ experimental[]
 
 Creates a model to perform an {infer} task.
 
-IMPORTANT: The {infer} APIs enable you to use certain services, such as ELSER, 
-OpenAI, or Hugging Face, in your cluster. This is not the same feature that you 
-can use on an ML node with custom {ml} models. If you want to train and use your 
-own model, use the <<ml-df-trained-models-apis>>.
+IMPORTANT: The {infer} APIs enable you to use certain services, such as built-in 
+{ml} models (ELSER, E5), models uploaded through Eland, Cohere, OpenAI, or 
+Hugging Face, in your cluster. For built-in models and models uploaded though 
+Eland, the {infer} APIs offer an alternative way to use and manage trained 
+models. However, if you do not plan to use the {infer} APIs to use these models 
+or if you want to use non-NLP models, use the <<ml-df-trained-models-apis>>.
 
 
 [discrete]
@@ -39,6 +41,7 @@ The following services are available through the {infer} API:
 * ELSER
 * Hugging Face
 * OpenAI
+* text embedding (for built-in models and models uploaded through Eland)
 
 
 [discrete]
@@ -70,13 +73,15 @@ Available services:
 * `hugging_face`: specify the `text_embedding` task type to use the Hugging Face 
 service.
 * `openai`: specify the `text_embedding` task type to use the OpenAI service.
+* `text_embedding`: specify the `text_embedding` task type to use the E5 
+built-in model or text embedding models uploaded by Eland.
 
 `service_settings`::
 (Required, object)
 Settings used to install the {infer} model. These settings are specific to the
 `service` you specified.
 +
-.`service_settings` for `cohere`
+.`service_settings` for the `cohere` service
 [%collapsible%closed]
 =====
 `api_key`:::
@@ -106,19 +111,22 @@ https://docs.cohere.com/reference/embed[Cohere docs]. Defaults to
 `embed-english-v2.0`.
 =====
 +
-.`service_settings` for `elser`
+.`service_settings` for the `elser` service
 [%collapsible%closed]
 =====
 `num_allocations`:::
 (Required, integer)
-The number of model allocations to create. 
+The number of model allocations to create. `num_allocations` must not exceed the 
+number of available processors per node divided by the `num_threads`.
 
 `num_threads`:::
 (Required, integer)
-The number of threads to use by each model allocation.
+The number of threads to use by each model allocation. `num_threads` must not 
+exceed the number of available processors per node divided by the number of 
+allocations. Must be a power of 2. Max allowed value is 32.
 =====
 +
-.`service_settings` for `hugging_face`
+.`service_settings` for the `hugging_face` service
 [%collapsible%closed]
 =====
 `api_key`:::
@@ -138,7 +146,7 @@ the same name and the updated API key.
 The URL endpoint to use for the requests.
 =====
 +
-.`service_settings` for `openai`
+.`service_settings` for the `openai` service
 [%collapsible%closed]
 =====
 `api_key`:::
@@ -164,13 +172,36 @@ https://platform.openai.com/account/organization[**Settings** > **Organizations*
 The URL endpoint to use for the requests. Can be changed for testing purposes.
 Defaults to `https://api.openai.com/v1/embeddings`.
 =====
++
+.`service_settings` for the `text_embedding` service
+[%collapsible%closed]
+=====
+`model_id`:::
+(Required, string)
+The name of the text embedding model to use for the {infer} task. It can be the 
+ID of either a built-in model (for example, `.multilingual-e5-small` for E5) or 
+a text embedding model already
+{ml-docs}/ml-nlp-import-model.html#ml-nlp-import-script[uploaded through Eland].
+
+`num_allocations`:::
+(Required, integer)
+The number of model allocations to create. `num_allocations` must not exceed the 
+number of available processors per node divided by the `num_threads`.
+
+`num_threads`:::
+(Required, integer)
+The number of threads to use by each model allocation. `num_threads` must not 
+exceed the number of available processors per node divided by the number of 
+allocations. Must be a power of 2. Max allowed value is 32.
+=====
+
 
 `task_settings`::
 (Optional, object)
 Settings to configure the {infer} task. These settings are specific to the
 `<task_type>` you specified.
 +
-.`task_settings` for `text_embedding`
+.`task_settings` for the `text_embedding` task type
 [%collapsible%closed]
 =====
 `input_type`:::
@@ -234,6 +265,31 @@ PUT _inference/text_embedding/cohere-embeddings
 // TEST[skip:TBD]
 
 
+[discrete]
+[[inference-example-e5]]
+===== E5 via the text embedding service
+
+The following example shows how to create an {infer} model called
+`my-e5-model` to perform a `text_embedding` task type.
+
+[source,console]
+------------------------------------------------------------
+PUT _inference/text_embedding/my-e5-model
+{
+  "service": "text_embedding",
+  "service_settings": {
+    "num_allocations": 1,
+    "num_threads": 1,
+    "model_id": ".multilingual-e5-small" <1>
+  }
+}
+------------------------------------------------------------
+// TEST[skip:TBD]
+<1> The `model_id` must be the ID of one of the built-in E5 models. Valid values 
+are `.multilingual-e5-small` and `.multilingual-e5-small_linux-x86_64`. For 
+further details, refer to the {ml-docs}/ml-nlp-e5.html[E5 model documentation].
+
+
 [discrete]
 [[inference-example-elser]]
 ===== ELSER service
@@ -304,6 +360,30 @@ endpoint URL. Select the model you want to use on the new endpoint creation page
 task under the Advanced configuration section. Create the endpoint. Copy the URL 
 after the endpoint initialization has been finished.
 
+[discrete]
+[[inference-example-eland]]
+===== Models uploaded by Eland via the text embedding service
+
+The following example shows how to create an {infer} model called
+`my-msmarco-minilm-model` to perform a `text_embedding` task type.
+
+[source,console]
+------------------------------------------------------------
+PUT _inference/text_embedding/my-msmarco-minilm-model
+{
+  "service": "text_embedding",
+  "service_settings": {
+    "num_allocations": 1,
+    "num_threads": 1,
+    "model_id": "msmarco-MiniLM-L12-cos-v5" <1>
+  }
+}
+------------------------------------------------------------
+// TEST[skip:TBD]
+<1> The `model_id` must be the ID of a text embedding model which has already 
+been 
+{ml-docs}/ml-nlp-import-model.html#ml-nlp-import-script[uploaded through Eland].
+
 
 [discrete]
 [[inference-example-openai]]

docs/reference/migration/index.asciidoc

@@ -1,5 +1,6 @@
 include::migration_intro.asciidoc[]
 
+* <<migrating-8.14,Migrating to 8.14>>
 * <<migrating-8.13,Migrating to 8.13>>
 * <<migrating-8.12,Migrating to 8.12>>
 * <<migrating-8.11,Migrating to 8.11>>
@@ -15,6 +16,7 @@ include::migration_intro.asciidoc[]
 * <<migrating-8.1,Migrating to 8.1>>
 * <<migrating-8.0,Migrating to 8.0>>
 
+include::migrate_8_14.asciidoc[]
 include::migrate_8_13.asciidoc[]
 include::migrate_8_12.asciidoc[]
 include::migrate_8_11.asciidoc[]

docs/reference/migration/migrate_8_14.asciidoc

@@ -0,0 +1,20 @@
+[[migrating-8.14]]
+== Migrating to 8.14
+++++
+<titleabbrev>8.14</titleabbrev>
+++++
+
+This section discusses the changes that you need to be aware of when migrating
+your application to {es} 8.14.
+
+See also <<release-highlights>> and <<es-release-notes>>.
+
+coming::[8.14.0]
+
+
+[discrete]
+[[breaking-changes-8.14]]
+=== Breaking changes
+
+There are no breaking changes in {es} 8.14.
+

docs/reference/modules/remote-clusters.asciidoc

@@ -30,6 +30,11 @@ capabilities, the local and remote cluster must be on the same
 [discrete]
 === Add remote clusters
 
+NOTE: The instructions that follow describe how to create a remote connection from a 
+self-managed cluster. You can also set up {ccs} and {ccr} from an 
+link:https://www.elastic.co/guide/en/cloud/current/ec-enable-ccs.html[{ess} deployment] 
+or from an link:https://www.elastic.co/guide/en/cloud-enterprise/current/ece-enable-ccs.html[{ece} deployment].
+
 To add remote clusters, you can choose between
 <<remote-clusters-security-models,two security models>> and
 <<sniff-proxy-modes,two connection modes>>. Both security models are compatible

docs/reference/release-notes.asciidoc

@@ -6,6 +6,7 @@
 
 This section summarizes the changes in each release.
 
+* <<release-notes-8.14.0>>
 * <<release-notes-8.13.0>>
 * <<release-notes-8.12.1>>
 * <<release-notes-8.12.0>>
@@ -60,6 +61,7 @@ This section summarizes the changes in each release.
 
 --
 
+include::release-notes/8.14.0.asciidoc[]
 include::release-notes/8.13.0.asciidoc[]
 include::release-notes/8.12.1.asciidoc[]
 include::release-notes/8.12.0.asciidoc[]

docs/reference/release-notes/8.14.0.asciidoc

@@ -0,0 +1,8 @@
+[[release-notes-8.14.0]]
+== {es} version 8.14.0
+
+coming[8.14.0]
+
+Also see <<breaking-changes-8.14,Breaking changes in 8.14>>.
+
+