Merge branch 'main' into custom-roles-poc-es

benchmarks/src/main/java/org/elasticsearch/benchmark/index/mapper/MapperServiceFactory.java

@@ -26,7 +26,6 @@
 import org.elasticsearch.index.mapper.ProvidedIdFieldMapper;
 import org.elasticsearch.index.similarity.SimilarityService;
 import org.elasticsearch.indices.IndicesModule;
-import org.elasticsearch.plugins.internal.DocumentParsingObserver;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.script.ScriptCompiler;
 import org.elasticsearch.script.ScriptContext;
@@ -72,8 +71,7 @@ public static MapperService create(String mappings) {
                 public <T> T compile(Script script, ScriptContext<T> scriptContext) {
                     throw new UnsupportedOperationException();
                 }
-            },
-            () -> DocumentParsingObserver.EMPTY_INSTANCE
+            }
         );
 
         try {

benchmarks/src/main/java/org/elasticsearch/benchmark/search/QueryParserHelperBenchmark.java

@@ -40,7 +40,6 @@
 import org.elasticsearch.index.similarity.SimilarityService;
 import org.elasticsearch.indices.IndicesModule;
 import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
-import org.elasticsearch.plugins.internal.DocumentParsingObserver;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.script.ScriptCompiler;
 import org.elasticsearch.script.ScriptContext;
@@ -187,8 +186,7 @@ protected final MapperService createMapperService(String mappings) {
                 public <T> T compile(Script script, ScriptContext<T> scriptContext) {
                     throw new UnsupportedOperationException();
                 }
-            },
-            () -> DocumentParsingObserver.EMPTY_INSTANCE
+            }
         );
 
         try {

docs/changelog/104859.yaml

@@ -0,0 +1,5 @@
+pr: 104859
+summary: ES - document observing with rejections
+area: Infra/Core
+type: enhancement
+issues: []

docs/changelog/105403.yaml

@@ -0,0 +1,6 @@
+pr: 105403
+summary: "ESQL: make `cidr_match` foldable"
+area: ES|QL
+type: bug
+issues:
+ - 105376

docs/reference/docs/bulk.asciidoc

@@ -246,7 +246,7 @@ on.
 
 `list_executed_pipelines`::
 (Optional, Boolean) If `true`, the response will include the ingest pipelines that
-were executed for each `index` or ``create`.
+were executed for each `index` or `create`.
 Defaults to `false`.
 
 include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=pipeline]

docs/reference/inference/put-inference.asciidoc

@@ -35,9 +35,10 @@ perform a specific {infer} task.
 
 The following services are available through the {infer} API:
 
+* Cohere
 * ELSER
-* OpenAI
 * Hugging Face
+* OpenAI
 
 
 [discrete]
@@ -64,16 +65,47 @@ The type of the {infer} task that the model will perform. Available task types:
 (Required, string)
 The type of service supported for the specified task type.
 Available services:
+* `cohere`: specify the `text_embedding` task type to use the Cohere service. 
 * `elser`: specify the `sparse_embedding` task type to use the ELSER service.
-* `openai`: specify the `text_embedding` task type to use the OpenAI service.
 * `hugging_face`: specify the `text_embedding` task type to use the Hugging Face 
 service.
+* `openai`: specify the `text_embedding` task type to use the OpenAI service.
 
 `service_settings`::
 (Required, object)
 Settings used to install the {infer} model. These settings are specific to the
 `service` you specified.
 +
+.`service_settings` for `cohere`
+[%collapsible%closed]
+=====
+`api_key`:::
+(Required, string)
+A valid API key of your Cohere account. You can find your Cohere API keys or you 
+can create a new one 
+https://dashboard.cohere.com/api-keys[on the API keys settings page].
+
+IMPORTANT: You need to provide the API key only once, during the {infer} model 
+creation. The <<get-inference-api>> does not retrieve your API key. After 
+creating the {infer} model, you cannot change the associated API key. If you 
+want to use a different API key, delete the {infer} model and recreate it with 
+the same name and the updated API key.
+
+`embedding_type`::
+(Optional, string)
+Specifies the types of embeddings you want to get back. Defaults to `float`.
+Valid values are:
+  * `float`: use it for the default float embeddings.
+  * `int8`: use it for signed int8 embeddings.
+
+`model_id`::
+(Optional, string)
+The name of the model to use for the {infer} task. To review the available 
+models, refer to the 
+https://docs.cohere.com/reference/embed[Cohere docs]. Defaults to 
+`embed-english-v2.0`.
+=====
++
 .`service_settings` for `elser`
 [%collapsible%closed]
 =====
@@ -86,51 +118,51 @@ The number of model allocations to create.
 The number of threads to use by each model allocation.
 =====
 +
-.`service_settings` for `openai`
+.`service_settings` for `hugging_face`
 [%collapsible%closed]
 =====
 `api_key`:::
 (Required, string)
-A valid API key of your OpenAI account. You can find your OpenAI API keys in 
-your OpenAI account under the 
-https://platform.openai.com/api-keys[API keys section].
+A valid access token of your Hugging Face account. You can find your Hugging 
+Face access tokens or you can create a new one 
+https://huggingface.co/settings/tokens[on the settings page].
 
 IMPORTANT: You need to provide the API key only once, during the {infer} model 
 creation. The <<get-inference-api>> does not retrieve your API key. After 
 creating the {infer} model, you cannot change the associated API key. If you 
 want to use a different API key, delete the {infer} model and recreate it with 
 the same name and the updated API key.
 
-`organization_id`:::
-(Optional, string)
-The unique identifier of your organization. You can find the Organization ID in 
-your OpenAI account under 
-https://platform.openai.com/account/organization[**Settings** > **Organizations**]. 
-
 `url`:::
-(Optional, string)
-The URL endpoint to use for the requests. Can be changed for testing purposes.
-Defaults to `https://api.openai.com/v1/embeddings`.
+(Required, string)
+The URL endpoint to use for the requests.
 =====
 +
-.`service_settings` for `hugging_face`
+.`service_settings` for `openai`
 [%collapsible%closed]
 =====
 `api_key`:::
 (Required, string)
-A valid access token of your Hugging Face account. You can find your Hugging 
-Face access tokens or you can create a new one 
-https://huggingface.co/settings/tokens[on the settings page].
+A valid API key of your OpenAI account. You can find your OpenAI API keys in 
+your OpenAI account under the 
+https://platform.openai.com/api-keys[API keys section].
 
 IMPORTANT: You need to provide the API key only once, during the {infer} model 
 creation. The <<get-inference-api>> does not retrieve your API key. After 
 creating the {infer} model, you cannot change the associated API key. If you 
 want to use a different API key, delete the {infer} model and recreate it with 
 the same name and the updated API key.
 
+`organization_id`:::
+(Optional, string)
+The unique identifier of your organization. You can find the Organization ID in 
+your OpenAI account under 
+https://platform.openai.com/account/organization[**Settings** > **Organizations**]. 
+
 `url`:::
-(Required, string)
-The URL endpoint to use for the requests.
+(Optional, string)
+The URL endpoint to use for the requests. Can be changed for testing purposes.
+Defaults to `https://api.openai.com/v1/embeddings`.
 =====
 
 `task_settings`::
@@ -141,11 +173,33 @@ Settings to configure the {infer} task. These settings are specific to the
 .`task_settings` for `text_embedding`
 [%collapsible%closed]
 =====
+`input_type`:::
+(optional, string)
+For `cohere` service only. Specifies the type of input passed to the model.
+Valid values are:
+  * `classification`: use it for embeddings passed through a text classifier.
+  * `clusterning`: use it for the embeddings run through a clustering algorithm.
+  * `ingest`: use it for storing document embeddings in a vector database.
+  * `search`: use it for storing embeddings of search queries run against a 
+  vector data base to find relevant documents.
+
 `model`:::
 (Optional, string)
-The name of the model to use for the {infer} task. Refer to the 
+For `openai` sevice only. The name of the model to use for the {infer} task. Refer 
+to the 
 https://platform.openai.com/docs/guides/embeddings/what-are-embeddings[OpenAI documentation]
 for the list of available text embedding models.
+
+`truncate`:::
+(Optional, string)
+For `cohere` service only. Specifies how the API handles inputs longer than the 
+maximum token length. Defaults to `END`. Valid values are:
+ * `NONE`: when the input exceeds the maximum input token length an error is 
+ returned.
+ * `START`: when the input exceeds the maximum input token length the start of 
+ the input is discarded.
+ * `END`: when the input exceeds the maximum input token length the end of 
+ the input is discarded. 
 =====
 
 
@@ -156,6 +210,30 @@ for the list of available text embedding models.
 This section contains example API calls for every service type.
 
 
+[discrete]
+[[inference-example-cohere]]
+===== Cohere service
+
+The following example shows how to create an {infer} model called
+`cohere_embeddings` to perform a `text_embedding` task type.
+
+[source,console]
+------------------------------------------------------------
+PUT _inference/text_embedding/cohere-embeddings
+{
+    "service": "cohere",
+    "service_settings": {
+        "api_key": "<api_key>",
+        "model": "embed-english-light-v3.0",
+        "embedding_type": "int8"
+    },
+    "task_settings": {
+    }
+}
+------------------------------------------------------------
+// TEST[skip:TBD]
+
+
 [discrete]
 [[inference-example-elser]]
 ===== ELSER service
@@ -196,29 +274,6 @@ Example response:
 // NOTCONSOLE
 
 
-[discrete]
-[[inference-example-openai]]
-===== OpenAI service
-
-The following example shows how to create an {infer} model called
-`openai_embeddings` to perform a `text_embedding` task type.
-
-[source,console]
-------------------------------------------------------------
-PUT _inference/text_embedding/openai_embeddings
-{
-    "service": "openai",
-    "service_settings": {
-        "api_key": "<api_key>"
-    },
-    "task_settings": {
-       "model": "text-embedding-ada-002"
-    }
-}
-------------------------------------------------------------
-// TEST[skip:TBD]
-
-
 [discrete]
 [[inference-example-hugging-face]]
 ===== Hugging Face service
@@ -247,4 +302,27 @@ https://ui.endpoints.huggingface.co/[the Hugging Face endpoint page] to get an
 endpoint URL. Select the model you want to use on the new endpoint creation page 
 - for example `intfloat/e5-small-v2` - then select the `Sentence Embeddings` 
 task under the Advanced configuration section. Create the endpoint. Copy the URL 
-after the endpoint initialization has been finished.
+after the endpoint initialization has been finished.
+
+
+[discrete]
+[[inference-example-openai]]
+===== OpenAI service
+
+The following example shows how to create an {infer} model called
+`openai_embeddings` to perform a `text_embedding` task type.
+
+[source,console]
+------------------------------------------------------------
+PUT _inference/text_embedding/openai_embeddings
+{
+    "service": "openai",
+    "service_settings": {
+        "api_key": "<api_key>"
+    },
+    "task_settings": {
+       "model": "text-embedding-ada-002"
+    }
+}
+------------------------------------------------------------
+// TEST[skip:TBD]