Merge branch 'main' into custom-roles-api-keys

n1v0lg · Mar 5, 2024 · ae6571b · ae6571b
2 parents 9d49912 + e9ff896
commit ae6571b
Show file tree

Hide file tree

Showing 209 changed files with 2,916 additions and 1,684 deletions.
diff --git a/README.asciidoc b/README.asciidoc
@@ -1,20 +1,24 @@
 = Elasticsearch
 
-Elasticsearch is a distributed, RESTful search engine optimized for speed and relevance on production-scale workloads. You can use Elasticsearch to perform real-time search over massive datasets for applications including:
+Elasticsearch is a distributed search and analytics engine optimized for speed and relevance on production-scale workloads. Elasticsearch is the foundation of Elastic's open Stack platform. Search in near real-time over massive datasets, perform vector searches, integrate with generative AI applications, and much more.
 
-* Vector search
+Use cases enabled by Elasticsearch include:
+
+* https://www.elastic.co/search-labs/blog/articles/retrieval-augmented-generation-rag[Retrieval Augmented Generation (RAG)] 
+* https://www.elastic.co/search-labs/blog/categories/vector-search[Vector search]
 * Full-text search
 * Logs
 * Metrics
 * Application performance monitoring (APM)
 * Security logs
 
-
 \... and more!
 
 To learn more about Elasticsearch's features and capabilities, see our
 https://www.elastic.co/products/elasticsearch[product page].
 
+To access information on https://www.elastic.co/search-labs/blog/categories/ml-research[machine learning innovations] and the latest https://www.elastic.co/search-labs/blog/categories/lucene[Lucene contributions from Elastic], more information can be found in https://www.elastic.co/search-labs[Search Labs]. 
+
 [[get-started]]
 == Get started
 

diff --git a/build-tools-internal/version.properties b/build-tools-internal/version.properties
@@ -1,5 +1,5 @@
 elasticsearch     = 8.14.0
-lucene            = 9.9.2
+lucene            = 9.10.0
 
 bundled_jdk_vendor = openjdk
 bundled_jdk = 21.0.2+13@f2283984656d49d69e91c558476027ac

diff --git a/build-tools/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchCluster.java b/build-tools/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchCluster.java
@@ -433,7 +433,7 @@ private void commonNodeConfig() {
             if (node.getTestDistribution().equals(TestDistribution.INTEG_TEST)) {
                 node.defaultConfig.put("xpack.security.enabled", "false");
             } else {
-                if (node.getVersion().onOrAfter("7.16.0")) {
+                if (hasDeprecationIndexing(node)) {
                     node.defaultConfig.put("cluster.deprecation_indexing.enabled", "false");
                 }
             }
@@ -474,13 +474,17 @@ public void nextNodeToNextVersion() {
         commonNodeConfig();
         nodeIndex += 1;
         if (node.getTestDistribution().equals(TestDistribution.DEFAULT)) {
-            if (node.getVersion().onOrAfter("7.16.0")) {
+            if (hasDeprecationIndexing(node)) {
                 node.setting("cluster.deprecation_indexing.enabled", "false");
             }
         }
         node.start();
     }
 
+    private static boolean hasDeprecationIndexing(ElasticsearchNode node) {
+        return node.getVersion().onOrAfter("7.16.0") && node.getSettingKeys().contains("stateless.enabled") == false;
+    }
+
     @Override
     public void extraConfigFile(String destination, File from) {
         nodes.all(node -> node.extraConfigFile(destination, from));

diff --git a/docs/Versions.asciidoc b/docs/Versions.asciidoc
@@ -1,8 +1,8 @@
 
 include::{docs-root}/shared/versions/stack/{source_branch}.asciidoc[]
 
-:lucene_version:        9.9.2
-:lucene_version_path:   9_9_2
+:lucene_version:        9.10.0
+:lucene_version_path:   9_10_0
 :jdk:                   11.0.2
 :jdk_major:             11
 :build_type:            tar

diff --git a/docs/changelog/105578.yaml b/docs/changelog/105578.yaml
@@ -0,0 +1,13 @@
+pr: 105578
+summary: Upgrade to Lucene 9.10.0
+area: Search
+type: enhancement
+issues: []
+highlight:
+  title: New Lucene 9.10 release
+  body: |-
+    - https://github.com/apache/lucene/pull/13090: Prevent humongous allocations in ScalarQuantizer when building quantiles.
+    - https://github.com/apache/lucene/pull/12962: Speedup concurrent multi-segment HNSW graph search
+    - https://github.com/apache/lucene/pull/13033: Range queries on numeric/date/ip fields now exit earlier on segments whose values don't intersect with the query range. This should especially help when there are other required clauses in the `bool` query and when the range filter is narrow, e.g. filtering on the last 5 minutes.
+    - https://github.com/apache/lucene/pull/13026: `bool` queries that mix `filter` and `should` clauses will now propagate minimum competitive scores through the `should` clauses. This should yield speedups when sorting by descending score.
+  notable: true
diff --git a/docs/changelog/105772.yaml b/docs/changelog/105772.yaml
@@ -0,0 +1,5 @@
+pr: 105772
+summary: "[ILM] Delete step deletes data stream with only one index"
+area: ILM+SLM
+type: bug
+issues: []
diff --git a/docs/changelog/105789.yaml b/docs/changelog/105789.yaml
@@ -0,0 +1,6 @@
+pr: 105789
+summary: Make Health API more resilient to multi-version clusters
+area: Health
+type: bug
+issues:
+ - 90183
diff --git a/docs/changelog/105848.yaml b/docs/changelog/105848.yaml
@@ -0,0 +1,5 @@
+pr: 105848
+summary: '`ProjectOperator` should not retain references to released blocks'
+area: ES|QL
+type: bug
+issues: []
diff --git a/docs/reference/api-conventions.asciidoc b/docs/reference/api-conventions.asciidoc
@@ -302,6 +302,14 @@ Content-Type: application/vnd.elasticsearch+json; compatible-with=7
 Accept: application/vnd.elasticsearch+json; compatible-with=7
 ----------------------------------------------------------------------
 
+[discrete]
+[[api-push-back]]
+=== HTTP `429 Too Many Requests` status code push back
+
+{es} APIs may respond with the HTTP `429 Too Many Requests` status code, indicating that the cluster is too busy
+to handle the request. When this happens, consider retrying after a short delay. If the retry also receives
+a `429 Too Many Requests` response, extend the delay by backing off exponentially before each subsequent retry.
+
 [discrete]
 [[api-url-access-control]]
 === URL-based access control

diff --git a/docs/reference/indices/shrink-index.asciidoc b/docs/reference/indices/shrink-index.asciidoc
@@ -101,7 +101,8 @@ A shrink operation:
   disks)
 
 . Recovers the target index as though it were a closed index which
-  had just been re-opened.
+  had just been re-opened. Recovers shards to <<indices-get-settings,Index Setting>> 
+  `.routing.allocation.initial_recovery._id`.
 
 
 [[_shrinking_an_index]]

diff --git a/docs/reference/ingest/processors/geoip.asciidoc b/docs/reference/ingest/processors/geoip.asciidoc
@@ -435,6 +435,8 @@ each node's <<es-tmpdir,temporary directory>> at `$ES_TMPDIR/geoip-databases/<no
 Note that {es} will make a GET request to `${ingest.geoip.downloader.endpoint}?elastic_geoip_service_tos=agree`,
 expecting the list of metadata about databases typically found in `overview.json`.
 
+The GeoIP downloader uses the JDK's builtin cacerts. If you're using a custom endpoint, add the custom https endpoint cacert(s) to the JDK's truststore.
+
 [[ingest-geoip-downloader-poll-interval]]
 `ingest.geoip.downloader.poll.interval`::
 (<<dynamic-cluster-setting,Dynamic>>, <<time-units,time value>>)

diff --git a/docs/reference/search/search-your-data/semantic-search-inference.asciidoc b/docs/reference/search/search-your-data/semantic-search-inference.asciidoc
@@ -4,18 +4,21 @@
 <titleabbrev>Semantic search with the {infer} API</titleabbrev>
 ++++
 
-The instructions in this tutorial shows you how to use the {infer} API with the
-Open AI service to perform semantic search on your data. The following example
-uses OpenAI's `text-embedding-ada-002` second generation embedding model. You
-can use any OpenAI models, they are all supported by the {infer} API.
+The instructions in this tutorial shows you how to use the {infer} API with
+various services to perform semantic search on your data. The following examples 
+use Cohere's `embed-english-light-v3.0` model and OpenAI's 
+`text-embedding-ada-002` second generation embedding model. You can use any
+Cohere and OpenAI models, they are all supported by the {infer} API.
+
+Click the name of the service you want to use on any of the widgets below to
+review the corresponding instructions.
 
 
 [discrete]
-[[infer-openai-requirements]]
+[[infer-service-requirements]]
 ==== Requirements
 
-An https://openai.com/[OpenAI account] is required to use the {infer} API with
-the OpenAI service.
+include::{es-repo-dir}/tab-widgets/inference-api/infer-api-requirements-widget.asciidoc[]
 
 
 [discrete]
@@ -24,113 +27,30 @@ the OpenAI service.
 
 Create the {infer} task by using the <<put-inference-api>>:
 
-[source,console]
-------------------------------------------------------------
-PUT _inference/text_embedding/openai_embeddings <1>
-{
-    "service": "openai",
-    "service_settings": {
-        "api_key": "<api_key>" <2>
-    },
-    "task_settings": {
-       "model": "text-embedding-ada-002" <3>
-    }
-}
-------------------------------------------------------------
-// TEST[skip:TBD]
-<1> The task type is `text_embedding` in the path.
-<2> The API key of your OpenAI account. You can find your OpenAI API keys in
-your OpenAI account under the
-https://platform.openai.com/api-keys[API keys section]. You need to provide
-your API key only once. The <<get-inference-api>> does not return your API
-key.
-<3> The name of the embedding model to use. You can find the list of OpenAI
-embedding models
-https://platform.openai.com/docs/guides/embeddings/embedding-models[here].
+include::{es-repo-dir}/tab-widgets/inference-api/infer-api-task-widget.asciidoc[]
 
 
 [discrete]
-[[infer-openai-mappings]]
+[[infer-service-mappings]]
 ==== Create the index mapping
 
 The mapping of the destination index - the index that contains the embeddings
 that the model will create based on your input text - must be created. The
 destination index must have a field with the <<dense-vector, `dense_vector`>>
-field type to index the output of the OpenAI model.
+field type to index the output of the used model.
 
-[source,console]
---------------------------------------------------
-PUT openai-embeddings
-{
-  "mappings": {
-    "properties": {
-      "content_embedding": { <1>
-        "type": "dense_vector", <2>
-        "dims": 1536, <3>
-        "element_type": "float",
-        "similarity": "dot_product" <4>
-      },
-      "content": { <5>
-        "type": "text" <6>
-      }
-    }
-  }
-}
---------------------------------------------------
-<1> The name of the field to contain the generated tokens. It must be refrenced
-in the {infer} pipeline configuration in the next step.
-<2> The field to contain the tokens is a `dense_vector` field.
-<3> The output dimensions of the model. Find this value in the
-https://platform.openai.com/docs/guides/embeddings/embedding-models[OpenAI documentation]
-of the model you use.
-<4> The faster` dot_product` function can be used to calculate similarity
-because OpenAI embeddings are normalised to unit length. You can check the
-https://platform.openai.com/docs/guides/embeddings/which-distance-function-should-i-use[OpenAI docs]
-about which similarity function to use.
-<5> The name of the field from which to create the sparse vector representation.
-In this example, the name of the field is `content`. It must be referenced in
-the {infer} pipeline configuration in the next step.
-<6> The field type which is text in this example.
+include::{es-repo-dir}/tab-widgets/inference-api/infer-api-mapping-widget.asciidoc[]
 
 
 [discrete]
-[[infer-openai-inference-ingest-pipeline]]
+[[infer-service-inference-ingest-pipeline]]
 ==== Create an ingest pipeline with an inference processor
 
 Create an <<ingest,ingest pipeline>> with an
-<<inference-processor,{infer} processor>> and use the OpenAI model you created
-above to infer against the data that is being ingested in the
-pipeline.
+<<inference-processor,{infer} processor>> and use the model you created above to
+infer against the data that is being ingested in the pipeline.
 
-[source,console]
---------------------------------------------------
-PUT _ingest/pipeline/openai_embeddings
-{
-  "processors": [
-    {
-      "inference": {
-        "model_id": "openai_embeddings", <1>
-        "input_output": { <2>
-          "input_field": "content",
-          "output_field": "content_embedding"
-        }
-      }
-    }
-  ]
-}
---------------------------------------------------
-<1> The name of the inference model you created by using the
-<<put-inference-api>>.
-<2> Configuration object that defines the `input_field` for the {infer} process
-and the `output_field` that will contain the {infer} results.
-
-////
-[source,console]
-----
-DELETE _ingest/pipeline/openai_embeddings
-----
-// TEST[continued]
-////
+include::{es-repo-dir}/tab-widgets/inference-api/infer-api-ingest-pipeline-widget.asciidoc[]
 
 
 [discrete]
@@ -157,32 +77,10 @@ you can see an index named `test-data` with 182469 documents.
 [[reindexing-data-infer]]
 ==== Ingest the data through the {infer} ingest pipeline
 
-Create the embeddings from the text by reindexing the data throught the {infer}
-pipeline that uses the OpenAI model as the inference model.
+Create the embeddings from the text by reindexing the data through the {infer}
+pipeline that uses the chosen model as the inference model.
 
-[source,console]
-----
-POST _reindex?wait_for_completion=false
-{
-  "source": {
-    "index": "test-data",
-    "size": 50 <1>
-  },
-  "dest": {
-    "index": "openai-embeddings",
-    "pipeline": "openai_embeddings"
-  }
-}
-----
-// TEST[skip:TBD]
-<1> The default batch size for reindexing is 1000. Reducing `size` to a smaller
-number makes the update of the reindexing process quicker which enables you to
-follow the progress closely and detect errors early.
-
-NOTE: The
-https://platform.openai.com/account/limits[rate limit of your OpenAI account]
-may affect the throughput of the reindexing process. If this happens, change
-`size` to `3` or a similar value in magnitude.
+include::{es-repo-dir}/tab-widgets/inference-api/infer-api-reindex-widget.asciidoc[]
 
 The call returns a task ID to monitor the progress:
 
@@ -214,63 +112,4 @@ provide the query text and the model you have used to create the embeddings.
 NOTE: If you cancelled the reindexing process, you run the query only a part of
 the data which affects the quality of your results.
 
-[source,console]
---------------------------------------------------
-GET openai-embeddings/_search
-{
-  "knn": {
-    "field": "content_embedding",
-    "query_vector_builder": {
-      "text_embedding": {
-        "model_id": "openai_embeddings",
-        "model_text": "Calculate fuel cost"
-      }
-    },
-    "k": 10,
-    "num_candidates": 100
-  },
-  "_source": [
-    "id",
-    "content"
-  ]
-}
---------------------------------------------------
-// TEST[skip:TBD]
-
-As a result, you receive the top 10 documents that are closest in meaning to the
-query from the `openai-embeddings` index sorted by their proximity to the query:
-
-[source,consol-result]
---------------------------------------------------
-"hits": [
-      {
-        "_index": "openai-embeddings",
-        "_id": "DDd5OowBHxQKHyc3TDSC",
-        "_score": 0.83704096,
-        "_source": {
-          "id": 862114,
-          "body": "How to calculate fuel cost for a road trip. By Tara Baukus Mello • Bankrate.com. Dear Driving for Dollars, My family is considering taking a long road trip to finish off the end of the summer, but I'm a little worried about gas prices and our overall fuel cost.It doesn't seem easy to calculate since we'll be traveling through many states and we are considering several routes.y family is considering taking a long road trip to finish off the end of the summer, but I'm a little worried about gas prices and our overall fuel cost. It doesn't seem easy to calculate since we'll be traveling through many states and we are considering several routes."
-        }
-      },
-      {
-        "_index": "openai-embeddings",
-        "_id": "ajd5OowBHxQKHyc3TDSC",
-        "_score": 0.8345704,
-        "_source": {
-          "id": 820622,
-          "body": "Home Heating Calculator. Typically, approximately 50% of the energy consumed in a home annually is for space heating. When deciding on a heating system, many factors will come into play: cost of fuel, installation cost, convenience and life style are all important.This calculator can help you estimate the cost of fuel for different heating appliances.hen deciding on a heating system, many factors will come into play: cost of fuel, installation cost, convenience and life style are all important. This calculator can help you estimate the cost of fuel for different heating appliances."
-        }
-      },
-      {
-        "_index": "openai-embeddings",
-        "_id": "Djd5OowBHxQKHyc3TDSC",
-        "_score": 0.8327426,
-        "_source": {
-          "id": 8202683,
-          "body": "Fuel is another important cost. This cost will depend on your boat, how far you travel, and how fast you travel. A 33-foot sailboat traveling at 7 knots should be able to travel 300 miles on 50 gallons of diesel fuel.If you are paying $4 per gallon, the trip would cost you $200.Most boats have much larger gas tanks than cars.uel is another important cost. This cost will depend on your boat, how far you travel, and how fast you travel. A 33-foot sailboat traveling at 7 knots should be able to travel 300 miles on 50 gallons of diesel fuel."
-        }
-      },
-      (...)
-    ]
---------------------------------------------------
-// NOTCONSOLE
+include::{es-repo-dir}/tab-widgets/inference-api/infer-api-search-widget.asciidoc[]