diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/ValuesSourceReaderBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/ValuesSourceReaderBenchmark.java index 9fa876a00c35c..40edc0b8b9b7f 100644 --- a/benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/ValuesSourceReaderBenchmark.java +++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/ValuesSourceReaderBenchmark.java @@ -8,8 +8,11 @@ package org.elasticsearch.benchmark.compute.operator; +import org.apache.lucene.document.FieldType; import org.apache.lucene.document.NumericDocValuesField; +import org.apache.lucene.document.StoredField; import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexWriterConfig; @@ -19,6 +22,7 @@ import org.apache.lucene.store.Directory; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.NumericUtils; +import org.elasticsearch.common.lucene.Lucene; import org.elasticsearch.compute.data.BlockFactory; import org.elasticsearch.compute.data.BytesRefBlock; import org.elasticsearch.compute.data.BytesRefVector; @@ -30,14 +34,16 @@ import org.elasticsearch.compute.data.LongBlock; import org.elasticsearch.compute.data.LongVector; import org.elasticsearch.compute.data.Page; -import org.elasticsearch.compute.lucene.BlockReaderFactories; import org.elasticsearch.compute.lucene.LuceneSourceOperator; import org.elasticsearch.compute.lucene.ValuesSourceReaderOperator; import org.elasticsearch.compute.operator.topn.TopNOperator; import org.elasticsearch.core.IOUtils; +import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.mapper.BlockLoader; import org.elasticsearch.index.mapper.KeywordFieldMapper; +import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.NumberFieldMapper; +import org.elasticsearch.search.lookup.SearchLookup; import org.openjdk.jmh.annotations.Benchmark; import org.openjdk.jmh.annotations.BenchmarkMode; import org.openjdk.jmh.annotations.Fork; @@ -56,7 +62,9 @@ import java.util.ArrayList; import java.util.Iterator; import java.util.List; +import java.util.Map; import java.util.PrimitiveIterator; +import java.util.Set; import java.util.concurrent.TimeUnit; import java.util.stream.IntStream; @@ -93,18 +101,113 @@ public class ValuesSourceReaderBenchmark { } } - private static BlockLoader blockLoader(String name) { + private static List fields(String name) { return switch (name) { - case "long" -> numericBlockLoader(name, NumberFieldMapper.NumberType.LONG); - case "int" -> numericBlockLoader(name, NumberFieldMapper.NumberType.INTEGER); - case "double" -> numericBlockLoader(name, NumberFieldMapper.NumberType.DOUBLE); - case "keyword" -> new KeywordFieldMapper.KeywordFieldType(name).blockLoader(null); - default -> throw new IllegalArgumentException("can't read [" + name + "]"); + case "3_stored_keywords" -> List.of( + new ValuesSourceReaderOperator.FieldInfo("keyword_1", List.of(blockLoader("stored_keyword_1"))), + new ValuesSourceReaderOperator.FieldInfo("keyword_2", List.of(blockLoader("stored_keyword_2"))), + new ValuesSourceReaderOperator.FieldInfo("keyword_3", List.of(blockLoader("stored_keyword_3"))) + ); + default -> List.of(new ValuesSourceReaderOperator.FieldInfo(name, List.of(blockLoader(name)))); }; } - private static BlockLoader numericBlockLoader(String name, NumberFieldMapper.NumberType numberType) { - return new NumberFieldMapper.NumberFieldType(name, numberType).blockLoader(null); + enum Where { + DOC_VALUES, + SOURCE, + STORED; + } + + private static BlockLoader blockLoader(String name) { + Where where = Where.DOC_VALUES; + if (name.startsWith("stored_")) { + name = name.substring("stored_".length()); + where = Where.STORED; + } else if (name.startsWith("source_")) { + name = name.substring("source_".length()); + where = Where.SOURCE; + } + switch (name) { + case "long": + return numericBlockLoader(name, where, NumberFieldMapper.NumberType.LONG); + case "int": + return numericBlockLoader(name, where, NumberFieldMapper.NumberType.INTEGER); + case "double": + return numericBlockLoader(name, where, NumberFieldMapper.NumberType.DOUBLE); + case "keyword": + name = "keyword_1"; + } + if (name.startsWith("keyword")) { + boolean syntheticSource = false; + FieldType ft = new FieldType(KeywordFieldMapper.Defaults.FIELD_TYPE); + switch (where) { + case DOC_VALUES: + break; + case SOURCE: + ft.setDocValuesType(DocValuesType.NONE); + break; + case STORED: + ft.setStored(true); + ft.setDocValuesType(DocValuesType.NONE); + syntheticSource = true; + break; + } + ft.freeze(); + return new KeywordFieldMapper.KeywordFieldType( + name, + ft, + Lucene.KEYWORD_ANALYZER, + Lucene.KEYWORD_ANALYZER, + Lucene.KEYWORD_ANALYZER, + new KeywordFieldMapper.Builder(name, IndexVersion.current()).docValues(ft.docValuesType() != DocValuesType.NONE), + syntheticSource + ).blockLoader(new MappedFieldType.BlockLoaderContext() { + @Override + public String indexName() { + return "benchmark"; + } + + @Override + public SearchLookup lookup() { + throw new UnsupportedOperationException(); + } + + @Override + public Set sourcePaths(String name) { + return Set.of(name); + } + }); + } + throw new IllegalArgumentException("can't read [" + name + "]"); + } + + private static BlockLoader numericBlockLoader(String name, Where where, NumberFieldMapper.NumberType numberType) { + boolean stored = false; + boolean docValues = true; + switch (where) { + case DOC_VALUES: + break; + case SOURCE: + stored = true; + docValues = false; + break; + case STORED: + throw new UnsupportedOperationException(); + } + return new NumberFieldMapper.NumberFieldType( + name, + numberType, + true, + stored, + docValues, + true, + null, + Map.of(), + null, + false, + null, + null + ).blockLoader(null); } /** @@ -122,7 +225,7 @@ private static BlockLoader numericBlockLoader(String name, NumberFieldMapper.Num @Param({ "in_order", "shuffled", "shuffled_singles" }) public String layout; - @Param({ "long", "int", "double", "keyword" }) + @Param({ "long", "int", "double", "keyword", "stored_keyword", "3_stored_keywords" }) public String name; private Directory directory; @@ -134,9 +237,9 @@ private static BlockLoader numericBlockLoader(String name, NumberFieldMapper.Num public void benchmark() { ValuesSourceReaderOperator op = new ValuesSourceReaderOperator( BlockFactory.getNonBreakingInstance(), - List.of(BlockReaderFactories.loaderToFactory(reader, blockLoader(name))), - 0, - name + fields(name), + List.of(reader), + 0 ); long sum = 0; for (Page page : pages) { @@ -160,7 +263,7 @@ public void benchmark() { sum += (long) values.getDouble(p); } } - case "keyword" -> { + case "keyword", "stored_keyword" -> { BytesRef scratch = new BytesRef(); BytesRefVector values = op.getOutput().getBlock(1).asVector(); for (int p = 0; p < values.getPositionCount(); p++) { @@ -170,21 +273,59 @@ public void benchmark() { sum += Integer.parseInt(r.utf8ToString()); } } + case "3_stored_keywords" -> { + BytesRef scratch = new BytesRef(); + Page out = op.getOutput(); + for (BytesRefVector values : new BytesRefVector[] { + out.getBlock(1).asVector(), + out.getBlock(2).asVector(), + out.getBlock(3).asVector() }) { + + for (int p = 0; p < values.getPositionCount(); p++) { + BytesRef r = values.getBytesRef(p, scratch); + r.offset++; + r.length--; + sum += Integer.parseInt(r.utf8ToString()); + } + } + } } } - long expected; - if (name.equals("keyword")) { - expected = 0; - for (int i = 0; i < INDEX_SIZE; i++) { - expected += i % 1000; - } - } else { - expected = INDEX_SIZE; - expected = expected * (expected - 1) / 2; + long expected = 0; + switch (name) { + case "keyword", "stored_keyword": + for (int i = 0; i < INDEX_SIZE; i++) { + expected += i % 1000; + } + break; + case "3_stored_keywords": + for (int i = 0; i < INDEX_SIZE; i++) { + expected += 3 * (i % 1000); + } + break; + default: + expected = INDEX_SIZE; + expected = expected * (expected - 1) / 2; } if (expected != sum) { throw new AssertionError("[" + layout + "][" + name + "] expected [" + expected + "] but was [" + sum + "]"); } + boolean foundStoredFieldLoader = false; + ValuesSourceReaderOperator.Status status = (ValuesSourceReaderOperator.Status) op.status(); + for (Map.Entry e : status.readersBuilt().entrySet()) { + if (e.getKey().indexOf("stored_fields") >= 0) { + foundStoredFieldLoader = true; + } + } + if (name.indexOf("stored") >= 0) { + if (foundStoredFieldLoader == false) { + throw new AssertionError("expected to use a stored field loader but only had: " + status.readersBuilt()); + } + } else { + if (foundStoredFieldLoader) { + throw new AssertionError("expected not to use a stored field loader but only had: " + status.readersBuilt()); + } + } } @Setup @@ -195,15 +336,23 @@ public void setup() throws IOException { private void setupIndex() throws IOException { directory = new ByteBuffersDirectory(); + FieldType keywordFieldType = new FieldType(KeywordFieldMapper.Defaults.FIELD_TYPE); + keywordFieldType.setStored(true); + keywordFieldType.freeze(); try (IndexWriter iw = new IndexWriter(directory, new IndexWriterConfig().setMergePolicy(NoMergePolicy.INSTANCE))) { for (int i = 0; i < INDEX_SIZE; i++) { String c = Character.toString('a' - ((i % 1000) % 26) + 26); iw.addDocument( List.of( new NumericDocValuesField("long", i), + new StoredField("long", i), new NumericDocValuesField("int", i), + new StoredField("int", i), new NumericDocValuesField("double", NumericUtils.doubleToSortableLong(i)), - new KeywordFieldMapper.KeywordField("keyword", new BytesRef(c + i % 1000), KeywordFieldMapper.Defaults.FIELD_TYPE) + new StoredField("double", (double) i), + new KeywordFieldMapper.KeywordField("keyword_1", new BytesRef(c + i % 1000), keywordFieldType), + new KeywordFieldMapper.KeywordField("keyword_2", new BytesRef(c + i % 1000), keywordFieldType), + new KeywordFieldMapper.KeywordField("keyword_3", new BytesRef(c + i % 1000), keywordFieldType) ) ); if (i % COMMIT_INTERVAL == 0) { diff --git a/client/rest/src/main/java/org/elasticsearch/client/RestClient.java b/client/rest/src/main/java/org/elasticsearch/client/RestClient.java index 7154a2be5bbd8..ed087bef0ac76 100644 --- a/client/rest/src/main/java/org/elasticsearch/client/RestClient.java +++ b/client/rest/src/main/java/org/elasticsearch/client/RestClient.java @@ -87,6 +87,7 @@ import static java.nio.charset.StandardCharsets.UTF_8; import static java.util.Collections.singletonList; +import static org.elasticsearch.client.RestClient.IGNORE_RESPONSE_CODES_PARAM; /** * Client that connects to an Elasticsearch cluster through HTTP. @@ -106,6 +107,9 @@ * Requests can be traced by enabling trace logging for "tracer". The trace logger outputs requests and responses in curl format. */ public class RestClient implements Closeable { + + public static final String IGNORE_RESPONSE_CODES_PARAM = "ignore"; + private static final Log logger = LogFactory.getLog(RestClient.class); private final CloseableHttpAsyncClient client; @@ -780,8 +784,8 @@ private class InternalRequest { this.request = request; Map params = new HashMap<>(request.getParameters()); params.putAll(request.getOptions().getParameters()); - // ignore is a special parameter supported by the clients, shouldn't be sent to es - String ignoreString = params.remove("ignore"); + // IGNORE_RESPONSE_CODES_PARAM is a special parameter supported by the clients, shouldn't be sent to es + String ignoreString = params.remove(IGNORE_RESPONSE_CODES_PARAM); this.ignoreErrorCodes = getIgnoreErrorCodes(ignoreString, request.getMethod()); URI uri = buildUri(pathPrefix, request.getEndpoint(), params); this.httpRequest = createHttpRequest(request.getMethod(), uri, request.getEntity(), compressionEnabled); diff --git a/client/rest/src/test/java/org/elasticsearch/client/RestClientSingleHostTests.java b/client/rest/src/test/java/org/elasticsearch/client/RestClientSingleHostTests.java index a1c4d3fab076a..10d24242ae620 100644 --- a/client/rest/src/test/java/org/elasticsearch/client/RestClientSingleHostTests.java +++ b/client/rest/src/test/java/org/elasticsearch/client/RestClientSingleHostTests.java @@ -275,6 +275,7 @@ public void testErrorStatusCodes() throws Exception { try { Request request = new Request(method, "/" + errorStatusCode); if (false == ignoreParam.isEmpty()) { + // literal "ignore" rather than IGNORE_RESPONSE_CODES_PARAM since this is something on which callers might rely request.addParameter("ignore", ignoreParam); } Response response = restClient.performRequest(request); @@ -568,6 +569,7 @@ private HttpUriRequest performRandomRequest(String method) throws Exception { if (randomBoolean()) { ignore += "," + randomFrom(RestClientTestUtil.getAllErrorStatusCodes()); } + // literal "ignore" rather than IGNORE_RESPONSE_CODES_PARAM since this is something on which callers might rely request.addParameter("ignore", ignore); } URI uri = uriBuilder.build(); diff --git a/docs/changelog/100408.yaml b/docs/changelog/100408.yaml new file mode 100644 index 0000000000000..275c3b4a0de48 --- /dev/null +++ b/docs/changelog/100408.yaml @@ -0,0 +1,5 @@ +pr: 100408 +summary: "ESQL: Make blocks ref counted" +area: ES|QL +type: enhancement +issues: [] diff --git a/docs/changelog/101845.yaml b/docs/changelog/101845.yaml new file mode 100644 index 0000000000000..0dd95bdabca57 --- /dev/null +++ b/docs/changelog/101845.yaml @@ -0,0 +1,5 @@ +pr: 101845 +summary: Introduce new endpoint to expose data stream lifecycle stats +area: Data streams +type: enhancement +issues: [] diff --git a/docs/changelog/102138.yaml b/docs/changelog/102138.yaml new file mode 100644 index 0000000000000..3819e3201150e --- /dev/null +++ b/docs/changelog/102138.yaml @@ -0,0 +1,5 @@ +pr: 102138 +summary: Skip shards that don't match the source query during checkpointing +area: Transform +type: enhancement +issues: [] diff --git a/docs/changelog/102192.yaml b/docs/changelog/102192.yaml new file mode 100644 index 0000000000000..531aa943c9e36 --- /dev/null +++ b/docs/changelog/102192.yaml @@ -0,0 +1,5 @@ +pr: 102192 +summary: "ESQL: Load more than one field at once" +area: ES|QL +type: enhancement +issues: [] diff --git a/docs/changelog/102282.yaml b/docs/changelog/102282.yaml new file mode 100644 index 0000000000000..4860d70f99ccc --- /dev/null +++ b/docs/changelog/102282.yaml @@ -0,0 +1,6 @@ +pr: 102282 +summary: "ES|QL: Fix drop of renamed grouping" +area: ES|QL +type: bug +issues: + - 102121 diff --git a/docs/changelog/102292.yaml b/docs/changelog/102292.yaml new file mode 100644 index 0000000000000..953c3ffdf6150 --- /dev/null +++ b/docs/changelog/102292.yaml @@ -0,0 +1,5 @@ +pr: 102292 +summary: Consider duplicate stacktraces in custom index +area: Application +type: enhancement +issues: [] diff --git a/docs/changelog/102350.yaml b/docs/changelog/102350.yaml new file mode 100644 index 0000000000000..00a311c5d99f8 --- /dev/null +++ b/docs/changelog/102350.yaml @@ -0,0 +1,6 @@ +pr: 102350 +summary: "ESQL: Fix rare bug with empty string" +area: ES|QL +type: bug +issues: + - 101969 diff --git a/docs/reference/data-streams/data-stream-apis.asciidoc b/docs/reference/data-streams/data-stream-apis.asciidoc index d3580ca4448a7..3c2e703d264ff 100644 --- a/docs/reference/data-streams/data-stream-apis.asciidoc +++ b/docs/reference/data-streams/data-stream-apis.asciidoc @@ -25,6 +25,8 @@ preview:[] preview:[] * <> preview:[] +* <> +preview:[] The following API is available for <>: @@ -55,4 +57,6 @@ include::{es-repo-dir}/data-streams/lifecycle/apis/delete-lifecycle.asciidoc[] include::{es-repo-dir}/data-streams/lifecycle/apis/explain-lifecycle.asciidoc[] +include::{es-repo-dir}/data-streams/lifecycle/apis/get-lifecycle-stats.asciidoc[] + include::{es-repo-dir}/indices/downsample-data-stream.asciidoc[] diff --git a/docs/reference/data-streams/lifecycle/apis/get-lifecycle-stats.asciidoc b/docs/reference/data-streams/lifecycle/apis/get-lifecycle-stats.asciidoc new file mode 100644 index 0000000000000..6fa82dc2a810c --- /dev/null +++ b/docs/reference/data-streams/lifecycle/apis/get-lifecycle-stats.asciidoc @@ -0,0 +1,93 @@ +[[data-streams-get-lifecycle-stats]] +=== Get data stream lifecycle stats +++++ +Get Data Stream Lifecycle +++++ + +preview::[] + +Gets stats about the execution of data stream lifecycle. + +[[get-lifecycle-stats-api-prereqs]] +==== {api-prereq-title} + +* If the {es} {security-features} are enabled, you must have the `monitor` or +`manage` <> to use this API. + +[[data-streams-get-lifecycle-stats-request]] +==== {api-request-title} + +`GET _lifecycle/stats` + +[[data-streams-get-lifecycle-stats-desc]] +==== {api-description-title} + +Gets stats about the execution of the data stream lifecycle. The data stream level stats include only stats about data streams +managed by the data stream lifecycle. + +[[get-lifecycle-stats-api-response-body]] +==== {api-response-body-title} + +`last_run_duration_in_millis`:: +(Optional, long) +The duration of the last data stream lifecycle execution. +`time_between_starts_in_millis`:: +(Optional, long) +The time passed between the start of the last two data stream lifecycle executions. This should amount approximately to +<>. +`data_stream_count`:: +(integer) +The count of data streams currently being managed by the data stream lifecycle. +`data_streams`:: +(array of objects) +Contains information about the retrieved data stream lifecycles. ++ +.Properties of objects in `data_streams` +[%collapsible%open] +==== +`name`:: +(string) +The name of the data stream. +`backing_indices_in_total`:: +(integer) +The count of the backing indices of this data stream that are managed by the data stream lifecycle. +`backing_indices_in_error`:: +(integer) +The count of the backing indices of this data stream that are managed by the data stream lifecycle and have encountered an error. +==== + +[[data-streams-get-lifecycle-stats-example]] +==== {api-examples-title} + +Let's retrieve the data stream lifecycle stats of a cluster that has already executed the lifecycle more than once: + +[source,console] +-------------------------------------------------- +GET _lifecycle/stats?human&pretty +-------------------------------------------------- +// TEST[skip:this is for demonstration purposes only, we cannot ensure that DSL has run] + +The response will look like the following: + +[source,console-result] +-------------------------------------------------- +{ + "last_run_duration_in_millis": 2, + "last_run_duration": "2ms", + "time_between_starts_in_millis": 9998, + "time_between_starts": "9.99s", + "data_streams_count": 2, + "data_streams": [ + { + "name": "my-data-stream", + "backing_indices_in_total": 2, + "backing_indices_in_error": 0 + }, + { + "name": "my-other-stream", + "backing_indices_in_total": 2, + "backing_indices_in_error": 1 + } + ] +} +-------------------------------------------------- \ No newline at end of file diff --git a/docs/reference/query-dsl/text-expansion-query.asciidoc b/docs/reference/query-dsl/text-expansion-query.asciidoc index d15fd40846529..e924cc05376d9 100644 --- a/docs/reference/query-dsl/text-expansion-query.asciidoc +++ b/docs/reference/query-dsl/text-expansion-query.asciidoc @@ -78,29 +78,7 @@ GET my-index/_search ---- // TEST[skip: TBD] -[discrete] -[[optimizing-text-expansion]] -=== Optimizing the search performance of the text_expansion query - -https://www.elastic.co/blog/faster-retrieval-of-top-hits-in-elasticsearch-with-block-max-wand[Max WAND] -is an optimization technique used by {es} to skip documents that cannot score -competitively against the current best matching documents. However, the tokens -generated by the ELSER model don't work well with the Max WAND optimization. -Consequently, enabling Max WAND can actually increase query latency for -`text_expansion`. For datasets of a significant size, disabling Max -WAND leads to lower query latencies. - -Max WAND is controlled by the -<> query parameter. Setting track_total_hits -to true forces {es} to consider all documents, resulting in lower query -latencies for the `text_expansion` query. However, other {es} queries run slower -when Max WAND is disabled. - -If you are combining the `text_expansion` query with standard text queries in a -compound search, it is recommended to measure the query performance before -deciding which setting to use. - -NOTE: The `track_total_hits` option applies to all queries in the search request -and may be optimal for some queries but not for others. Take into account the -characteristics of all your queries to determine the most suitable -configuration. +[NOTE] +==== +Depending on your data, the text expansion query may be faster with `track_total_hits: false`. +==== diff --git a/docs/reference/search/search-your-data/search-api.asciidoc b/docs/reference/search/search-your-data/search-api.asciidoc index f3e271918b9b2..496812a0cedb4 100644 --- a/docs/reference/search/search-your-data/search-api.asciidoc +++ b/docs/reference/search/search-your-data/search-api.asciidoc @@ -440,6 +440,17 @@ GET my-index-000001/_search Finally you can force an accurate count by setting `"track_total_hits"` to `true` in the request. +[TIP] +========================================= +The `track_total_hits` parameter allows you to trade hit count accuracy for performance. +In general the lower the value of `track_total_hits` the faster the query will be, +with `false` returning the fastest results. +Setting `track_total_hits` to true will cause {es} to return exact hit counts, which could +hurt query performance because it disables the +https://www.elastic.co/blog/faster-retrieval-of-top-hits-in-elasticsearch-with-block-max-wand[Max WAND] +optimization. +========================================= + [discrete] [[quickly-check-for-matching-docs]] === Quickly check for matching docs diff --git a/docs/reference/search/search-your-data/semantic-search-elser.asciidoc b/docs/reference/search/search-your-data/semantic-search-elser.asciidoc index 164beb221cd4f..0bee9533cd358 100644 --- a/docs/reference/search/search-your-data/semantic-search-elser.asciidoc +++ b/docs/reference/search/search-your-data/semantic-search-elser.asciidoc @@ -45,7 +45,7 @@ you must provide suitably sized nodes yourself. First, the mapping of the destination index - the index that contains the tokens that the model created based on your text - must be created. The destination index must have a field with the -<> or <> field +<> or <> field type to index the ELSER output. NOTE: ELSER output must be ingested into a field with the `sparse_vector` or @@ -72,11 +72,11 @@ PUT my-index } ---- // TEST[skip:TBD] -<1> The name of the field to contain the generated tokens. It must be refrenced +<1> The name of the field to contain the generated tokens. It must be refrenced in the {infer} pipeline configuration in the next step. <2> The field to contain the tokens is a `sparse_vector` field. -<3> The name of the field from which to create the sparse vector representation. -In this example, the name of the field is `content`. It must be referenced in the +<3> The name of the field from which to create the sparse vector representation. +In this example, the name of the field is `content`. It must be referenced in the {infer} pipeline configuration in the next step. <4> The field type which is text in this example. @@ -93,24 +93,24 @@ that is being ingested in the pipeline. [source,console] ---- -PUT _ingest/pipeline/elser-v2-test -{ - "processors": [ - { - "inference": { - "model_id": ".elser_model_2", - "input_output": [ <1> - { - "input_field": "content", - "output_field": "content_embedding" - } - ] - } - } - ] +PUT _ingest/pipeline/elser-v2-test +{ + "processors": [ + { + "inference": { + "model_id": ".elser_model_2", + "input_output": [ <1> + { + "input_field": "content", + "output_field": "content_embedding" + } + ] + } + } + ] } ---- -<1> Configuration object that defines the `input_field` for the {infer} process +<1> Configuration object that defines the `input_field` for the {infer} process and the `output_field` that will contain the {infer} results. //// @@ -137,8 +137,8 @@ https://github.com/elastic/stack-docs/blob/main/docs/en/stack/ml/nlp/data/msmarc Download the file and upload it to your cluster using the {kibana-ref}/connect-to-elasticsearch.html#upload-data-kibana[Data Visualizer] -in the {ml-app} UI. Assign the name `id` to the first column and `content` to -the second column. The index name is `test-data`. Once the upload is complete, +in the {ml-app} UI. Assign the name `id` to the first column and `content` to +the second column. The index name is `test-data`. Once the upload is complete, you can see an index named `test-data` with 182469 documents. @@ -184,9 +184,9 @@ follow the progress. [[text-expansion-query]] ==== Semantic search by using the `text_expansion` query -To perform semantic search, use the `text_expansion` query, and provide the -query text and the ELSER model ID. The example below uses the query text "How to -avoid muscle soreness after running?", the `content_embedding` field contains +To perform semantic search, use the `text_expansion` query, and provide the +query text and the ELSER model ID. The example below uses the query text "How to +avoid muscle soreness after running?", the `content_embedding` field contains the generated ELSER output: [source,console] @@ -208,9 +208,9 @@ GET my-index/_search The result is the top 10 documents that are closest in meaning to your query text from the `my-index` index sorted by their relevancy. The result also contains the extracted tokens for each of the relevant search results with their -weights. Tokens are learned associations capturing relevance, they are not -synonyms. To learn more about what tokens are, refer to -{ml-docs}/ml-nlp-elser.html#elser-tokens[this page]. It is possible to exclude +weights. Tokens are learned associations capturing relevance, they are not +synonyms. To learn more about what tokens are, refer to +{ml-docs}/ml-nlp-elser.html#elser-tokens[this page]. It is possible to exclude tokens from source, refer to <> to learn more. [source,consol-result] @@ -253,9 +253,6 @@ tokens from source, refer to <> to learn more. ---- // NOTCONSOLE -To learn about optimizing your `text_expansion` query, refer to -<>. - [discrete] [[text-expansion-compound-query]] @@ -281,7 +278,7 @@ GET my-index/_search "bool": { <1> "should": [ { - "text_expansion": { + "text_expansion": { "content_embedding": { "model_text": "How to avoid muscle soreness after running?", "model_id": ".elser_model_2", @@ -333,12 +330,12 @@ WARNING: Reindex uses the document source to populate the destination index. space-saving optimsation that should only be applied if you are certain that reindexing will not be required in the future! It's important to carefully consider this trade-off and make sure that excluding the ELSER terms from the -source aligns with your specific requirements and use case. Review the -<> and <> sections carefully to learn +source aligns with your specific requirements and use case. Review the +<> and <> sections carefully to learn more about the possible consequences of excluding the tokens from the `_source`. -The mapping that excludes `content_embedding` from the `_source` field can be -created by the following API call: +The mapping that excludes `content_embedding` from the `_source` field can be +created by the following API call: [source,console] ---- @@ -352,10 +349,10 @@ PUT my-index }, "properties": { "content_embedding": { - "type": "sparse_vector" + "type": "sparse_vector" }, - "content": { - "type": "text" + "content": { + "type": "text" } } } @@ -363,6 +360,10 @@ PUT my-index ---- // TEST[skip:TBD] +[NOTE] +==== +Depending on your data, the text expansion query may be faster with `track_total_hits: false`. +==== [discrete] [[further-reading]] diff --git a/libs/core/src/main/java/org/elasticsearch/core/RefCounted.java b/libs/core/src/main/java/org/elasticsearch/core/RefCounted.java index 0f7dec4968ba7..49c030609951b 100644 --- a/libs/core/src/main/java/org/elasticsearch/core/RefCounted.java +++ b/libs/core/src/main/java/org/elasticsearch/core/RefCounted.java @@ -38,7 +38,7 @@ public interface RefCounted { void incRef(); /** - * Tries to increment the refCount of this instance. This method will return {@code true} iff the refCount was + * Tries to increment the refCount of this instance. This method will return {@code true} iff the refCount was successfully incremented. * * @see #decRef() * @see #incRef() diff --git a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceIT.java b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceIT.java index 5bbc007cfb272..7ac86c8aee614 100644 --- a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceIT.java +++ b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceIT.java @@ -622,35 +622,6 @@ public void testDataLifecycleServiceConfiguresTheMergePolicy() throws Exception }); } - private static List getBackingIndices(String dataStreamName) { - GetDataStreamAction.Request getDataStreamRequest = new GetDataStreamAction.Request(new String[] { dataStreamName }); - GetDataStreamAction.Response getDataStreamResponse = client().execute(GetDataStreamAction.INSTANCE, getDataStreamRequest) - .actionGet(); - assertThat(getDataStreamResponse.getDataStreams().size(), equalTo(1)); - assertThat(getDataStreamResponse.getDataStreams().get(0).getDataStream().getName(), equalTo(dataStreamName)); - return getDataStreamResponse.getDataStreams().get(0).getDataStream().getIndices().stream().map(Index::getName).toList(); - } - - static void indexDocs(String dataStream, int numDocs) { - BulkRequest bulkRequest = new BulkRequest(); - for (int i = 0; i < numDocs; i++) { - String value = DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.formatMillis(System.currentTimeMillis()); - bulkRequest.add( - new IndexRequest(dataStream).opType(DocWriteRequest.OpType.CREATE) - .source(String.format(Locale.ROOT, "{\"%s\":\"%s\"}", DEFAULT_TIMESTAMP_FIELD, value), XContentType.JSON) - ); - } - BulkResponse bulkResponse = client().bulk(bulkRequest).actionGet(); - assertThat(bulkResponse.getItems().length, equalTo(numDocs)); - String backingIndexPrefix = DataStream.BACKING_INDEX_PREFIX + dataStream; - for (BulkItemResponse itemResponse : bulkResponse) { - assertThat(itemResponse.getFailureMessage(), nullValue()); - assertThat(itemResponse.status(), equalTo(RestStatus.CREATED)); - assertThat(itemResponse.getIndex(), startsWith(backingIndexPrefix)); - } - indicesAdmin().refresh(new RefreshRequest(dataStream)).actionGet(); - } - public void testReenableDataStreamLifecycle() throws Exception { // start with a lifecycle that's not enabled DataStreamLifecycle lifecycle = new DataStreamLifecycle(null, null, false); @@ -700,6 +671,35 @@ public void testReenableDataStreamLifecycle() throws Exception { }); } + private static List getBackingIndices(String dataStreamName) { + GetDataStreamAction.Request getDataStreamRequest = new GetDataStreamAction.Request(new String[] { dataStreamName }); + GetDataStreamAction.Response getDataStreamResponse = client().execute(GetDataStreamAction.INSTANCE, getDataStreamRequest) + .actionGet(); + assertThat(getDataStreamResponse.getDataStreams().size(), equalTo(1)); + assertThat(getDataStreamResponse.getDataStreams().get(0).getDataStream().getName(), equalTo(dataStreamName)); + return getDataStreamResponse.getDataStreams().get(0).getDataStream().getIndices().stream().map(Index::getName).toList(); + } + + static void indexDocs(String dataStream, int numDocs) { + BulkRequest bulkRequest = new BulkRequest(); + for (int i = 0; i < numDocs; i++) { + String value = DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.formatMillis(System.currentTimeMillis()); + bulkRequest.add( + new IndexRequest(dataStream).opType(DocWriteRequest.OpType.CREATE) + .source(String.format(Locale.ROOT, "{\"%s\":\"%s\"}", DEFAULT_TIMESTAMP_FIELD, value), XContentType.JSON) + ); + } + BulkResponse bulkResponse = client().bulk(bulkRequest).actionGet(); + assertThat(bulkResponse.getItems().length, equalTo(numDocs)); + String backingIndexPrefix = DataStream.BACKING_INDEX_PREFIX + dataStream; + for (BulkItemResponse itemResponse : bulkResponse) { + assertThat(itemResponse.getFailureMessage(), nullValue()); + assertThat(itemResponse.status(), equalTo(RestStatus.CREATED)); + assertThat(itemResponse.getIndex(), startsWith(backingIndexPrefix)); + } + indicesAdmin().refresh(new RefreshRequest(dataStream)).actionGet(); + } + static void putComposableIndexTemplate( String id, @Nullable String mappings, diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleStatsIT.java b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleStatsIT.java new file mode 100644 index 0000000000000..cce9132d99d19 --- /dev/null +++ b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleStatsIT.java @@ -0,0 +1,86 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ +package org.elasticsearch.datastreams.lifecycle; + +import org.elasticsearch.client.Request; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.datastreams.DisabledSecurityDataStreamTestCase; +import org.junit.After; +import org.junit.Before; + +import java.io.IOException; +import java.util.List; +import java.util.Map; + +import static org.hamcrest.Matchers.greaterThanOrEqualTo; +import static org.hamcrest.Matchers.is; + +public class DataStreamLifecycleStatsIT extends DisabledSecurityDataStreamTestCase { + + @Before + public void updateClusterSettings() throws IOException { + updateClusterSettings( + Settings.builder() + .put("data_streams.lifecycle.poll_interval", "1s") + .put("cluster.lifecycle.default.rollover", "min_docs=1,max_docs=1") + .build() + ); + } + + @After + public void cleanUp() throws IOException { + adminClient().performRequest(new Request("DELETE", "_data_stream/*?expand_wildcards=hidden")); + } + + @SuppressWarnings("unchecked") + public void testStats() throws Exception { + // Check empty stats and wait until we have 2 executions + assertBusy(() -> { + Request request = new Request("GET", "/_lifecycle/stats"); + Map response = entityAsMap(client().performRequest(request)); + assertThat(response.get("data_stream_count"), is(0)); + assertThat(response.get("data_streams"), is(List.of())); + assertThat(response.containsKey("last_run_duration_in_millis"), is(true)); + assertThat(response.containsKey("time_between_starts_in_millis"), is(true)); + }); + + // Create a template + Request putComposableIndexTemplateRequest = new Request("POST", "/_index_template/1"); + putComposableIndexTemplateRequest.setJsonEntity(""" + { + "index_patterns": ["my-data-stream-*"], + "data_stream": {}, + "template": { + "lifecycle": {} + } + } + """); + assertOK(client().performRequest(putComposableIndexTemplateRequest)); + + // Create two data streams with one doc each + Request createDocRequest = new Request("POST", "/my-data-stream-1/_doc?refresh=true"); + createDocRequest.setJsonEntity("{ \"@timestamp\": \"2022-12-12\"}"); + assertOK(client().performRequest(createDocRequest)); + createDocRequest = new Request("POST", "/my-data-stream-2/_doc?refresh=true"); + createDocRequest.setJsonEntity("{ \"@timestamp\": \"2022-12-12\"}"); + assertOK(client().performRequest(createDocRequest)); + + Request request = new Request("GET", "/_lifecycle/stats"); + Map response = entityAsMap(client().performRequest(request)); + assertThat(response.get("data_stream_count"), is(2)); + List> dataStreams = (List>) response.get("data_streams"); + assertThat(dataStreams.get(0).get("name"), is("my-data-stream-1")); + assertThat((Integer) dataStreams.get(0).get("backing_indices_in_total"), greaterThanOrEqualTo(1)); + assertThat((Integer) dataStreams.get(0).get("backing_indices_in_error"), is(0)); + assertThat(dataStreams.get(1).get("name"), is("my-data-stream-2")); + assertThat((Integer) dataStreams.get(1).get("backing_indices_in_total"), greaterThanOrEqualTo(1)); + assertThat((Integer) dataStreams.get(0).get("backing_indices_in_error"), is(0)); + assertThat(response.containsKey("last_run_duration_in_millis"), is(true)); + assertThat(response.containsKey("time_between_starts_in_millis"), is(true)); + } +} diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamsPlugin.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamsPlugin.java index 2cf44dc0e3218..dd8e13cf18408 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamsPlugin.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamsPlugin.java @@ -40,11 +40,14 @@ import org.elasticsearch.datastreams.lifecycle.action.DeleteDataStreamLifecycleAction; import org.elasticsearch.datastreams.lifecycle.action.ExplainDataStreamLifecycleAction; import org.elasticsearch.datastreams.lifecycle.action.GetDataStreamLifecycleAction; +import org.elasticsearch.datastreams.lifecycle.action.GetDataStreamLifecycleStatsAction; import org.elasticsearch.datastreams.lifecycle.action.PutDataStreamLifecycleAction; import org.elasticsearch.datastreams.lifecycle.action.TransportDeleteDataStreamLifecycleAction; import org.elasticsearch.datastreams.lifecycle.action.TransportExplainDataStreamLifecycleAction; import org.elasticsearch.datastreams.lifecycle.action.TransportGetDataStreamLifecycleAction; +import org.elasticsearch.datastreams.lifecycle.action.TransportGetDataStreamLifecycleStatsAction; import org.elasticsearch.datastreams.lifecycle.action.TransportPutDataStreamLifecycleAction; +import org.elasticsearch.datastreams.lifecycle.rest.RestDataStreamLifecycleStatsAction; import org.elasticsearch.datastreams.lifecycle.rest.RestDeleteDataStreamLifecycleAction; import org.elasticsearch.datastreams.lifecycle.rest.RestExplainDataStreamLifecycleAction; import org.elasticsearch.datastreams.lifecycle.rest.RestGetDataStreamLifecycleAction; @@ -189,6 +192,7 @@ public Collection createComponents(PluginServices services) { actions.add(new ActionHandler<>(GetDataStreamLifecycleAction.INSTANCE, TransportGetDataStreamLifecycleAction.class)); actions.add(new ActionHandler<>(DeleteDataStreamLifecycleAction.INSTANCE, TransportDeleteDataStreamLifecycleAction.class)); actions.add(new ActionHandler<>(ExplainDataStreamLifecycleAction.INSTANCE, TransportExplainDataStreamLifecycleAction.class)); + actions.add(new ActionHandler<>(GetDataStreamLifecycleStatsAction.INSTANCE, TransportGetDataStreamLifecycleStatsAction.class)); return actions; } @@ -218,6 +222,7 @@ public List getRestHandlers( handlers.add(new RestGetDataStreamLifecycleAction()); handlers.add(new RestDeleteDataStreamLifecycleAction()); handlers.add(new RestExplainDataStreamLifecycleAction()); + handlers.add(new RestDataStreamLifecycleStatsAction()); return handlers; } diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleErrorStore.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleErrorStore.java index 47589fd7276f4..01ccbdbe3ffec 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleErrorStore.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleErrorStore.java @@ -13,7 +13,7 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.core.Nullable; -import java.util.List; +import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.function.LongSupplier; @@ -87,7 +87,7 @@ public ErrorEntry getError(String indexName) { /** * Return an immutable view (a snapshot) of the tracked indices at the moment this method is called. */ - public List getAllIndices() { - return List.copyOf(indexNameToError.keySet()); + public Set getAllIndices() { + return Set.copyOf(indexNameToError.keySet()); } } diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleService.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleService.java index 03d1340c14dbb..9f9a90704167d 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleService.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleService.java @@ -175,6 +175,13 @@ public class DataStreamLifecycleService implements ClusterStateListener, Closeab */ private volatile int signallingErrorRetryInterval; + /** + * The following stats are tracking how the data stream lifecycle runs are performing time wise + */ + private volatile Long lastRunStartedAt = null; + private volatile Long lastRunDuration = null; + private volatile Long timeBetweenStarts = null; + private static final SimpleBatchedExecutor FORCE_MERGE_STATE_UPDATE_TASK_EXECUTOR = new SimpleBatchedExecutor<>() { @Override @@ -299,6 +306,11 @@ public void triggered(SchedulerEngine.Event event) { */ // default visibility for testing purposes void run(ClusterState state) { + long startTime = nowSupplier.getAsLong(); + if (lastRunStartedAt != null) { + timeBetweenStarts = startTime - lastRunStartedAt; + } + lastRunStartedAt = startTime; int affectedIndices = 0; int affectedDataStreams = 0; for (DataStream dataStream : state.metadata().dataStreams().values()) { @@ -396,8 +408,10 @@ void run(ClusterState state) { affectedIndices += indicesToExcludeForRemainingRun.size(); affectedDataStreams++; } + lastRunDuration = nowSupplier.getAsLong() - lastRunStartedAt; logger.trace( - "Data stream lifecycle service performed operations on [{}] indices, part of [{}] data streams", + "Data stream lifecycle service run for {} and performed operations on [{}] indices, part of [{}] data streams", + TimeValue.timeValueMillis(lastRunDuration).toHumanReadableString(2), affectedIndices, affectedDataStreams ); @@ -1193,6 +1207,22 @@ static TimeValue getRetentionConfiguration(DataStream dataStream) { return dataStream.getLifecycle().getEffectiveDataRetention(); } + /** + * @return the duration of the last run in millis or null if the service hasn't completed a run yet. + */ + @Nullable + public Long getLastRunDuration() { + return lastRunDuration; + } + + /** + * @return the time passed between the start times of the last two consecutive runs or null if the service hasn't started twice yet. + */ + @Nullable + public Long getTimeBetweenStarts() { + return timeBetweenStarts; + } + /** * Action listener that records the encountered failure using the provided recordError callback for the * provided target index. If the listener is notified of success it will clear the recorded entry for the provided diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/GetDataStreamLifecycleStatsAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/GetDataStreamLifecycleStatsAction.java new file mode 100644 index 0000000000000..c3444a67b847c --- /dev/null +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/GetDataStreamLifecycleStatsAction.java @@ -0,0 +1,154 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ +package org.elasticsearch.datastreams.lifecycle.action; + +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.action.ActionType; +import org.elasticsearch.action.support.master.MasterNodeReadRequest; +import org.elasticsearch.common.collect.Iterators; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.ChunkedToXContentObject; +import org.elasticsearch.core.Nullable; +import org.elasticsearch.core.TimeValue; +import org.elasticsearch.xcontent.ToXContent; + +import java.io.IOException; +import java.util.Iterator; +import java.util.List; +import java.util.Objects; + +/** + * This action retrieves the data stream lifecycle stats from the master node. + */ +public class GetDataStreamLifecycleStatsAction extends ActionType { + + public static final GetDataStreamLifecycleStatsAction INSTANCE = new GetDataStreamLifecycleStatsAction(); + public static final String NAME = "cluster:monitor/data_stream/lifecycle/stats"; + + private GetDataStreamLifecycleStatsAction() { + super(NAME, Response::new); + } + + public static class Request extends MasterNodeReadRequest { + + public Request(StreamInput in) throws IOException { + super(in); + } + + public Request() {} + + @Override + public ActionRequestValidationException validate() { + return null; + } + } + + public static class Response extends ActionResponse implements ChunkedToXContentObject { + + private final Long runDuration; + private final Long timeBetweenStarts; + private final List dataStreamStats; + + public Response(@Nullable Long runDuration, @Nullable Long timeBetweenStarts, List dataStreamStats) { + this.runDuration = runDuration; + this.timeBetweenStarts = timeBetweenStarts; + this.dataStreamStats = dataStreamStats; + } + + public Response(StreamInput in) throws IOException { + super(in); + this.runDuration = in.readOptionalVLong(); + this.timeBetweenStarts = in.readOptionalVLong(); + this.dataStreamStats = in.readCollectionAsImmutableList(DataStreamStats::read); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeOptionalVLong(runDuration); + out.writeOptionalVLong(timeBetweenStarts); + out.writeCollection(dataStreamStats, (o, v) -> v.writeTo(o)); + } + + public Long getRunDuration() { + return runDuration; + } + + public Long getTimeBetweenStarts() { + return timeBetweenStarts; + } + + public List getDataStreamStats() { + return dataStreamStats; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Response other = (Response) o; + return Objects.equals(runDuration, other.runDuration) + && Objects.equals(timeBetweenStarts, other.timeBetweenStarts) + && Objects.equals(dataStreamStats, other.dataStreamStats); + } + + @Override + public int hashCode() { + return Objects.hash(runDuration, timeBetweenStarts, dataStreamStats); + } + + @Override + public Iterator toXContentChunked(ToXContent.Params outerParams) { + return Iterators.concat(Iterators.single((builder, params) -> { + builder.startObject(); + if (runDuration != null) { + builder.field("last_run_duration_in_millis", runDuration); + if (builder.humanReadable()) { + builder.field("last_run_duration", TimeValue.timeValueMillis(runDuration).toHumanReadableString(2)); + } + } + if (timeBetweenStarts != null) { + builder.field("time_between_starts_in_millis", timeBetweenStarts); + if (builder.humanReadable()) { + builder.field("time_between_starts", TimeValue.timeValueMillis(timeBetweenStarts).toHumanReadableString(2)); + } + } + builder.field("data_stream_count", dataStreamStats.size()); + builder.startArray("data_streams"); + return builder; + }), Iterators.map(dataStreamStats.iterator(), stat -> (builder, params) -> { + builder.startObject(); + builder.field("name", stat.dataStreamName); + builder.field("backing_indices_in_total", stat.backingIndicesInTotal); + builder.field("backing_indices_in_error", stat.backingIndicesInError); + builder.endObject(); + return builder; + }), Iterators.single((builder, params) -> { + builder.endArray(); + builder.endObject(); + return builder; + })); + } + + public record DataStreamStats(String dataStreamName, int backingIndicesInTotal, int backingIndicesInError) implements Writeable { + + public static DataStreamStats read(StreamInput in) throws IOException { + return new DataStreamStats(in.readString(), in.readVInt(), in.readVInt()); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(dataStreamName); + out.writeVInt(backingIndicesInTotal); + out.writeVInt(backingIndicesInError); + } + } + } +} diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/TransportGetDataStreamLifecycleStatsAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/TransportGetDataStreamLifecycleStatsAction.java new file mode 100644 index 0000000000000..03bc1d129eaba --- /dev/null +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/TransportGetDataStreamLifecycleStatsAction.java @@ -0,0 +1,110 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ +package org.elasticsearch.datastreams.lifecycle.action; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.master.TransportMasterNodeReadAction; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.block.ClusterBlockException; +import org.elasticsearch.cluster.block.ClusterBlockLevel; +import org.elasticsearch.cluster.metadata.DataStream; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.metadata.Metadata; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.util.concurrent.EsExecutors; +import org.elasticsearch.datastreams.lifecycle.DataStreamLifecycleService; +import org.elasticsearch.index.Index; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportService; + +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; +import java.util.Set; + +/** + * Exposes stats about the latest lifecycle run and the error store. + */ +public class TransportGetDataStreamLifecycleStatsAction extends TransportMasterNodeReadAction< + GetDataStreamLifecycleStatsAction.Request, + GetDataStreamLifecycleStatsAction.Response> { + + private final DataStreamLifecycleService lifecycleService; + + @Inject + public TransportGetDataStreamLifecycleStatsAction( + TransportService transportService, + ClusterService clusterService, + ThreadPool threadPool, + ActionFilters actionFilters, + IndexNameExpressionResolver indexNameExpressionResolver, + DataStreamLifecycleService lifecycleService + ) { + super( + GetDataStreamLifecycleStatsAction.NAME, + transportService, + clusterService, + threadPool, + actionFilters, + GetDataStreamLifecycleStatsAction.Request::new, + indexNameExpressionResolver, + GetDataStreamLifecycleStatsAction.Response::new, + EsExecutors.DIRECT_EXECUTOR_SERVICE + ); + this.lifecycleService = lifecycleService; + } + + @Override + protected void masterOperation( + Task task, + GetDataStreamLifecycleStatsAction.Request request, + ClusterState state, + ActionListener listener + ) throws Exception { + listener.onResponse(collectStats(state)); + } + + // Visible for testing + GetDataStreamLifecycleStatsAction.Response collectStats(ClusterState state) { + Metadata metadata = state.metadata(); + Set indicesInErrorStore = lifecycleService.getErrorStore().getAllIndices(); + List dataStreamStats = new ArrayList<>(); + for (DataStream dataStream : state.metadata().dataStreams().values()) { + if (dataStream.getLifecycle() != null && dataStream.getLifecycle().isEnabled()) { + int total = 0; + int inError = 0; + for (Index index : dataStream.getIndices()) { + if (dataStream.isIndexManagedByDataStreamLifecycle(index, metadata::index)) { + total++; + if (indicesInErrorStore.contains(index.getName())) { + inError++; + } + } + } + dataStreamStats.add(new GetDataStreamLifecycleStatsAction.Response.DataStreamStats(dataStream.getName(), total, inError)); + } + } + return new GetDataStreamLifecycleStatsAction.Response( + lifecycleService.getLastRunDuration(), + lifecycleService.getTimeBetweenStarts(), + dataStreamStats.isEmpty() + ? dataStreamStats + : dataStreamStats.stream() + .sorted(Comparator.comparing(GetDataStreamLifecycleStatsAction.Response.DataStreamStats::dataStreamName)) + .toList() + ); + } + + @Override + protected ClusterBlockException checkBlock(GetDataStreamLifecycleStatsAction.Request request, ClusterState state) { + return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_READ); + } +} diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/rest/RestDataStreamLifecycleStatsAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/rest/RestDataStreamLifecycleStatsAction.java new file mode 100644 index 0000000000000..2daff2a05940c --- /dev/null +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/rest/RestDataStreamLifecycleStatsAction.java @@ -0,0 +1,45 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.datastreams.lifecycle.rest; + +import org.elasticsearch.client.internal.node.NodeClient; +import org.elasticsearch.datastreams.lifecycle.action.GetDataStreamLifecycleStatsAction; +import org.elasticsearch.rest.BaseRestHandler; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.Scope; +import org.elasticsearch.rest.ServerlessScope; +import org.elasticsearch.rest.action.RestChunkedToXContentListener; + +import java.util.List; + +import static org.elasticsearch.rest.RestRequest.Method.GET; + +@ServerlessScope(Scope.PUBLIC) +public class RestDataStreamLifecycleStatsAction extends BaseRestHandler { + + @Override + public String getName() { + return "data_stream_lifecycle_stats_action"; + } + + @Override + public List routes() { + return List.of(new Route(GET, "/_lifecycle/stats")); + } + + @Override + protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) { + String masterNodeTimeout = restRequest.param("master_timeout"); + GetDataStreamLifecycleStatsAction.Request request = new GetDataStreamLifecycleStatsAction.Request(); + if (masterNodeTimeout != null) { + request.masterNodeTimeout(masterNodeTimeout); + } + return channel -> client.execute(GetDataStreamLifecycleStatsAction.INSTANCE, request, new RestChunkedToXContentListener<>(channel)); + } +} diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleErrorStoreTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleErrorStoreTests.java index c1255cc9e3a72..9f1928374eb5f 100644 --- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleErrorStoreTests.java +++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleErrorStoreTests.java @@ -12,12 +12,13 @@ import org.elasticsearch.test.ESTestCase; import org.junit.Before; -import java.util.List; +import java.util.Set; import java.util.stream.Stream; import static org.elasticsearch.datastreams.lifecycle.DataStreamLifecycleErrorStore.MAX_ERROR_MESSAGE_LENGTH; import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.hasItem; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.nullValue; @@ -36,7 +37,7 @@ public void testRecordAndRetrieveError() { assertThat(existingRecordedError, is(nullValue())); assertThat(errorStore.getError("test"), is(notNullValue())); assertThat(errorStore.getAllIndices().size(), is(1)); - assertThat(errorStore.getAllIndices().get(0), is("test")); + assertThat(errorStore.getAllIndices(), hasItem("test")); existingRecordedError = errorStore.recordError("test", new IllegalStateException("bad state")); assertThat(existingRecordedError, is(notNullValue())); @@ -51,7 +52,7 @@ public void testRetrieveAfterClear() { public void testGetAllIndicesIsASnapshotViewOfTheStore() { Stream.iterate(0, i -> i + 1).limit(5).forEach(i -> errorStore.recordError("test" + i, new NullPointerException("testing"))); - List initialAllIndices = errorStore.getAllIndices(); + Set initialAllIndices = errorStore.getAllIndices(); assertThat(initialAllIndices.size(), is(5)); assertThat( initialAllIndices, diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceTests.java index 6e22f835c397e..2445e6b0d72df 100644 --- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceTests.java +++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceTests.java @@ -94,6 +94,7 @@ import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; import static org.elasticsearch.cluster.metadata.IndexMetadata.APIBlock.WRITE; @@ -119,6 +120,7 @@ import static org.hamcrest.Matchers.lessThanOrEqualTo; import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.nullValue; +import static org.mockito.Mockito.mock; public class DataStreamLifecycleServiceTests extends ESTestCase { @@ -1378,6 +1380,31 @@ public void testTimeSeriesIndicesStillWithinTimeBounds() { } } + public void testTrackingTimeStats() { + AtomicLong now = new AtomicLong(0); + long delta = randomLongBetween(10, 10000); + DataStreamLifecycleService service = new DataStreamLifecycleService( + Settings.EMPTY, + getTransportRequestsRecordingClient(), + clusterService, + Clock.systemUTC(), + threadPool, + () -> now.getAndAdd(delta), + new DataStreamLifecycleErrorStore(() -> Clock.systemUTC().millis()), + mock(AllocationService.class) + ); + assertThat(service.getLastRunDuration(), is(nullValue())); + assertThat(service.getTimeBetweenStarts(), is(nullValue())); + + service.run(ClusterState.EMPTY_STATE); + assertThat(service.getLastRunDuration(), is(delta)); + assertThat(service.getTimeBetweenStarts(), is(nullValue())); + + service.run(ClusterState.EMPTY_STATE); + assertThat(service.getLastRunDuration(), is(delta)); + assertThat(service.getTimeBetweenStarts(), is(2 * delta)); + } + /* * Creates a test cluster state with the given indexName. If customDataStreamLifecycleMetadata is not null, it is added as the value * of the index's custom metadata named "data_stream_lifecycle". diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/action/DataStreamLifecycleStatsResponseTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/action/DataStreamLifecycleStatsResponseTests.java new file mode 100644 index 0000000000000..7242f72fe9f68 --- /dev/null +++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/action/DataStreamLifecycleStatsResponseTests.java @@ -0,0 +1,156 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.datastreams.lifecycle.action; + +import org.apache.lucene.tests.util.LuceneTestCase; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.core.TimeValue; +import org.elasticsearch.test.AbstractWireSerializingTestCase; +import org.elasticsearch.xcontent.ToXContent; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.XContentType; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.stream.IntStream; + +import static org.elasticsearch.xcontent.ToXContent.EMPTY_PARAMS; +import static org.hamcrest.Matchers.is; + +@LuceneTestCase.AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/102373") +public class DataStreamLifecycleStatsResponseTests extends AbstractWireSerializingTestCase { + + @Override + protected GetDataStreamLifecycleStatsAction.Response createTestInstance() { + boolean hasRun = usually(); + var runDuration = hasRun ? randomLongBetween(10, 100000000) : null; + var timeBetweenStarts = hasRun && usually() ? randomLongBetween(10, 100000000) : null; + var dataStreams = IntStream.range(0, randomInt(10)) + .mapToObj( + ignored -> new GetDataStreamLifecycleStatsAction.Response.DataStreamStats( + randomAlphaOfLength(10), + randomIntBetween(1, 1000), + randomIntBetween(0, 100) + ) + ) + .toList(); + return new GetDataStreamLifecycleStatsAction.Response(runDuration, timeBetweenStarts, dataStreams); + } + + @Override + protected GetDataStreamLifecycleStatsAction.Response mutateInstance(GetDataStreamLifecycleStatsAction.Response instance) { + var runDuration = instance.getRunDuration(); + var timeBetweenStarts = instance.getTimeBetweenStarts(); + var dataStreams = instance.getDataStreamStats(); + switch (randomInt(2)) { + case 0 -> runDuration = runDuration != null && randomBoolean() + ? null + : randomValueOtherThan(runDuration, () -> randomLongBetween(10, 100000000)); + case 1 -> timeBetweenStarts = timeBetweenStarts != null && randomBoolean() + ? null + : randomValueOtherThan(timeBetweenStarts, () -> randomLongBetween(10, 100000000)); + default -> dataStreams = mutateDataStreamStats(dataStreams); + } + return new GetDataStreamLifecycleStatsAction.Response(runDuration, timeBetweenStarts, dataStreams); + } + + private List mutateDataStreamStats( + List dataStreamStats + ) { + // change the stats of a data stream + List mutated = new ArrayList<>(dataStreamStats); + if (randomBoolean() && dataStreamStats.isEmpty() == false) { + int i = randomInt(dataStreamStats.size() - 1); + GetDataStreamLifecycleStatsAction.Response.DataStreamStats instance = dataStreamStats.get(i); + mutated.set(i, switch (randomInt(2)) { + case 0 -> new GetDataStreamLifecycleStatsAction.Response.DataStreamStats( + instance.dataStreamName() + randomAlphaOfLength(2), + instance.backingIndicesInTotal(), + instance.backingIndicesInError() + ); + case 1 -> new GetDataStreamLifecycleStatsAction.Response.DataStreamStats( + instance.dataStreamName(), + instance.backingIndicesInTotal() + randomIntBetween(1, 10), + instance.backingIndicesInError() + ); + default -> new GetDataStreamLifecycleStatsAction.Response.DataStreamStats( + instance.dataStreamName(), + instance.backingIndicesInTotal(), + instance.backingIndicesInError() + randomIntBetween(1, 10) + ); + + }); + } else if (dataStreamStats.isEmpty() || randomBoolean()) { + mutated.add( + new GetDataStreamLifecycleStatsAction.Response.DataStreamStats( + randomAlphaOfLength(10), + randomIntBetween(1, 1000), + randomIntBetween(0, 100) + ) + ); + } else { + mutated.remove(randomInt(dataStreamStats.size() - 1)); + } + return mutated; + } + + @SuppressWarnings("unchecked") + public void testXContentSerialization() throws IOException { + GetDataStreamLifecycleStatsAction.Response testInstance = createTestInstance(); + try (XContentBuilder builder = XContentBuilder.builder(XContentType.JSON.xContent())) { + builder.humanReadable(true); + testInstance.toXContentChunked(ToXContent.EMPTY_PARAMS).forEachRemaining(xcontent -> { + try { + xcontent.toXContent(builder, EMPTY_PARAMS); + } catch (IOException e) { + logger.error(e.getMessage(), e); + fail(e.getMessage()); + } + }); + Map xContentMap = XContentHelper.convertToMap(BytesReference.bytes(builder), false, builder.contentType()).v2(); + assertThat(xContentMap.get("last_run_duration_in_millis"), is(testInstance.getRunDuration().intValue())); + assertThat( + xContentMap.get("last_run_duration"), + is(TimeValue.timeValueMillis(testInstance.getRunDuration()).toHumanReadableString(2)) + ); + assertThat(xContentMap.get("time_between_starts_in_millis"), is(testInstance.getTimeBetweenStarts().intValue())); + assertThat( + xContentMap.get("time_between_starts"), + is(TimeValue.timeValueMillis(testInstance.getTimeBetweenStarts()).toHumanReadableString(2)) + ); + assertThat(xContentMap.get("data_stream_count"), is(testInstance.getDataStreamStats().size())); + List> dataStreams = (List>) xContentMap.get("data_streams"); + if (testInstance.getDataStreamStats().isEmpty()) { + assertThat(dataStreams.isEmpty(), is(true)); + } else { + assertThat(dataStreams.size(), is(testInstance.getDataStreamStats().size())); + for (int i = 0; i < dataStreams.size(); i++) { + assertThat(dataStreams.get(i).get("name"), is(testInstance.getDataStreamStats().get(i).dataStreamName())); + assertThat( + dataStreams.get(i).get("backing_indices_in_total"), + is(testInstance.getDataStreamStats().get(i).backingIndicesInTotal()) + ); + assertThat( + dataStreams.get(i).get("backing_indices_in_error"), + is(testInstance.getDataStreamStats().get(i).backingIndicesInError()) + ); + } + } + } + } + + @Override + protected Writeable.Reader instanceReader() { + return GetDataStreamLifecycleStatsAction.Response::new; + } +} diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/action/TransportGetDataStreamLifecycleStatsActionTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/action/TransportGetDataStreamLifecycleStatsActionTests.java new file mode 100644 index 0000000000000..8c423107ea2f4 --- /dev/null +++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/action/TransportGetDataStreamLifecycleStatsActionTests.java @@ -0,0 +1,153 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.datastreams.lifecycle.action; + +import org.elasticsearch.action.admin.indices.rollover.MaxAgeCondition; +import org.elasticsearch.action.admin.indices.rollover.RolloverInfo; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.cluster.ClusterName; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.metadata.DataStream; +import org.elasticsearch.cluster.metadata.DataStreamLifecycle; +import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.metadata.Metadata; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.core.TimeValue; +import org.elasticsearch.datastreams.lifecycle.DataStreamLifecycleErrorStore; +import org.elasticsearch.datastreams.lifecycle.DataStreamLifecycleService; +import org.elasticsearch.index.Index; +import org.elasticsearch.index.IndexVersion; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportService; +import org.junit.Before; + +import java.time.Clock; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import static org.elasticsearch.cluster.metadata.DataStreamTestHelper.newInstance; +import static org.elasticsearch.datastreams.lifecycle.DataStreamLifecycleFixtures.createDataStream; +import static org.hamcrest.Matchers.is; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class TransportGetDataStreamLifecycleStatsActionTests extends ESTestCase { + + private final DataStreamLifecycleService dataStreamLifecycleService = mock(DataStreamLifecycleService.class); + private final DataStreamLifecycleErrorStore errorStore = mock(DataStreamLifecycleErrorStore.class); + private final TransportGetDataStreamLifecycleStatsAction action = new TransportGetDataStreamLifecycleStatsAction( + mock(TransportService.class), + mock(ClusterService.class), + mock(ThreadPool.class), + mock(ActionFilters.class), + mock(IndexNameExpressionResolver.class), + dataStreamLifecycleService + ); + private Long lastRunDuration; + private Long timeBetweenStarts; + + @Before + public void setUp() throws Exception { + super.setUp(); + lastRunDuration = randomBoolean() ? randomLongBetween(0, 100000) : null; + timeBetweenStarts = randomBoolean() ? randomLongBetween(0, 100000) : null; + when(dataStreamLifecycleService.getLastRunDuration()).thenReturn(lastRunDuration); + when(dataStreamLifecycleService.getTimeBetweenStarts()).thenReturn(timeBetweenStarts); + when(dataStreamLifecycleService.getErrorStore()).thenReturn(errorStore); + when(errorStore.getAllIndices()).thenReturn(Set.of()); + } + + public void testEmptyClusterState() { + GetDataStreamLifecycleStatsAction.Response response = action.collectStats(ClusterState.EMPTY_STATE); + assertThat(response.getRunDuration(), is(lastRunDuration)); + assertThat(response.getTimeBetweenStarts(), is(timeBetweenStarts)); + assertThat(response.getDataStreamStats().isEmpty(), is(true)); + } + + public void testMixedDataStreams() { + Set indicesInError = new HashSet<>(); + int numBackingIndices = 3; + Metadata.Builder builder = Metadata.builder(); + DataStream ilmDataStream = createDataStream( + builder, + "ilm-managed-index", + numBackingIndices, + Settings.builder() + .put(IndexMetadata.LIFECYCLE_NAME, "ILM_policy") + .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current()), + null, + Clock.systemUTC().millis() + ); + builder.put(ilmDataStream); + DataStream dslDataStream = createDataStream( + builder, + "dsl-managed-index", + numBackingIndices, + settings(IndexVersion.current()), + DataStreamLifecycle.newBuilder().dataRetention(TimeValue.timeValueDays(10)).build(), + Clock.systemUTC().millis() + ); + indicesInError.add(dslDataStream.getIndices().get(randomInt(numBackingIndices - 1)).getName()); + builder.put(dslDataStream); + { + String dataStreamName = "mixed"; + final List backingIndices = new ArrayList<>(); + for (int k = 1; k <= 2; k++) { + IndexMetadata.Builder indexMetaBuilder = IndexMetadata.builder(DataStream.getDefaultBackingIndexName(dataStreamName, k)) + .settings( + Settings.builder() + .put(IndexMetadata.LIFECYCLE_NAME, "ILM_policy") + .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current()) + ) + .numberOfShards(1) + .numberOfReplicas(1) + .creationDate(Clock.systemUTC().millis()); + + IndexMetadata indexMetadata = indexMetaBuilder.build(); + builder.put(indexMetadata, false); + backingIndices.add(indexMetadata.getIndex()); + } + // DSL managed write index + IndexMetadata.Builder indexMetaBuilder = IndexMetadata.builder(DataStream.getDefaultBackingIndexName(dataStreamName, 3)) + .settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current())) + .numberOfShards(1) + .numberOfReplicas(1) + .creationDate(Clock.systemUTC().millis()); + MaxAgeCondition rolloverCondition = new MaxAgeCondition(TimeValue.timeValueMillis(Clock.systemUTC().millis() - 2000L)); + indexMetaBuilder.putRolloverInfo( + new RolloverInfo(dataStreamName, List.of(rolloverCondition), Clock.systemUTC().millis() - 2000L) + ); + IndexMetadata indexMetadata = indexMetaBuilder.build(); + builder.put(indexMetadata, false); + backingIndices.add(indexMetadata.getIndex()); + builder.put(newInstance(dataStreamName, backingIndices, 3, null, false, DataStreamLifecycle.newBuilder().build())); + } + ClusterState state = ClusterState.builder(ClusterName.DEFAULT).metadata(builder).build(); + when(errorStore.getAllIndices()).thenReturn(indicesInError); + GetDataStreamLifecycleStatsAction.Response response = action.collectStats(state); + assertThat(response.getRunDuration(), is(lastRunDuration)); + assertThat(response.getTimeBetweenStarts(), is(timeBetweenStarts)); + assertThat(response.getDataStreamStats().size(), is(2)); + for (GetDataStreamLifecycleStatsAction.Response.DataStreamStats stats : response.getDataStreamStats()) { + if (stats.dataStreamName().equals("dsl-managed-index")) { + assertThat(stats.backingIndicesInTotal(), is(3)); + assertThat(stats.backingIndicesInError(), is(1)); + } + if (stats.dataStreamName().equals("mixed")) { + assertThat(stats.backingIndicesInTotal(), is(1)); + assertThat(stats.backingIndicesInError(), is(0)); + } + } + } +} diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapper.java index ee04346591009..161cb1674a7b9 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapper.java @@ -324,9 +324,9 @@ public Query phrasePrefixQuery(TokenStream stream, int slop, int maxExpansions, @Override public BlockLoader blockLoader(BlockLoaderContext blContext) { if (textFieldType.isSyntheticSource()) { - return BlockStoredFieldsReader.bytesRefsFromStrings(storedFieldNameForSyntheticSource()); + return new BlockStoredFieldsReader.BytesFromStringsBlockLoader(storedFieldNameForSyntheticSource()); } - return BlockSourceReader.bytesRefs(SourceValueFetcher.toString(blContext.sourcePaths(name()))); + return new BlockSourceReader.BytesRefsBlockLoader(SourceValueFetcher.toString(blContext.sourcePaths(name()))); } @Override diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java index abed23621d5e9..b35fb09c2d053 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java @@ -310,13 +310,13 @@ public Query rangeQuery( public BlockLoader blockLoader(BlockLoaderContext blContext) { if (indexMode == IndexMode.TIME_SERIES && metricType == TimeSeriesParams.MetricType.COUNTER) { // Counters are not supported by ESQL so we load them in null - return BlockDocValuesReader.nulls(); + return BlockLoader.CONSTANT_NULLS; } if (hasDocValues()) { double scalingFactorInverse = 1d / scalingFactor; - return BlockDocValuesReader.doubles(name(), l -> l * scalingFactorInverse); + return new BlockDocValuesReader.DoublesBlockLoader(name(), l -> l * scalingFactorInverse); } - return BlockSourceReader.doubles(sourceValueFetcher(blContext.sourcePaths(name()))); + return new BlockSourceReader.DoublesBlockLoader(sourceValueFetcher(blContext.sourcePaths(name()))); } @Override diff --git a/modules/reindex/src/main/java/org/elasticsearch/reindex/AbstractBaseReindexRestHandler.java b/modules/reindex/src/main/java/org/elasticsearch/reindex/AbstractBaseReindexRestHandler.java index 952dd0585e7ba..8e7fab68ac697 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/reindex/AbstractBaseReindexRestHandler.java +++ b/modules/reindex/src/main/java/org/elasticsearch/reindex/AbstractBaseReindexRestHandler.java @@ -11,7 +11,7 @@ import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.ActionType; import org.elasticsearch.action.support.ActiveShardCount; -import org.elasticsearch.action.support.ListenableActionFuture; +import org.elasticsearch.action.support.SubscribableListener; import org.elasticsearch.client.internal.node.NodeClient; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.index.reindex.AbstractBulkByScrollRequest; @@ -64,9 +64,9 @@ protected RestChannelConsumer doPrepareRequest(RestRequest request, NodeClient c if (validationException != null) { throw validationException; } - final var responseFuture = new ListenableActionFuture(); - final var task = client.executeLocally(action, internal, responseFuture); - responseFuture.addListener(new LoggingTaskListener<>(task)); + final var responseListener = new SubscribableListener(); + final var task = client.executeLocally(action, internal, responseListener); + responseListener.addListener(new LoggingTaskListener<>(task)); return sendTask(client.getLocalNodeId(), task); } diff --git a/modules/repository-s3/src/internalClusterTest/java/org/elasticsearch/repositories/s3/S3BlobStoreRepositoryTests.java b/modules/repository-s3/src/internalClusterTest/java/org/elasticsearch/repositories/s3/S3BlobStoreRepositoryTests.java index aee61361ebd10..7f46440647a54 100644 --- a/modules/repository-s3/src/internalClusterTest/java/org/elasticsearch/repositories/s3/S3BlobStoreRepositoryTests.java +++ b/modules/repository-s3/src/internalClusterTest/java/org/elasticsearch/repositories/s3/S3BlobStoreRepositoryTests.java @@ -57,7 +57,7 @@ import org.elasticsearch.telemetry.metric.MeterRegistry; import org.elasticsearch.test.BackgroundIndexer; import org.elasticsearch.test.ESIntegTestCase; -import org.elasticsearch.test.junit.annotations.TestLogging; +import org.elasticsearch.test.junit.annotations.TestIssueLogging; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.xcontent.NamedXContentRegistry; import org.elasticsearch.xcontent.XContentFactory; @@ -179,7 +179,7 @@ protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) { } @Override - @TestLogging(reason = "Enable request logging to debug #88841", value = "com.amazonaws.request:DEBUG") + @TestIssueLogging(issueUrl = "https://github.com/elastic/elasticsearch/issues/88841", value = "com.amazonaws.request:DEBUG") public void testRequestStats() throws Exception { super.testRequestStats(); } @@ -225,6 +225,7 @@ public void testAbortRequestStats() throws Exception { assertEquals(assertionErrorMsg, mockCalls, sdkRequestCounts); } + @TestIssueLogging(issueUrl = "https://github.com/elastic/elasticsearch/issues/101608", value = "com.amazonaws.request:DEBUG") public void testMetrics() throws Exception { // Create the repository and perform some activities final String repository = createRepository(randomRepositoryName()); @@ -281,8 +282,12 @@ public void testMetrics() throws Exception { operation, OperationPurpose.parse((String) metric.attributes().get("purpose")) ); - assertThat(statsCollectors, hasKey(statsKey)); - assertThat(metric.getLong(), equalTo(statsCollectors.get(statsKey).counter.sum())); + assertThat(nodeName + "/" + statsKey + " exists", statsCollectors, hasKey(statsKey)); + assertThat( + nodeName + "/" + statsKey + " has correct sum", + metric.getLong(), + equalTo(statsCollectors.get(statsKey).counter.sum()) + ); aggregatedMetrics.compute(operation.getKey(), (k, v) -> v == null ? metric.getLong() : v + metric.getLong()); }); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/ConcurrentSnapshotsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/ConcurrentSnapshotsIT.java index ca522064e3d04..f91a848ed2362 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/ConcurrentSnapshotsIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/ConcurrentSnapshotsIT.java @@ -39,6 +39,7 @@ import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.disruption.NetworkDisruption; import org.elasticsearch.test.transport.MockTransportService; +import org.elasticsearch.transport.RemoteTransportException; import java.io.IOException; import java.nio.file.Files; @@ -768,7 +769,18 @@ public void testQueuedOperationsAndBrokenRepoOnMasterFailOver() throws Exception ensureStableCluster(3); awaitNoMoreRunningOperations(); - expectThrows(RepositoryException.class, deleteFuture::actionGet); + var innerException = expectThrows(ExecutionException.class, RuntimeException.class, deleteFuture::get); + + // There may be many layers of RTE to unwrap here, see https://github.com/elastic/elasticsearch/issues/102351. + // ExceptionsHelper#unwrapCause gives up at 10 layers of wrapping so we must unwrap more tenaciously by hand here: + while (true) { + if (innerException instanceof RemoteTransportException remoteTransportException) { + innerException = asInstanceOf(RuntimeException.class, remoteTransportException.getCause()); + } else { + assertThat(innerException, instanceOf(RepositoryException.class)); + break; + } + } } public void testQueuedSnapshotOperationsAndBrokenRepoOnMasterFailOver() throws Exception { diff --git a/server/src/internalClusterTest/java/org/elasticsearch/versioning/ConcurrentSeqNoVersioningIT.java b/server/src/internalClusterTest/java/org/elasticsearch/versioning/ConcurrentSeqNoVersioningIT.java index 0277c569fbe5d..0f9b63f2757cf 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/versioning/ConcurrentSeqNoVersioningIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/versioning/ConcurrentSeqNoVersioningIT.java @@ -33,6 +33,8 @@ import java.io.FileInputStream; import java.io.IOException; import java.io.OutputStream; +import java.io.OutputStreamWriter; +import java.nio.charset.StandardCharsets; import java.util.Arrays; import java.util.Base64; import java.util.List; @@ -116,6 +118,7 @@ public class ConcurrentSeqNoVersioningIT extends AbstractDisruptionTestCase { // multiple threads doing CAS updates. // Wait up to 1 minute (+10s in thread to ensure it does not time out) for threads to complete previous round before initiating next // round. + @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/102255") public void testSeqNoCASLinearizability() { final int disruptTimeSeconds = scaledRandomIntBetween(1, 8); @@ -448,13 +451,24 @@ public void assertLinearizable() { var chunkedLoggingStream = ChunkedLoggingStream.create( logger, Level.ERROR, - "unlinearizable history", + "raw unlinearizable history in partition " + id, ReferenceDocs.LOGGING // any old docs link will do ); var output = new OutputStreamStreamOutput(chunkedLoggingStream) ) { writeHistory(output, history); } + try ( + var chunkedLoggingStream = ChunkedLoggingStream.create( + logger, + Level.ERROR, + "visualisation of unlinearizable history in partition " + id, + ReferenceDocs.LOGGING // any old docs link will do + ); + var writer = new OutputStreamWriter(chunkedLoggingStream, StandardCharsets.UTF_8) + ) { + LinearizabilityChecker.writeVisualisation(spec, history, missingResponseGenerator(), writer); + } } } catch (IOException e) { logger.error("failure writing out history", e); diff --git a/server/src/main/java/org/elasticsearch/TransportVersions.java b/server/src/main/java/org/elasticsearch/TransportVersions.java index baae500b70d55..5ad1d43c0d4f8 100644 --- a/server/src/main/java/org/elasticsearch/TransportVersions.java +++ b/server/src/main/java/org/elasticsearch/TransportVersions.java @@ -172,6 +172,7 @@ static TransportVersion def(int id) { public static final TransportVersion DATA_STREAM_FAILURE_STORE_ADDED = def(8_541_00_0); public static final TransportVersion ML_INFERENCE_OPENAI_ADDED = def(8_542_00_0); public static final TransportVersion SHUTDOWN_MIGRATION_STATUS_INCLUDE_COUNTS = def(8_543_00_0); + public static final TransportVersion TRANSFORM_GET_CHECKPOINT_QUERY_AND_CLUSTER_ADDED = def(8_544_00_0); /* * STOP! READ THIS FIRST! No, really, diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/alias/get/GetAliasesRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/alias/get/GetAliasesRequest.java index ee6797ca58fb9..9d10065c9c3e9 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/alias/get/GetAliasesRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/alias/get/GetAliasesRequest.java @@ -15,6 +15,7 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.core.UpdateForV9; import org.elasticsearch.tasks.CancellableTask; import org.elasticsearch.tasks.Task; import org.elasticsearch.tasks.TaskId; @@ -22,6 +23,7 @@ import java.io.IOException; import java.util.Map; +@UpdateForV9 // make this class a regular ActionRequest rather than a MasterNodeReadRequest public class GetAliasesRequest extends MasterNodeReadRequest implements AliasesRequest { public static final IndicesOptions DEFAULT_INDICES_OPTIONS = IndicesOptions.strictExpandHidden(); @@ -40,9 +42,10 @@ public GetAliasesRequest() {} /** * NB prior to 8.12 get-aliases was a TransportMasterNodeReadAction so for BwC we must remain able to read these requests until we no - * longer need to support {@link org.elasticsearch.TransportVersions#CLUSTER_FEATURES_ADDED} and earlier. Once we remove this we can - * also make this class a regular ActionRequest instead of a MasterNodeReadRequest. + * longer need to support calling this action remotely. Once we remove this we can also make this class a regular ActionRequest instead + * of a MasterNodeReadRequest. */ + @UpdateForV9 // remove this constructor public GetAliasesRequest(StreamInput in) throws IOException { super(in); indices = in.readStringArray(); diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/alias/get/GetAliasesResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/alias/get/GetAliasesResponse.java index c0e26b16585c4..edb05b0fcef75 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/alias/get/GetAliasesResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/alias/get/GetAliasesResponse.java @@ -12,6 +12,7 @@ import org.elasticsearch.cluster.metadata.AliasMetadata; import org.elasticsearch.cluster.metadata.DataStreamAlias; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.core.UpdateForV9; import java.io.IOException; import java.util.List; @@ -38,8 +39,9 @@ public Map> getDataStreamAliases() { /** * NB prior to 8.12 get-aliases was a TransportMasterNodeReadAction so for BwC we must remain able to write these responses until we no - * longer need to support {@link org.elasticsearch.TransportVersions#CLUSTER_FEATURES_ADDED} and earlier. + * longer need to support calling this action remotely. */ + @UpdateForV9 // replace this implementation with TransportAction.localOnly() @Override public void writeTo(StreamOutput out) throws IOException { out.writeMap(aliases, StreamOutput::writeCollection); diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/alias/get/TransportGetAliasesAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/alias/get/TransportGetAliasesAction.java index e43d1a825c233..9b9fb49c1bbe0 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/alias/get/TransportGetAliasesAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/alias/get/TransportGetAliasesAction.java @@ -24,6 +24,7 @@ import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.common.regex.Regex; import org.elasticsearch.common.util.concurrent.ThreadContext; +import org.elasticsearch.core.UpdateForV9; import org.elasticsearch.indices.SystemIndices; import org.elasticsearch.indices.SystemIndices.SystemIndexAccessLevel; import org.elasticsearch.tasks.CancellableTask; @@ -41,9 +42,9 @@ /** * NB prior to 8.12 this was a TransportMasterNodeReadAction so for BwC it must be registered with the TransportService (i.e. a - * HandledTransportAction) until we no longer need to support {@link org.elasticsearch.TransportVersions#CLUSTER_FEATURES_ADDED} and - * earlier. + * HandledTransportAction) until we no longer need to support calling this action remotely. */ +@UpdateForV9 // remove the HandledTransportAction superclass, this action need not be registered with the TransportService public class TransportGetAliasesAction extends TransportLocalClusterStateAction { private static final DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(TransportGetAliasesAction.class); diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/JoinValidationService.java b/server/src/main/java/org/elasticsearch/cluster/coordination/JoinValidationService.java index d9911ad12df84..dc18a7950394a 100644 --- a/server/src/main/java/org/elasticsearch/cluster/coordination/JoinValidationService.java +++ b/server/src/main/java/org/elasticsearch/cluster/coordination/JoinValidationService.java @@ -12,7 +12,7 @@ import org.apache.logging.log4j.Logger; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.TransportVersion; -import org.elasticsearch.Version; +import org.elasticsearch.TransportVersions; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.ActionRunnable; import org.elasticsearch.cluster.ClusterState; @@ -30,6 +30,7 @@ import org.elasticsearch.core.Nullable; import org.elasticsearch.core.RefCounted; import org.elasticsearch.core.TimeValue; +import org.elasticsearch.core.UpdateForV9; import org.elasticsearch.env.Environment; import org.elasticsearch.node.NodeClosedException; import org.elasticsearch.threadpool.ThreadPool; @@ -148,10 +149,23 @@ public JoinValidationService( } public void validateJoin(DiscoveryNode discoveryNode, ActionListener listener) { - if (discoveryNode.getVersion().onOrAfter(Version.V_8_3_0)) { + // This node isn't in the cluster yet so ClusterState#getMinTransportVersion() doesn't apply, we must obtain a specific connection + // so we can check its transport version to decide how to proceed. + + final Transport.Connection connection; + try { + connection = transportService.getConnection(discoveryNode); + assert connection != null; + } catch (Exception e) { + assert e instanceof NodeNotConnectedException : e; + listener.onFailure(e); + return; + } + + if (connection.getTransportVersion().onOrAfter(TransportVersions.V_8_3_0)) { if (executeRefs.tryIncRef()) { try { - execute(new JoinValidation(discoveryNode, listener)); + execute(new JoinValidation(discoveryNode, connection, listener)); } finally { executeRefs.decRef(); } @@ -159,39 +173,44 @@ public void validateJoin(DiscoveryNode discoveryNode, ActionListener liste listener.onFailure(new NodeClosedException(transportService.getLocalNode())); } } else { - final var responseHandler = TransportResponseHandler.empty(responseExecutor, listener.delegateResponse((l, e) -> { - logger.warn(() -> "failed to validate incoming join request from node [" + discoveryNode + "]", e); - listener.onFailure( - new IllegalStateException( - String.format( - Locale.ROOT, - "failure when sending a join validation request from [%s] to [%s]", - transportService.getLocalNode().descriptionWithoutAttributes(), - discoveryNode.descriptionWithoutAttributes() - ), - e - ) - ); - })); - final var clusterState = clusterStateSupplier.get(); - if (clusterState != null) { - assert clusterState.nodes().isLocalNodeElectedMaster(); - transportService.sendRequest( - discoveryNode, - JOIN_VALIDATE_ACTION_NAME, - new ValidateJoinRequest(clusterState), - REQUEST_OPTIONS, - responseHandler - ); - } else { - transportService.sendRequest( - discoveryNode, - JoinHelper.JOIN_PING_ACTION_NAME, - TransportRequest.Empty.INSTANCE, - REQUEST_OPTIONS, - responseHandler - ); - } + legacyValidateJoin(discoveryNode, listener, connection); + } + } + + @UpdateForV9 + private void legacyValidateJoin(DiscoveryNode discoveryNode, ActionListener listener, Transport.Connection connection) { + final var responseHandler = TransportResponseHandler.empty(responseExecutor, listener.delegateResponse((l, e) -> { + logger.warn(() -> "failed to validate incoming join request from node [" + discoveryNode + "]", e); + listener.onFailure( + new IllegalStateException( + String.format( + Locale.ROOT, + "failure when sending a join validation request from [%s] to [%s]", + transportService.getLocalNode().descriptionWithoutAttributes(), + discoveryNode.descriptionWithoutAttributes() + ), + e + ) + ); + })); + final var clusterState = clusterStateSupplier.get(); + if (clusterState != null) { + assert clusterState.nodes().isLocalNodeElectedMaster(); + transportService.sendRequest( + connection, + JOIN_VALIDATE_ACTION_NAME, + new ValidateJoinRequest(clusterState), + REQUEST_OPTIONS, + responseHandler + ); + } else { + transportService.sendRequest( + connection, + JoinHelper.JOIN_PING_ACTION_NAME, + TransportRequest.Empty.INSTANCE, + REQUEST_OPTIONS, + responseHandler + ); } } @@ -312,27 +331,22 @@ public String toString() { private class JoinValidation extends ActionRunnable { private final DiscoveryNode discoveryNode; + private final Transport.Connection connection; - JoinValidation(DiscoveryNode discoveryNode, ActionListener listener) { + JoinValidation(DiscoveryNode discoveryNode, Transport.Connection connection, ActionListener listener) { super(listener); this.discoveryNode = discoveryNode; + this.connection = connection; } @Override - protected void doRun() throws Exception { - assert discoveryNode.getVersion().onOrAfter(Version.V_8_3_0) : discoveryNode.getVersion(); + protected void doRun() { + assert connection.getTransportVersion().onOrAfter(TransportVersions.V_8_3_0) : discoveryNode.getVersion(); // NB these things never run concurrently to each other, or to the cache cleaner (see IMPLEMENTATION NOTES above) so it is safe // to do these (non-atomic) things to the (unsynchronized) statesByVersion map. - Transport.Connection connection; - try { - connection = transportService.getConnection(discoveryNode); - } catch (NodeNotConnectedException e) { - listener.onFailure(e); - return; - } - var version = connection.getTransportVersion(); - var cachedBytes = statesByVersion.get(version); - var bytes = maybeSerializeClusterState(cachedBytes, discoveryNode, version); + var transportVersion = connection.getTransportVersion(); + var cachedBytes = statesByVersion.get(transportVersion); + var bytes = maybeSerializeClusterState(cachedBytes, discoveryNode, transportVersion); if (bytes == null) { // Normally if we're not the master then the Coordinator sends a ping message just to validate connectivity instead of // getting here. But if we were the master when the Coordinator checked then we might not be the master any more, so we @@ -354,7 +368,7 @@ protected void doRun() throws Exception { transportService.sendRequest( connection, JOIN_VALIDATE_ACTION_NAME, - new BytesTransportRequest(bytes, version), + new BytesTransportRequest(bytes, transportVersion), REQUEST_OPTIONS, new CleanableResponseHandler<>( listener.map(ignored -> null), diff --git a/server/src/main/java/org/elasticsearch/common/util/BigByteArray.java b/server/src/main/java/org/elasticsearch/common/util/BigByteArray.java index 72a2fc41a9a12..2c623882afe14 100644 --- a/server/src/main/java/org/elasticsearch/common/util/BigByteArray.java +++ b/server/src/main/java/org/elasticsearch/common/util/BigByteArray.java @@ -64,6 +64,10 @@ public byte set(long index, byte value) { @Override public boolean get(long index, int len, BytesRef ref) { assert index + len <= size(); + if (len == 0) { + ref.length = 0; + return false; + } int pageIndex = pageIndex(index); final int indexInPage = indexInPage(index); if (indexInPage + len <= pageSize()) { diff --git a/server/src/main/java/org/elasticsearch/index/engine/Engine.java b/server/src/main/java/org/elasticsearch/index/engine/Engine.java index ed7fab325408e..43437529cd301 100644 --- a/server/src/main/java/org/elasticsearch/index/engine/Engine.java +++ b/server/src/main/java/org/elasticsearch/index/engine/Engine.java @@ -48,6 +48,7 @@ import org.elasticsearch.core.Releasable; import org.elasticsearch.core.Releasables; import org.elasticsearch.core.TimeValue; +import org.elasticsearch.core.UpdateForV9; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.VersionType; import org.elasticsearch.index.mapper.DocumentParser; @@ -100,7 +101,8 @@ public abstract class Engine implements Closeable { - public static final String SYNC_COMMIT_ID = "sync_id"; // TODO: Remove sync_id in 9.0 + @UpdateForV9 // TODO: Remove sync_id in 9.0 + public static final String SYNC_COMMIT_ID = "sync_id"; public static final String HISTORY_UUID_KEY = "history_uuid"; public static final String FORCE_MERGE_UUID_KEY = "force_merge_uuid"; public static final String MIN_RETAINED_SEQNO = "min_retained_seq_no"; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/BlockDocValuesReader.java b/server/src/main/java/org/elasticsearch/index/mapper/BlockDocValuesReader.java index 90a295e5a25f2..6e572eceeafc4 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/BlockDocValuesReader.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/BlockDocValuesReader.java @@ -8,6 +8,7 @@ package org.elasticsearch.index.mapper; +import org.apache.lucene.index.BinaryDocValues; import org.apache.lucene.index.DocValues; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.NumericDocValues; @@ -15,171 +16,94 @@ import org.apache.lucene.index.SortedNumericDocValues; import org.apache.lucene.index.SortedSetDocValues; import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.UnicodeUtil; -import org.elasticsearch.core.CheckedFunction; -import org.elasticsearch.index.fielddata.SortedBinaryDocValues; +import org.elasticsearch.common.io.stream.ByteArrayStreamInput; +import org.elasticsearch.index.mapper.BlockLoader.BlockFactory; import org.elasticsearch.index.mapper.BlockLoader.BooleanBuilder; import org.elasticsearch.index.mapper.BlockLoader.Builder; -import org.elasticsearch.index.mapper.BlockLoader.BuilderFactory; import org.elasticsearch.index.mapper.BlockLoader.BytesRefBuilder; import org.elasticsearch.index.mapper.BlockLoader.Docs; import org.elasticsearch.index.mapper.BlockLoader.DoubleBuilder; import org.elasticsearch.index.mapper.BlockLoader.IntBuilder; import org.elasticsearch.index.mapper.BlockLoader.LongBuilder; +import org.elasticsearch.search.fetch.StoredFieldsSpec; import java.io.IOException; /** * A reader that supports reading doc-values from a Lucene segment in Block fashion. */ -public abstract class BlockDocValuesReader { - public interface Factory { - BlockDocValuesReader build(int segment) throws IOException; - - boolean supportsOrdinals(); - - SortedSetDocValues ordinals(int segment) throws IOException; - } - - protected final Thread creationThread; +public abstract class BlockDocValuesReader implements BlockLoader.AllReader { + private final Thread creationThread; public BlockDocValuesReader() { this.creationThread = Thread.currentThread(); } - /** - * Returns the current doc that this reader is on. - */ - public abstract int docID(); - - /** - * The {@link BlockLoader.Builder} for data of this type. - */ - public abstract Builder builder(BuilderFactory factory, int expectedCount); - - /** - * Reads the values of the given documents specified in the input block - */ - public abstract BlockLoader.Block readValues(BuilderFactory factory, Docs docs) throws IOException; - - /** - * Reads the values of the given document into the builder - */ - public abstract void readValuesFromSingleDoc(int docId, Builder builder) throws IOException; + protected abstract int docId(); /** * Checks if the reader can be used to read a range documents starting with the given docID by the current thread. */ - public static boolean canReuse(BlockDocValuesReader reader, int startingDocID) { - return reader != null && reader.creationThread == Thread.currentThread() && reader.docID() <= startingDocID; + @Override + public final boolean canReuse(int startingDocID) { + return creationThread == Thread.currentThread() && docId() <= startingDocID; } - public static BlockLoader booleans(String fieldName) { - return context -> { - SortedNumericDocValues docValues = DocValues.getSortedNumeric(context.reader(), fieldName); - NumericDocValues singleton = DocValues.unwrapSingleton(docValues); - if (singleton != null) { - return new SingletonBooleans(singleton); - } - return new Booleans(docValues); - }; - } + @Override + public abstract String toString(); - public static BlockLoader bytesRefsFromOrds(String fieldName) { - return new BlockLoader() { - @Override - public BlockDocValuesReader reader(LeafReaderContext context) throws IOException { - SortedSetDocValues docValues = ordinals(context); - SortedDocValues singleton = DocValues.unwrapSingleton(docValues); - if (singleton != null) { - return new SingletonOrdinals(singleton); - } - return new Ordinals(docValues); - } + public abstract static class DocValuesBlockLoader implements BlockLoader { + public abstract AllReader reader(LeafReaderContext context) throws IOException; - @Override - public boolean supportsOrdinals() { - return true; - } + @Override + public final ColumnAtATimeReader columnAtATimeReader(LeafReaderContext context) throws IOException { + return reader(context); + } - @Override - public SortedSetDocValues ordinals(LeafReaderContext context) throws IOException { - return DocValues.getSortedSet(context.reader(), fieldName); - } - }; - } + @Override + public final RowStrideReader rowStrideReader(LeafReaderContext context) throws IOException { + return reader(context); + } - /** - * Load {@link BytesRef} values from doc values. Prefer {@link #bytesRefsFromOrds} if - * doc values are indexed with ordinals because that's generally much faster. It's - * possible to use this with field data, but generally should be avoided because field - * data has higher per invocation overhead. - */ - public static BlockLoader bytesRefsFromDocValues(CheckedFunction fieldData) { - return context -> new Bytes(fieldData.apply(context)); - } + @Override + public final StoredFieldsSpec rowStrideStoredFieldSpec() { + return StoredFieldsSpec.NO_REQUIREMENTS; + } - /** - * Convert from the stored {@link long} into the {@link double} to load. - * Sadly, this will go megamorphic pretty quickly and slow us down, - * but it gets the job done for now. - */ - public interface ToDouble { - double convert(long v); - } + @Override + public boolean supportsOrdinals() { + return false; + } - /** - * Load {@code double} values from doc values. - */ - public static BlockLoader doubles(String fieldName, ToDouble toDouble) { - return context -> { - SortedNumericDocValues docValues = DocValues.getSortedNumeric(context.reader(), fieldName); - NumericDocValues singleton = DocValues.unwrapSingleton(docValues); - if (singleton != null) { - return new SingletonDoubles(singleton, toDouble); - } - return new Doubles(docValues, toDouble); - }; + @Override + public SortedSetDocValues ordinals(LeafReaderContext context) throws IOException { + throw new UnsupportedOperationException(); + } } - /** - * Load {@code int} values from doc values. - */ - public static BlockLoader ints(String fieldName) { - return context -> { - SortedNumericDocValues docValues = DocValues.getSortedNumeric(context.reader(), fieldName); - NumericDocValues singleton = DocValues.unwrapSingleton(docValues); - if (singleton != null) { - return new SingletonInts(singleton); - } - return new Ints(docValues); - }; - } + public static class LongsBlockLoader extends DocValuesBlockLoader { + private final String fieldName; - /** - * Load a block of {@code long}s from doc values. - */ - public static BlockLoader longs(String fieldName) { - return context -> { + public LongsBlockLoader(String fieldName) { + this.fieldName = fieldName; + } + + @Override + public Builder builder(BlockFactory factory, int expectedCount) { + return factory.longs(expectedCount); + } + + @Override + public AllReader reader(LeafReaderContext context) throws IOException { SortedNumericDocValues docValues = DocValues.getSortedNumeric(context.reader(), fieldName); NumericDocValues singleton = DocValues.unwrapSingleton(docValues); if (singleton != null) { return new SingletonLongs(singleton); } return new Longs(docValues); - }; - } - - /** - * Load blocks with only null. - */ - public static BlockLoader nulls() { - return context -> new Nulls(); + } } - @Override - public abstract String toString(); - private static class SingletonLongs extends BlockDocValuesReader { private final NumericDocValues numericDocValues; @@ -188,13 +112,8 @@ private static class SingletonLongs extends BlockDocValuesReader { } @Override - public BlockLoader.LongBuilder builder(BuilderFactory factory, int expectedCount) { - return factory.longsFromDocValues(expectedCount); - } - - @Override - public BlockLoader.Block readValues(BuilderFactory factory, Docs docs) throws IOException { - try (BlockLoader.LongBuilder builder = builder(factory, docs.count())) { + public BlockLoader.Block read(BlockFactory factory, Docs docs) throws IOException { + try (BlockLoader.LongBuilder builder = factory.longsFromDocValues(docs.count())) { int lastDoc = -1; for (int i = 0; i < docs.count(); i++) { int doc = docs.get(i); @@ -213,7 +132,7 @@ public BlockLoader.Block readValues(BuilderFactory factory, Docs docs) throws IO } @Override - public void readValuesFromSingleDoc(int docId, Builder builder) throws IOException { + public void read(int docId, BlockLoader.StoredFields storedFields, Builder builder) throws IOException { BlockLoader.LongBuilder blockBuilder = (BlockLoader.LongBuilder) builder; if (numericDocValues.advanceExact(docId)) { blockBuilder.appendLong(numericDocValues.longValue()); @@ -223,13 +142,13 @@ public void readValuesFromSingleDoc(int docId, Builder builder) throws IOExcepti } @Override - public int docID() { + public int docId() { return numericDocValues.docID(); } @Override public String toString() { - return "SingletonLongs"; + return "BlockDocValuesReader.SingletonLongs"; } } @@ -242,13 +161,8 @@ private static class Longs extends BlockDocValuesReader { } @Override - public BlockLoader.LongBuilder builder(BuilderFactory factory, int expectedCount) { - return factory.longsFromDocValues(expectedCount); - } - - @Override - public BlockLoader.Block readValues(BuilderFactory factory, Docs docs) throws IOException { - try (BlockLoader.LongBuilder builder = builder(factory, docs.count())) { + public BlockLoader.Block read(BlockFactory factory, Docs docs) throws IOException { + try (BlockLoader.LongBuilder builder = factory.longsFromDocValues(docs.count())) { for (int i = 0; i < docs.count(); i++) { int doc = docs.get(i); if (doc < this.docID) { @@ -261,7 +175,7 @@ public BlockLoader.Block readValues(BuilderFactory factory, Docs docs) throws IO } @Override - public void readValuesFromSingleDoc(int docId, Builder builder) throws IOException { + public void read(int docId, BlockLoader.StoredFields storedFields, Builder builder) throws IOException { read(docId, (LongBuilder) builder); } @@ -284,14 +198,37 @@ private void read(int doc, LongBuilder builder) throws IOException { } @Override - public int docID() { + public int docId() { // There is a .docID on the numericDocValues but it is often not implemented. return docID; } @Override public String toString() { - return "Longs"; + return "BlockDocValuesReader.Longs"; + } + } + + public static class IntsBlockLoader extends DocValuesBlockLoader { + private final String fieldName; + + public IntsBlockLoader(String fieldName) { + this.fieldName = fieldName; + } + + @Override + public Builder builder(BlockFactory factory, int expectedCount) { + return factory.ints(expectedCount); + } + + @Override + public AllReader reader(LeafReaderContext context) throws IOException { + SortedNumericDocValues docValues = DocValues.getSortedNumeric(context.reader(), fieldName); + NumericDocValues singleton = DocValues.unwrapSingleton(docValues); + if (singleton != null) { + return new SingletonInts(singleton); + } + return new Ints(docValues); } } @@ -303,13 +240,8 @@ private static class SingletonInts extends BlockDocValuesReader { } @Override - public IntBuilder builder(BuilderFactory factory, int expectedCount) { - return factory.intsFromDocValues(expectedCount); - } - - @Override - public BlockLoader.Block readValues(BuilderFactory factory, Docs docs) throws IOException { - try (BlockLoader.IntBuilder builder = builder(factory, docs.count())) { + public BlockLoader.Block read(BlockFactory factory, Docs docs) throws IOException { + try (BlockLoader.IntBuilder builder = factory.intsFromDocValues(docs.count())) { int lastDoc = -1; for (int i = 0; i < docs.count(); i++) { int doc = docs.get(i); @@ -328,7 +260,7 @@ public BlockLoader.Block readValues(BuilderFactory factory, Docs docs) throws IO } @Override - public void readValuesFromSingleDoc(int docId, Builder builder) throws IOException { + public void read(int docId, BlockLoader.StoredFields storedFields, Builder builder) throws IOException { IntBuilder blockBuilder = (IntBuilder) builder; if (numericDocValues.advanceExact(docId)) { blockBuilder.appendInt(Math.toIntExact(numericDocValues.longValue())); @@ -338,13 +270,13 @@ public void readValuesFromSingleDoc(int docId, Builder builder) throws IOExcepti } @Override - public int docID() { + public int docId() { return numericDocValues.docID(); } @Override public String toString() { - return "SingletonInts"; + return "BlockDocValuesReader.SingletonInts"; } } @@ -357,13 +289,8 @@ private static class Ints extends BlockDocValuesReader { } @Override - public IntBuilder builder(BuilderFactory factory, int expectedCount) { - return factory.intsFromDocValues(expectedCount); - } - - @Override - public BlockLoader.Block readValues(BuilderFactory factory, Docs docs) throws IOException { - try (BlockLoader.IntBuilder builder = builder(factory, docs.count())) { + public BlockLoader.Block read(BlockFactory factory, Docs docs) throws IOException { + try (BlockLoader.IntBuilder builder = factory.intsFromDocValues(docs.count())) { for (int i = 0; i < docs.count(); i++) { int doc = docs.get(i); if (doc < this.docID) { @@ -376,7 +303,7 @@ public BlockLoader.Block readValues(BuilderFactory factory, Docs docs) throws IO } @Override - public void readValuesFromSingleDoc(int docId, Builder builder) throws IOException { + public void read(int docId, BlockLoader.StoredFields storedFields, Builder builder) throws IOException { read(docId, (IntBuilder) builder); } @@ -399,14 +326,48 @@ private void read(int doc, IntBuilder builder) throws IOException { } @Override - public int docID() { - // There is a .docID on on the numericDocValues but it is often not implemented. + public int docId() { + // There is a .docID on the numericDocValues but it is often not implemented. return docID; } @Override public String toString() { - return "Ints"; + return "BlockDocValuesReader.Ints"; + } + } + + /** + * Convert from the stored {@link long} into the {@link double} to load. + * Sadly, this will go megamorphic pretty quickly and slow us down, + * but it gets the job done for now. + */ + public interface ToDouble { + double convert(long v); + } + + public static class DoublesBlockLoader extends DocValuesBlockLoader { + private final String fieldName; + private final ToDouble toDouble; + + public DoublesBlockLoader(String fieldName, ToDouble toDouble) { + this.fieldName = fieldName; + this.toDouble = toDouble; + } + + @Override + public Builder builder(BlockFactory factory, int expectedCount) { + return factory.doubles(expectedCount); + } + + @Override + public AllReader reader(LeafReaderContext context) throws IOException { + SortedNumericDocValues docValues = DocValues.getSortedNumeric(context.reader(), fieldName); + NumericDocValues singleton = DocValues.unwrapSingleton(docValues); + if (singleton != null) { + return new SingletonDoubles(singleton, toDouble); + } + return new Doubles(docValues, toDouble); } } @@ -421,13 +382,8 @@ private static class SingletonDoubles extends BlockDocValuesReader { } @Override - public DoubleBuilder builder(BuilderFactory factory, int expectedCount) { - return factory.doublesFromDocValues(expectedCount); - } - - @Override - public BlockLoader.Block readValues(BuilderFactory factory, Docs docs) throws IOException { - try (BlockLoader.DoubleBuilder builder = builder(factory, docs.count())) { + public BlockLoader.Block read(BlockFactory factory, Docs docs) throws IOException { + try (BlockLoader.DoubleBuilder builder = factory.doublesFromDocValues(docs.count())) { int lastDoc = -1; for (int i = 0; i < docs.count(); i++) { int doc = docs.get(i); @@ -447,7 +403,7 @@ public BlockLoader.Block readValues(BuilderFactory factory, Docs docs) throws IO } @Override - public void readValuesFromSingleDoc(int docId, Builder builder) throws IOException { + public void read(int docId, BlockLoader.StoredFields storedFields, Builder builder) throws IOException { this.docID = docId; DoubleBuilder blockBuilder = (DoubleBuilder) builder; if (docValues.advanceExact(this.docID)) { @@ -458,13 +414,13 @@ public void readValuesFromSingleDoc(int docId, Builder builder) throws IOExcepti } @Override - public int docID() { + public int docId() { return docID; } @Override public String toString() { - return "SingletonDoubles"; + return "BlockDocValuesReader.SingletonDoubles"; } } @@ -479,13 +435,8 @@ private static class Doubles extends BlockDocValuesReader { } @Override - public DoubleBuilder builder(BuilderFactory factory, int expectedCount) { - return factory.doublesFromDocValues(expectedCount); - } - - @Override - public BlockLoader.Block readValues(BuilderFactory factory, Docs docs) throws IOException { - try (BlockLoader.DoubleBuilder builder = builder(factory, docs.count())) { + public BlockLoader.Block read(BlockFactory factory, Docs docs) throws IOException { + try (BlockLoader.DoubleBuilder builder = factory.doublesFromDocValues(docs.count())) { for (int i = 0; i < docs.count(); i++) { int doc = docs.get(i); if (doc < this.docID) { @@ -498,7 +449,7 @@ public BlockLoader.Block readValues(BuilderFactory factory, Docs docs) throws IO } @Override - public void readValuesFromSingleDoc(int docId, Builder builder) throws IOException { + public void read(int docId, BlockLoader.StoredFields storedFields, Builder builder) throws IOException { read(docId, (DoubleBuilder) builder); } @@ -521,13 +472,46 @@ private void read(int doc, DoubleBuilder builder) throws IOException { } @Override - public int docID() { + public int docId() { return docID; } @Override public String toString() { - return "Doubles"; + return "BlockDocValuesReader.Doubles"; + } + } + + public static class BytesRefsFromOrdsBlockLoader extends DocValuesBlockLoader { + private final String fieldName; + + public BytesRefsFromOrdsBlockLoader(String fieldName) { + this.fieldName = fieldName; + } + + @Override + public BytesRefBuilder builder(BlockFactory factory, int expectedCount) { + return factory.bytesRefs(expectedCount); + } + + @Override + public AllReader reader(LeafReaderContext context) throws IOException { + SortedSetDocValues docValues = ordinals(context); + SortedDocValues singleton = DocValues.unwrapSingleton(docValues); + if (singleton != null) { + return new SingletonOrdinals(singleton); + } + return new Ordinals(docValues); + } + + @Override + public boolean supportsOrdinals() { + return true; + } + + @Override + public SortedSetDocValues ordinals(LeafReaderContext context) throws IOException { + return DocValues.getSortedSet(context.reader(), fieldName); } } @@ -539,12 +523,7 @@ private static class SingletonOrdinals extends BlockDocValuesReader { } @Override - public BytesRefBuilder builder(BuilderFactory factory, int expectedCount) { - return factory.bytesRefsFromDocValues(expectedCount); - } - - @Override - public BlockLoader.Block readValues(BuilderFactory factory, Docs docs) throws IOException { + public BlockLoader.Block read(BlockFactory factory, Docs docs) throws IOException { try (BlockLoader.SingletonOrdinalsBuilder builder = factory.singletonOrdinalsBuilder(ordinals, docs.count())) { for (int i = 0; i < docs.count(); i++) { int doc = docs.get(i); @@ -562,8 +541,8 @@ public BlockLoader.Block readValues(BuilderFactory factory, Docs docs) throws IO } @Override - public void readValuesFromSingleDoc(int doc, Builder builder) throws IOException { - if (ordinals.advanceExact(doc)) { + public void read(int docId, BlockLoader.StoredFields storedFields, Builder builder) throws IOException { + if (ordinals.advanceExact(docId)) { ((BytesRefBuilder) builder).appendBytesRef(ordinals.lookupOrd(ordinals.ordValue())); } else { builder.appendNull(); @@ -571,13 +550,13 @@ public void readValuesFromSingleDoc(int doc, Builder builder) throws IOException } @Override - public int docID() { + public int docId() { return ordinals.docID(); } @Override public String toString() { - return "SingletonOrdinals"; + return "BlockDocValuesReader.SingletonOrdinals"; } } @@ -589,13 +568,8 @@ private static class Ordinals extends BlockDocValuesReader { } @Override - public BytesRefBuilder builder(BuilderFactory factory, int expectedCount) { - return factory.bytesRefsFromDocValues(expectedCount); - } - - @Override - public BlockLoader.Block readValues(BuilderFactory factory, Docs docs) throws IOException { - try (BytesRefBuilder builder = builder(factory, docs.count())) { + public BlockLoader.Block read(BlockFactory factory, Docs docs) throws IOException { + try (BytesRefBuilder builder = factory.bytesRefsFromDocValues(docs.count())) { for (int i = 0; i < docs.count(); i++) { int doc = docs.get(i); if (doc < ordinals.docID()) { @@ -608,12 +582,12 @@ public BlockLoader.Block readValues(BuilderFactory factory, Docs docs) throws IO } @Override - public void readValuesFromSingleDoc(int doc, Builder builder) throws IOException { - read(doc, (BytesRefBuilder) builder); + public void read(int docId, BlockLoader.StoredFields storedFields, Builder builder) throws IOException { + read(docId, (BytesRefBuilder) builder); } - private void read(int doc, BytesRefBuilder builder) throws IOException { - if (false == ordinals.advanceExact(doc)) { + private void read(int docId, BytesRefBuilder builder) throws IOException { + if (false == ordinals.advanceExact(docId)) { builder.appendNull(); return; } @@ -630,32 +604,52 @@ private void read(int doc, BytesRefBuilder builder) throws IOException { } @Override - public int docID() { + public int docId() { return ordinals.docID(); } @Override public String toString() { - return "Ordinals"; + return "BlockDocValuesReader.Ordinals"; } } - private static class Bytes extends BlockDocValuesReader { - private final SortedBinaryDocValues docValues; - private int docID = -1; + public static class BytesRefsFromBinaryBlockLoader extends DocValuesBlockLoader { + private final String fieldName; - Bytes(SortedBinaryDocValues docValues) { - this.docValues = docValues; + public BytesRefsFromBinaryBlockLoader(String fieldName) { + this.fieldName = fieldName; } @Override - public BytesRefBuilder builder(BuilderFactory factory, int expectedCount) { - return factory.bytesRefsFromDocValues(expectedCount); + public Builder builder(BlockFactory factory, int expectedCount) { + return factory.bytesRefs(expectedCount); } @Override - public BlockLoader.Block readValues(BuilderFactory factory, Docs docs) throws IOException { - try (BlockLoader.BytesRefBuilder builder = builder(factory, docs.count())) { + public AllReader reader(LeafReaderContext context) throws IOException { + BinaryDocValues docValues = context.reader().getBinaryDocValues(fieldName); + if (docValues == null) { + return new ConstantNullsReader(); + } + return new BytesRefsFromBinary(docValues); + } + } + + private static class BytesRefsFromBinary extends BlockDocValuesReader { + private final BinaryDocValues docValues; + private final ByteArrayStreamInput in = new ByteArrayStreamInput(); + private final BytesRef scratch = new BytesRef(); + + private int docID = -1; + + BytesRefsFromBinary(BinaryDocValues docValues) { + this.docValues = docValues; + } + + @Override + public BlockLoader.Block read(BlockFactory factory, Docs docs) throws IOException { + try (BlockLoader.BytesRefBuilder builder = factory.bytesRefs(docs.count())) { for (int i = 0; i < docs.count(); i++) { int doc = docs.get(i); if (doc < docID) { @@ -668,7 +662,7 @@ public BlockLoader.Block readValues(BuilderFactory factory, Docs docs) throws IO } @Override - public void readValuesFromSingleDoc(int docId, Builder builder) throws IOException { + public void read(int docId, BlockLoader.StoredFields storedFields, Builder builder) throws IOException { read(docId, (BytesRefBuilder) builder); } @@ -678,27 +672,59 @@ private void read(int doc, BytesRefBuilder builder) throws IOException { builder.appendNull(); return; } - int count = docValues.docValueCount(); + BytesRef bytes = docValues.binaryValue(); + assert bytes.length > 0; + in.reset(bytes.bytes, bytes.offset, bytes.length); + int count = in.readVInt(); + scratch.bytes = bytes.bytes; + if (count == 1) { - // TODO read ords in ascending order. Buffers and stuff. - builder.appendBytesRef(docValues.nextValue()); + scratch.length = in.readVInt(); + scratch.offset = in.getPosition(); + builder.appendBytesRef(scratch); return; } builder.beginPositionEntry(); for (int v = 0; v < count; v++) { - builder.appendBytesRef(docValues.nextValue()); + scratch.length = in.readVInt(); + scratch.offset = in.getPosition(); + in.setPosition(scratch.offset + scratch.length); + builder.appendBytesRef(scratch); } builder.endPositionEntry(); } @Override - public int docID() { + public int docId() { return docID; } @Override public String toString() { - return "Bytes"; + return "BlockDocValuesReader.Bytes"; + } + } + + public static class BooleansBlockLoader extends DocValuesBlockLoader { + private final String fieldName; + + public BooleansBlockLoader(String fieldName) { + this.fieldName = fieldName; + } + + @Override + public BooleanBuilder builder(BlockFactory factory, int expectedCount) { + return factory.booleans(expectedCount); + } + + @Override + public AllReader reader(LeafReaderContext context) throws IOException { + SortedNumericDocValues docValues = DocValues.getSortedNumeric(context.reader(), fieldName); + NumericDocValues singleton = DocValues.unwrapSingleton(docValues); + if (singleton != null) { + return new SingletonBooleans(singleton); + } + return new Booleans(docValues); } } @@ -710,13 +736,8 @@ private static class SingletonBooleans extends BlockDocValuesReader { } @Override - public BooleanBuilder builder(BuilderFactory factory, int expectedCount) { - return factory.booleansFromDocValues(expectedCount); - } - - @Override - public BlockLoader.Block readValues(BuilderFactory factory, Docs docs) throws IOException { - try (BlockLoader.BooleanBuilder builder = builder(factory, docs.count())) { + public BlockLoader.Block read(BlockFactory factory, Docs docs) throws IOException { + try (BlockLoader.BooleanBuilder builder = factory.booleansFromDocValues(docs.count())) { int lastDoc = -1; for (int i = 0; i < docs.count(); i++) { int doc = docs.get(i); @@ -735,7 +756,7 @@ public BlockLoader.Block readValues(BuilderFactory factory, Docs docs) throws IO } @Override - public void readValuesFromSingleDoc(int docId, Builder builder) throws IOException { + public void read(int docId, BlockLoader.StoredFields storedFields, Builder builder) throws IOException { BooleanBuilder blockBuilder = (BooleanBuilder) builder; if (numericDocValues.advanceExact(docId)) { blockBuilder.appendBoolean(numericDocValues.longValue() != 0); @@ -745,13 +766,13 @@ public void readValuesFromSingleDoc(int docId, Builder builder) throws IOExcepti } @Override - public int docID() { + public int docId() { return numericDocValues.docID(); } @Override public String toString() { - return "SingletonBooleans"; + return "BlockDocValuesReader.SingletonBooleans"; } } @@ -764,13 +785,8 @@ private static class Booleans extends BlockDocValuesReader { } @Override - public BooleanBuilder builder(BuilderFactory factory, int expectedCount) { - return factory.booleansFromDocValues(expectedCount); - } - - @Override - public BlockLoader.Block readValues(BuilderFactory factory, Docs docs) throws IOException { - try (BlockLoader.BooleanBuilder builder = builder(factory, docs.count())) { + public BlockLoader.Block read(BlockFactory factory, Docs docs) throws IOException { + try (BlockLoader.BooleanBuilder builder = factory.booleansFromDocValues(docs.count())) { for (int i = 0; i < docs.count(); i++) { int doc = docs.get(i); if (doc < this.docID) { @@ -783,7 +799,7 @@ public BlockLoader.Block readValues(BuilderFactory factory, Docs docs) throws IO } @Override - public void readValuesFromSingleDoc(int docId, Builder builder) throws IOException { + public void read(int docId, BlockLoader.StoredFields storedFields, Builder builder) throws IOException { read(docId, (BooleanBuilder) builder); } @@ -806,61 +822,14 @@ private void read(int doc, BooleanBuilder builder) throws IOException { } @Override - public int docID() { + public int docId() { // There is a .docID on the numericDocValues but it is often not implemented. return docID; } @Override public String toString() { - return "Booleans"; - } - } - - private static class Nulls extends BlockDocValuesReader { - private int docID = -1; - - @Override - public BlockLoader.Builder builder(BuilderFactory factory, int expectedCount) { - return factory.nulls(expectedCount); - } - - @Override - public BlockLoader.Block readValues(BuilderFactory factory, Docs docs) throws IOException { - try (BlockLoader.Builder builder = builder(factory, docs.count())) { - for (int i = 0; i < docs.count(); i++) { - builder.appendNull(); - } - return builder.build(); - } - } - - @Override - public void readValuesFromSingleDoc(int docId, Builder builder) { - this.docID = docId; - builder.appendNull(); - } - - @Override - public int docID() { - return docID; - } - - @Override - public String toString() { - return "Nulls"; - } - } - - /** - * Convert a {@link String} into a utf-8 {@link BytesRef}. - */ - protected static BytesRef toBytesRef(BytesRef scratch, String v) { - int len = UnicodeUtil.maxUTF8Length(v.length()); - if (scratch.bytes.length < len) { - scratch.bytes = new byte[len]; + return "BlockDocValuesReader.Booleans"; } - scratch.length = UnicodeUtil.UTF16toUTF8(v, 0, v.length(), scratch.bytes); - return scratch; } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/BlockLoader.java b/server/src/main/java/org/elasticsearch/index/mapper/BlockLoader.java index af53ab42d35d9..a8f3b919f33cc 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/BlockLoader.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/BlockLoader.java @@ -13,8 +13,12 @@ import org.apache.lucene.index.SortedSetDocValues; import org.apache.lucene.util.BytesRef; import org.elasticsearch.core.Releasable; +import org.elasticsearch.search.fetch.StoredFieldsSpec; +import org.elasticsearch.search.lookup.Source; import java.io.IOException; +import java.util.List; +import java.util.Map; /** * Interface for loading data in a block shape. Instances of this class @@ -22,26 +26,292 @@ */ public interface BlockLoader { /** - * Build a {@link LeafReaderContext leaf} level reader. + * The {@link BlockLoader.Builder} for data of this type. Called when + * loading from a multi-segment or unsorted block. */ - BlockDocValuesReader reader(LeafReaderContext context) throws IOException; + Builder builder(BlockFactory factory, int expectedCount); + + interface Reader { + /** + * Checks if the reader can be used to read a range documents starting with the given docID by the current thread. + */ + boolean canReuse(int startingDocID); + } + + interface ColumnAtATimeReader extends Reader { + /** + * Reads the values of all documents in {@code docs}. + */ + BlockLoader.Block read(BlockFactory factory, Docs docs) throws IOException; + } + + interface RowStrideReader extends Reader { + /** + * Reads the values of the given document into the builder. + */ + void read(int docId, StoredFields storedFields, Builder builder) throws IOException; + } + + interface AllReader extends ColumnAtATimeReader, RowStrideReader {} + + interface StoredFields { + Source source(); + + /** + * @return the ID for the current document + */ + String id(); + + /** + * @return the routing path for the current document + */ + String routing(); + + /** + * @return stored fields for the current document + */ + Map> storedFields(); + } + + ColumnAtATimeReader columnAtATimeReader(LeafReaderContext context) throws IOException; + + RowStrideReader rowStrideReader(LeafReaderContext context) throws IOException; + + StoredFieldsSpec rowStrideStoredFieldSpec(); /** * Does this loader support loading bytes via calling {@link #ordinals}. */ - default boolean supportsOrdinals() { - return false; - } + boolean supportsOrdinals(); /** * Load ordinals for the provided context. */ - default SortedSetDocValues ordinals(LeafReaderContext context) throws IOException { - throw new IllegalStateException("ordinals not supported"); + SortedSetDocValues ordinals(LeafReaderContext context) throws IOException; + + /** + * Load blocks with only null. + */ + BlockLoader CONSTANT_NULLS = new BlockLoader() { + @Override + public Builder builder(BlockFactory factory, int expectedCount) { + return factory.nulls(expectedCount); + } + + @Override + public ColumnAtATimeReader columnAtATimeReader(LeafReaderContext context) { + return new ConstantNullsReader(); + } + + @Override + public RowStrideReader rowStrideReader(LeafReaderContext context) { + return new ConstantNullsReader(); + } + + @Override + public StoredFieldsSpec rowStrideStoredFieldSpec() { + return StoredFieldsSpec.NO_REQUIREMENTS; + } + + @Override + public boolean supportsOrdinals() { + return false; + } + + @Override + public SortedSetDocValues ordinals(LeafReaderContext context) { + throw new UnsupportedOperationException(); + } + + @Override + public String toString() { + return "ConstantNull"; + } + }; + + /** + * Implementation of {@link ColumnAtATimeReader} and {@link RowStrideReader} that always + * loads {@code null}. + */ + class ConstantNullsReader implements AllReader { + @Override + public Block read(BlockFactory factory, Docs docs) throws IOException { + return factory.constantNulls(docs.count()); + } + + @Override + public void read(int docId, StoredFields storedFields, Builder builder) throws IOException { + builder.appendNull(); + } + + @Override + public boolean canReuse(int startingDocID) { + return true; + } + + @Override + public String toString() { + return "constant_nulls"; + } } /** - * A list of documents to load. + * Load blocks with only {@code value}. + */ + static BlockLoader constantBytes(BytesRef value) { + return new BlockLoader() { + @Override + public Builder builder(BlockFactory factory, int expectedCount) { + return factory.bytesRefs(expectedCount); + } + + @Override + public ColumnAtATimeReader columnAtATimeReader(LeafReaderContext context) { + return new ColumnAtATimeReader() { + @Override + public Block read(BlockFactory factory, Docs docs) { + return factory.constantBytes(value, docs.count()); + } + + @Override + public boolean canReuse(int startingDocID) { + return true; + } + + @Override + public String toString() { + return "constant[" + value + "]"; + } + }; + } + + @Override + public RowStrideReader rowStrideReader(LeafReaderContext context) { + return new RowStrideReader() { + @Override + public void read(int docId, StoredFields storedFields, Builder builder) { + ((BlockLoader.BytesRefBuilder) builder).appendBytesRef(value); + } + + @Override + public boolean canReuse(int startingDocID) { + return true; + } + + @Override + public String toString() { + return "constant[" + value + "]"; + } + }; + } + + @Override + public StoredFieldsSpec rowStrideStoredFieldSpec() { + return StoredFieldsSpec.NO_REQUIREMENTS; + } + + @Override + public boolean supportsOrdinals() { + return false; + } + + @Override + public SortedSetDocValues ordinals(LeafReaderContext context) { + throw new UnsupportedOperationException(); + } + + @Override + public String toString() { + return "ConstantBytes[" + value + "]"; + } + }; + } + + abstract class Delegating implements BlockLoader { + protected final BlockLoader delegate; + + protected Delegating(BlockLoader delegate) { + this.delegate = delegate; + } + + @Override + public Builder builder(BlockFactory factory, int expectedCount) { + return delegate.builder(factory, expectedCount); + } + + @Override + public ColumnAtATimeReader columnAtATimeReader(LeafReaderContext context) throws IOException { + ColumnAtATimeReader reader = delegate.columnAtATimeReader(context); + if (reader == null) { + return null; + } + return new ColumnAtATimeReader() { + @Override + public Block read(BlockFactory factory, Docs docs) throws IOException { + return reader.read(factory, docs); + } + + @Override + public boolean canReuse(int startingDocID) { + return reader.canReuse(startingDocID); + } + + @Override + public String toString() { + return "Delegating[to=" + delegatingTo() + ", impl=" + reader + "]"; + } + }; + } + + @Override + public RowStrideReader rowStrideReader(LeafReaderContext context) throws IOException { + RowStrideReader reader = delegate.rowStrideReader(context); + if (reader == null) { + return null; + } + return new RowStrideReader() { + @Override + public void read(int docId, StoredFields storedFields, Builder builder) throws IOException { + reader.read(docId, storedFields, builder); + } + + @Override + public boolean canReuse(int startingDocID) { + return reader.canReuse(startingDocID); + } + + @Override + public String toString() { + return "Delegating[to=" + delegatingTo() + ", impl=" + reader + "]"; + } + }; + } + + @Override + public StoredFieldsSpec rowStrideStoredFieldSpec() { + return delegate.rowStrideStoredFieldSpec(); + } + + @Override + public boolean supportsOrdinals() { + return delegate.supportsOrdinals(); + } + + @Override + public SortedSetDocValues ordinals(LeafReaderContext context) throws IOException { + return delegate.ordinals(context); + } + + protected abstract String delegatingTo(); + + @Override + public final String toString() { + return "Delegating[to=" + delegatingTo() + ", impl=" + delegate + "]"; + } + } + + /** + * A list of documents to load. Documents are always in non-decreasing order. */ interface Docs { int count(); @@ -55,7 +325,7 @@ interface Docs { * production code. That implementation sits in the "compute" project. The is * also a test implementation, but there may be no more other implementations. */ - interface BuilderFactory { + interface BlockFactory { /** * Build a builder to load booleans as loaded from doc values. Doc values * load booleans deduplicated and in sorted order. @@ -112,11 +382,21 @@ interface BuilderFactory { LongBuilder longs(int expectedCount); /** - * Build a builder that can only load null values. - * TODO this should return a block directly instead of a builder + * Build a builder to load only {@code null}s. */ Builder nulls(int expectedCount); + /** + * Build a block that contains only {@code null}. + */ + Block constantNulls(int size); + + /** + * Build a block that contains {@code value} repeated + * {@code size} times. + */ + Block constantBytes(BytesRef value, int size); + /** * Build a reader for reading keyword ordinals. */ @@ -129,7 +409,7 @@ interface BuilderFactory { * Marker interface for block results. The compute engine has a fleshed * out implementation. */ - interface Block {} + interface Block extends Releasable {} /** * A builder for typed values. For each document you may either call diff --git a/server/src/main/java/org/elasticsearch/index/mapper/BlockLoaderStoredFieldsFromLeafLoader.java b/server/src/main/java/org/elasticsearch/index/mapper/BlockLoaderStoredFieldsFromLeafLoader.java new file mode 100644 index 0000000000000..8b1b794f1df55 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/index/mapper/BlockLoaderStoredFieldsFromLeafLoader.java @@ -0,0 +1,54 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.index.mapper; + +import org.elasticsearch.index.fieldvisitor.LeafStoredFieldLoader; +import org.elasticsearch.search.lookup.Source; + +import java.io.IOException; +import java.util.List; +import java.util.Map; + +public class BlockLoaderStoredFieldsFromLeafLoader implements BlockLoader.StoredFields { + private final LeafStoredFieldLoader loader; + private final boolean loadSource; + private Source source; + + public BlockLoaderStoredFieldsFromLeafLoader(LeafStoredFieldLoader loader, boolean loadSource) { + this.loader = loader; + this.loadSource = loadSource; + } + + public void advanceTo(int doc) throws IOException { + loader.advanceTo(doc); + if (loadSource) { + source = Source.fromBytes(loader.source()); + } + } + + @Override + public Source source() { + return source; + } + + @Override + public String id() { + return loader.id(); + } + + @Override + public String routing() { + return loader.routing(); + } + + @Override + public Map> storedFields() { + return loader.storedFields(); + } +} diff --git a/server/src/main/java/org/elasticsearch/index/mapper/BlockSourceReader.java b/server/src/main/java/org/elasticsearch/index/mapper/BlockSourceReader.java index 1261a3612d3cb..289b28949cdab 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/BlockSourceReader.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/BlockSourceReader.java @@ -8,172 +8,32 @@ package org.elasticsearch.index.mapper; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.SortedSetDocValues; import org.apache.lucene.util.BytesRef; -import org.elasticsearch.index.fieldvisitor.LeafStoredFieldLoader; -import org.elasticsearch.index.fieldvisitor.StoredFieldLoader; -import org.elasticsearch.search.lookup.Source; +import org.apache.lucene.util.UnicodeUtil; +import org.elasticsearch.search.fetch.StoredFieldsSpec; import java.io.IOException; import java.util.ArrayList; import java.util.List; -import java.util.Set; /** * Loads values from {@code _source}. This whole process is very slow and cast-tastic, * so it doesn't really try to avoid megamorphic invocations. It's just going to be * slow. - * - * Note that this extends {@link BlockDocValuesReader} because it pretends to load - * doc values because, for now, ESQL only knows how to load things in a doc values - * order. */ -public abstract class BlockSourceReader extends BlockDocValuesReader { - /** - * Read {@code boolean}s from {@code _source}. - */ - public static BlockLoader booleans(ValueFetcher fetcher) { - StoredFieldLoader loader = StoredFieldLoader.create(true, Set.of()); - return context -> new BlockSourceReader(fetcher, loader.getLoader(context, null)) { - @Override - public BlockLoader.Builder builder(BlockLoader.BuilderFactory factory, int expectedCount) { - return factory.booleans(expectedCount); - } - - @Override - protected void append(BlockLoader.Builder builder, Object v) { - ((BlockLoader.BooleanBuilder) builder).appendBoolean((Boolean) v); - } - - @Override - public String toString() { - return "SourceBooleans"; - } - }; - } - - /** - * Read {@link BytesRef}s from {@code _source}. - */ - public static BlockLoader bytesRefs(ValueFetcher fetcher) { - StoredFieldLoader loader = StoredFieldLoader.create(true, Set.of()); - return context -> new BlockSourceReader(fetcher, loader.getLoader(context, null)) { - BytesRef scratch = new BytesRef(); - - @Override - public BlockLoader.Builder builder(BlockLoader.BuilderFactory factory, int expectedCount) { - return factory.bytesRefs(expectedCount); - } - - @Override - protected void append(BlockLoader.Builder builder, Object v) { - ((BlockLoader.BytesRefBuilder) builder).appendBytesRef(toBytesRef(scratch, (String) v)); - } - - @Override - public String toString() { - return "SourceBytes"; - } - }; - } - - /** - * Read {@code double}s from {@code _source}. - */ - public static BlockLoader doubles(ValueFetcher fetcher) { - StoredFieldLoader loader = StoredFieldLoader.create(true, Set.of()); - return context -> new BlockSourceReader(fetcher, loader.getLoader(context, null)) { - @Override - public BlockLoader.Builder builder(BlockLoader.BuilderFactory factory, int expectedCount) { - return factory.doubles(expectedCount); - } - - @Override - protected void append(BlockLoader.Builder builder, Object v) { - ((BlockLoader.DoubleBuilder) builder).appendDouble(((Number) v).doubleValue()); - } - - @Override - public String toString() { - return "SourceDoubles"; - } - }; - } - - /** - * Read {@code int}s from {@code _source}. - */ - public static BlockLoader ints(ValueFetcher fetcher) { - StoredFieldLoader loader = StoredFieldLoader.create(true, Set.of()); - return context -> new BlockSourceReader(fetcher, loader.getLoader(context, null)) { - @Override - public BlockLoader.Builder builder(BlockLoader.BuilderFactory factory, int expectedCount) { - return factory.ints(expectedCount); - } - - @Override - protected void append(BlockLoader.Builder builder, Object v) { - ((BlockLoader.IntBuilder) builder).appendInt(((Number) v).intValue()); - } - - @Override - public String toString() { - return "SourceInts"; - } - }; - } - - /** - * Read {@code long}s from {@code _source}. - */ - public static BlockLoader longs(ValueFetcher fetcher) { - StoredFieldLoader loader = StoredFieldLoader.create(true, Set.of()); - return context -> new BlockSourceReader(fetcher, loader.getLoader(context, null)) { - @Override - public BlockLoader.Builder builder(BlockLoader.BuilderFactory factory, int expectedCount) { - return factory.longs(expectedCount); - } - - @Override - protected void append(BlockLoader.Builder builder, Object v) { - ((BlockLoader.LongBuilder) builder).appendLong(((Number) v).longValue()); - } - - @Override - public String toString() { - return "SourceLongs"; - } - }; - } - +public abstract class BlockSourceReader implements BlockLoader.RowStrideReader { private final ValueFetcher fetcher; - private final LeafStoredFieldLoader loader; private final List ignoredValues = new ArrayList<>(); - private int docID = -1; - BlockSourceReader(ValueFetcher fetcher, LeafStoredFieldLoader loader) { + BlockSourceReader(ValueFetcher fetcher) { this.fetcher = fetcher; - this.loader = loader; - } - - @Override - public BlockLoader.Block readValues(BlockLoader.BuilderFactory factory, BlockLoader.Docs docs) throws IOException { - try (BlockLoader.Builder builder = builder(factory, docs.count())) { - for (int i = 0; i < docs.count(); i++) { - int doc = docs.get(i); - if (doc < this.docID) { - throw new IllegalStateException("docs within same block must be in order"); - } - readValuesFromSingleDoc(doc, builder); - } - return builder.build(); - } } @Override - public void readValuesFromSingleDoc(int doc, BlockLoader.Builder builder) throws IOException { - this.docID = doc; - loader.advanceTo(doc); - List values = fetcher.fetchValues(Source.fromBytes(loader.source()), doc, ignoredValues); + public final void read(int docId, BlockLoader.StoredFields storedFields, BlockLoader.Builder builder) throws IOException { + List values = fetcher.fetchValues(storedFields.source(), docId, ignoredValues); ignoredValues.clear(); // TODO do something with these? if (values == null) { builder.appendNull(); @@ -193,7 +53,213 @@ public void readValuesFromSingleDoc(int doc, BlockLoader.Builder builder) throws protected abstract void append(BlockLoader.Builder builder, Object v); @Override - public int docID() { - return docID; + public boolean canReuse(int startingDocID) { + return true; + } + + private abstract static class SourceBlockLoader implements BlockLoader { + @Override + public final ColumnAtATimeReader columnAtATimeReader(LeafReaderContext context) throws IOException { + return null; + } + + @Override + public final StoredFieldsSpec rowStrideStoredFieldSpec() { + return StoredFieldsSpec.NEEDS_SOURCE; + } + + @Override + public final boolean supportsOrdinals() { + return false; + } + + @Override + public final SortedSetDocValues ordinals(LeafReaderContext context) { + throw new UnsupportedOperationException(); + } + } + + public static class BooleansBlockLoader extends SourceBlockLoader { + private final ValueFetcher fetcher; + + public BooleansBlockLoader(ValueFetcher fetcher) { + this.fetcher = fetcher; + } + + @Override + public Builder builder(BlockFactory factory, int expectedCount) { + return factory.booleans(expectedCount); + } + + @Override + public RowStrideReader rowStrideReader(LeafReaderContext context) { + return new Booleans(fetcher); + } + } + + private static class Booleans extends BlockSourceReader { + Booleans(ValueFetcher fetcher) { + super(fetcher); + } + + @Override + protected void append(BlockLoader.Builder builder, Object v) { + ((BlockLoader.BooleanBuilder) builder).appendBoolean((Boolean) v); + } + + @Override + public String toString() { + return "BlockSourceReader.Booleans"; + } + } + + public static class BytesRefsBlockLoader extends SourceBlockLoader { + private final ValueFetcher fetcher; + + public BytesRefsBlockLoader(ValueFetcher fetcher) { + this.fetcher = fetcher; + } + + @Override + public Builder builder(BlockFactory factory, int expectedCount) { + return factory.bytesRefs(expectedCount); + } + + @Override + public RowStrideReader rowStrideReader(LeafReaderContext context) { + return new BytesRefs(fetcher); + } + } + + private static class BytesRefs extends BlockSourceReader { + BytesRef scratch = new BytesRef(); + + BytesRefs(ValueFetcher fetcher) { + super(fetcher); + } + + @Override + protected void append(BlockLoader.Builder builder, Object v) { + ((BlockLoader.BytesRefBuilder) builder).appendBytesRef(toBytesRef(scratch, (String) v)); + } + + @Override + public String toString() { + return "BlockSourceReader.Bytes"; + } + } + + public static class DoublesBlockLoader extends SourceBlockLoader { + private final ValueFetcher fetcher; + + public DoublesBlockLoader(ValueFetcher fetcher) { + this.fetcher = fetcher; + } + + @Override + public Builder builder(BlockFactory factory, int expectedCount) { + return factory.doubles(expectedCount); + } + + @Override + public RowStrideReader rowStrideReader(LeafReaderContext context) { + return new Doubles(fetcher); + } + } + + private static class Doubles extends BlockSourceReader { + Doubles(ValueFetcher fetcher) { + super(fetcher); + } + + @Override + protected void append(BlockLoader.Builder builder, Object v) { + ((BlockLoader.DoubleBuilder) builder).appendDouble(((Number) v).doubleValue()); + } + + @Override + public String toString() { + return "BlockSourceReader.Doubles"; + } + } + + public static class IntsBlockLoader extends SourceBlockLoader { + private final ValueFetcher fetcher; + + public IntsBlockLoader(ValueFetcher fetcher) { + this.fetcher = fetcher; + } + + @Override + public Builder builder(BlockFactory factory, int expectedCount) { + return factory.ints(expectedCount); + } + + @Override + public RowStrideReader rowStrideReader(LeafReaderContext context) { + return new Ints(fetcher); + } + } + + private static class Ints extends BlockSourceReader { + Ints(ValueFetcher fetcher) { + super(fetcher); + } + + @Override + protected void append(BlockLoader.Builder builder, Object v) { + ((BlockLoader.IntBuilder) builder).appendInt(((Number) v).intValue()); + } + + @Override + public String toString() { + return "BlockSourceReader.Ints"; + } + } + + public static class LongsBlockLoader extends SourceBlockLoader { + private final ValueFetcher fetcher; + + public LongsBlockLoader(ValueFetcher fetcher) { + this.fetcher = fetcher; + } + + @Override + public Builder builder(BlockFactory factory, int expectedCount) { + return factory.longs(expectedCount); + } + + @Override + public RowStrideReader rowStrideReader(LeafReaderContext context) { + return new Longs(fetcher); + } + } + + private static class Longs extends BlockSourceReader { + Longs(ValueFetcher fetcher) { + super(fetcher); + } + + @Override + protected void append(BlockLoader.Builder builder, Object v) { + ((BlockLoader.LongBuilder) builder).appendLong(((Number) v).longValue()); + } + + @Override + public String toString() { + return "BlockSourceReader.Longs"; + } + } + + /** + * Convert a {@link String} into a utf-8 {@link BytesRef}. + */ + static BytesRef toBytesRef(BytesRef scratch, String v) { + int len = UnicodeUtil.maxUTF8Length(v.length()); + if (scratch.bytes.length < len) { + scratch.bytes = new byte[len]; + } + scratch.length = UnicodeUtil.UTF16toUTF8(v, 0, v.length(), scratch.bytes); + return scratch; } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/BlockStoredFieldsReader.java b/server/src/main/java/org/elasticsearch/index/mapper/BlockStoredFieldsReader.java index 5984482fd9441..043ca38b1c78b 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/BlockStoredFieldsReader.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/BlockStoredFieldsReader.java @@ -9,10 +9,11 @@ package org.elasticsearch.index.mapper; import org.apache.lucene.index.LeafReader; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.SortedSetDocValues; import org.apache.lucene.util.BytesRef; -import org.elasticsearch.index.fieldvisitor.LeafStoredFieldLoader; -import org.elasticsearch.index.fieldvisitor.StoredFieldLoader; import org.elasticsearch.index.mapper.BlockLoader.BytesRefBuilder; +import org.elasticsearch.search.fetch.StoredFieldsSpec; import java.io.IOException; import java.util.List; @@ -27,86 +28,101 @@ * doc values because, for now, ESQL only knows how to load things in a doc values * order. */ -public abstract class BlockStoredFieldsReader extends BlockDocValuesReader { - public static BlockLoader bytesRefsFromBytesRefs(String field) { - StoredFieldLoader loader = StoredFieldLoader.create(false, Set.of(field)); - return context -> new Bytes(loader.getLoader(context, null), field) { - @Override - protected BytesRef toBytesRef(Object v) { - return (BytesRef) v; - } - }; +public abstract class BlockStoredFieldsReader implements BlockLoader.RowStrideReader { + @Override + public boolean canReuse(int startingDocID) { + return true; } - public static BlockLoader bytesRefsFromStrings(String field) { - StoredFieldLoader loader = StoredFieldLoader.create(false, Set.of(field)); - return context -> new Bytes(loader.getLoader(context, null), field) { - private final BytesRef scratch = new BytesRef(); + private abstract static class StoredFieldsBlockLoader implements BlockLoader { + protected final String field; - @Override - protected BytesRef toBytesRef(Object v) { - return toBytesRef(scratch, (String) v); - } - }; - } + StoredFieldsBlockLoader(String field) { + this.field = field; + } - public static BlockLoader id() { - StoredFieldLoader loader = StoredFieldLoader.create(false, Set.of(IdFieldMapper.NAME)); - return context -> new Id(loader.getLoader(context, null)); - } + @Override + public final ColumnAtATimeReader columnAtATimeReader(LeafReaderContext context) throws IOException { + return null; + } - private final LeafStoredFieldLoader loader; - private int docID = -1; + @Override + public final StoredFieldsSpec rowStrideStoredFieldSpec() { + return new StoredFieldsSpec(false, false, Set.of(field)); + } - protected BlockStoredFieldsReader(LeafStoredFieldLoader loader) { - this.loader = loader; - } + @Override + public final boolean supportsOrdinals() { + return false; + } - @Override - public final BlockLoader.Block readValues(BlockLoader.BuilderFactory factory, BlockLoader.Docs docs) throws IOException { - try (BlockLoader.Builder builder = builder(factory, docs.count())) { - for (int i = 0; i < docs.count(); i++) { - readValuesFromSingleDoc(docs.get(i), builder); - } - return builder.build(); + @Override + public final SortedSetDocValues ordinals(LeafReaderContext context) { + throw new UnsupportedOperationException(); } } - @Override - public final void readValuesFromSingleDoc(int docId, BlockLoader.Builder builder) throws IOException { - if (docId < this.docID) { - throw new IllegalStateException("docs within same block must be in order"); + /** + * Load {@link BytesRef} blocks from stored {@link BytesRef}s. + */ + public static class BytesFromBytesRefsBlockLoader extends StoredFieldsBlockLoader { + public BytesFromBytesRefsBlockLoader(String field) { + super(field); + } + + @Override + public Builder builder(BlockFactory factory, int expectedCount) { + return factory.bytesRefs(expectedCount); + } + + @Override + public RowStrideReader rowStrideReader(LeafReaderContext context) throws IOException { + return new Bytes(field) { + @Override + protected BytesRef toBytesRef(Object v) { + return (BytesRef) v; + } + }; } - this.docID = docId; - loader.advanceTo(docId); - read(loader, builder); } - protected abstract void read(LeafStoredFieldLoader loader, BlockLoader.Builder builder) throws IOException; + /** + * Load {@link BytesRef} blocks from stored {@link String}s. + */ + public static class BytesFromStringsBlockLoader extends StoredFieldsBlockLoader { + public BytesFromStringsBlockLoader(String field) { + super(field); + } - @Override - public final int docID() { - return docID; + public Builder builder(BlockFactory factory, int expectedCount) { + return factory.bytesRefs(expectedCount); + } + + @Override + public RowStrideReader rowStrideReader(LeafReaderContext context) throws IOException { + return new Bytes(field) { + private final BytesRef scratch = new BytesRef(); + + @Override + protected BytesRef toBytesRef(Object v) { + return BlockSourceReader.toBytesRef(scratch, (String) v); + } + }; + } } private abstract static class Bytes extends BlockStoredFieldsReader { private final String field; - Bytes(LeafStoredFieldLoader loader, String field) { - super(loader); + Bytes(String field) { this.field = field; } - @Override - public BytesRefBuilder builder(BlockLoader.BuilderFactory factory, int expectedCount) { - return factory.bytesRefs(expectedCount); - } - protected abstract BytesRef toBytesRef(Object v); @Override - protected void read(LeafStoredFieldLoader loader, BlockLoader.Builder builder) throws IOException { - List values = loader.storedFields().get(field); + public void read(int docId, BlockLoader.StoredFields storedFields, BlockLoader.Builder builder) throws IOException { + List values = storedFields.storedFields().get(field); if (values == null) { builder.appendNull(); return; @@ -128,21 +144,31 @@ public String toString() { } } - private static class Id extends BlockStoredFieldsReader { - private final BytesRef scratch = new BytesRef(); - - Id(LeafStoredFieldLoader loader) { - super(loader); + /** + * Load {@link BytesRef} blocks from stored {@link String}s. + */ + public static class IdBlockLoader extends StoredFieldsBlockLoader { + public IdBlockLoader() { + super(IdFieldMapper.NAME); } @Override - public BlockLoader.BytesRefBuilder builder(BlockLoader.BuilderFactory factory, int expectedCount) { + public Builder builder(BlockFactory factory, int expectedCount) { return factory.bytesRefs(expectedCount); } @Override - protected void read(LeafStoredFieldLoader loader, BlockLoader.Builder builder) throws IOException { - ((BytesRefBuilder) builder).appendBytesRef(toBytesRef(scratch, loader.id())); + public RowStrideReader rowStrideReader(LeafReaderContext context) throws IOException { + return new Id(); + } + } + + private static class Id extends BlockStoredFieldsReader { + private final BytesRef scratch = new BytesRef(); + + @Override + public void read(int docId, BlockLoader.StoredFields storedFields, BlockLoader.Builder builder) throws IOException { + ((BytesRefBuilder) builder).appendBytesRef(BlockSourceReader.toBytesRef(scratch, storedFields.id())); } @Override diff --git a/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java index a5793df3b82e0..7f175982dc28e 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java @@ -257,9 +257,9 @@ public Boolean valueForDisplay(Object value) { @Override public BlockLoader blockLoader(BlockLoaderContext blContext) { if (hasDocValues()) { - return BlockDocValuesReader.booleans(name()); + return new BlockDocValuesReader.BooleansBlockLoader(name()); } - return BlockSourceReader.booleans(sourceValueFetcher(blContext.sourcePaths(name()))); + return new BlockSourceReader.BooleansBlockLoader(sourceValueFetcher(blContext.sourcePaths(name()))); } @Override diff --git a/server/src/main/java/org/elasticsearch/index/mapper/BooleanScriptBlockDocValuesReader.java b/server/src/main/java/org/elasticsearch/index/mapper/BooleanScriptBlockDocValuesReader.java index b59df56791fbe..953e13dc69eb0 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/BooleanScriptBlockDocValuesReader.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/BooleanScriptBlockDocValuesReader.java @@ -8,14 +8,31 @@ package org.elasticsearch.index.mapper; +import org.apache.lucene.index.LeafReaderContext; import org.elasticsearch.script.BooleanFieldScript; +import java.io.IOException; + /** * {@link BlockDocValuesReader} implementation for {@code boolean} scripts. */ public class BooleanScriptBlockDocValuesReader extends BlockDocValuesReader { - public static BlockLoader blockLoader(BooleanFieldScript.LeafFactory factory) { - return context -> new BooleanScriptBlockDocValuesReader(factory.newInstance(context)); + static class BooleanScriptBlockLoader extends DocValuesBlockLoader { + private final BooleanFieldScript.LeafFactory factory; + + BooleanScriptBlockLoader(BooleanFieldScript.LeafFactory factory) { + this.factory = factory; + } + + @Override + public Builder builder(BlockFactory factory, int expectedCount) { + return factory.booleans(expectedCount); + } + + @Override + public AllReader reader(LeafReaderContext context) throws IOException { + return new BooleanScriptBlockDocValuesReader(factory.newInstance(context)); + } } private final BooleanFieldScript script; @@ -26,19 +43,14 @@ public static BlockLoader blockLoader(BooleanFieldScript.LeafFactory factory) { } @Override - public int docID() { + public int docId() { return docId; } @Override - public BlockLoader.BooleanBuilder builder(BlockLoader.BuilderFactory factory, int expectedCount) { + public BlockLoader.Block read(BlockLoader.BlockFactory factory, BlockLoader.Docs docs) throws IOException { // Note that we don't emit falses before trues so we conform to the doc values contract and can use booleansFromDocValues - return factory.booleansFromDocValues(expectedCount); - } - - @Override - public BlockLoader.Block readValues(BlockLoader.BuilderFactory factory, BlockLoader.Docs docs) { - try (BlockLoader.BooleanBuilder builder = builder(factory, docs.count())) { + try (BlockLoader.BooleanBuilder builder = factory.booleans(docs.count())) { for (int i = 0; i < docs.count(); i++) { read(docs.get(i), builder); } @@ -47,7 +59,7 @@ public BlockLoader.Block readValues(BlockLoader.BuilderFactory factory, BlockLoa } @Override - public void readValuesFromSingleDoc(int docId, BlockLoader.Builder builder) { + public void read(int docId, BlockLoader.StoredFields storedFields, BlockLoader.Builder builder) throws IOException { this.docId = docId; read(docId, (BlockLoader.BooleanBuilder) builder); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/BooleanScriptFieldType.java b/server/src/main/java/org/elasticsearch/index/mapper/BooleanScriptFieldType.java index 6e3876644567f..749bb279cfed4 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/BooleanScriptFieldType.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/BooleanScriptFieldType.java @@ -112,7 +112,7 @@ public DocValueFormat docValueFormat(String format, ZoneId timeZone) { @Override public BlockLoader blockLoader(BlockLoaderContext blContext) { - return BooleanScriptBlockDocValuesReader.blockLoader(leafFactory(blContext.lookup())); + return new BooleanScriptBlockDocValuesReader.BooleanScriptBlockLoader(leafFactory(blContext.lookup())); } @Override diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java index 9d12fc6910d66..e90bea103c4cb 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java @@ -775,9 +775,9 @@ public Function pointReaderIfPossible() { @Override public BlockLoader blockLoader(BlockLoaderContext blContext) { if (hasDocValues()) { - return BlockDocValuesReader.longs(name()); + return new BlockDocValuesReader.LongsBlockLoader(name()); } - return BlockSourceReader.longs(sourceValueFetcher(blContext.sourcePaths(name()))); + return new BlockSourceReader.LongsBlockLoader(sourceValueFetcher(blContext.sourcePaths(name()))); } @Override diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DateScriptBlockDocValuesReader.java b/server/src/main/java/org/elasticsearch/index/mapper/DateScriptBlockDocValuesReader.java index ad630a71870a4..a5303f27573eb 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DateScriptBlockDocValuesReader.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DateScriptBlockDocValuesReader.java @@ -8,14 +8,31 @@ package org.elasticsearch.index.mapper; +import org.apache.lucene.index.LeafReaderContext; import org.elasticsearch.script.DateFieldScript; +import java.io.IOException; + /** * {@link BlockDocValuesReader} implementation for date scripts. */ public class DateScriptBlockDocValuesReader extends BlockDocValuesReader { - public static BlockLoader blockLoader(DateFieldScript.LeafFactory factory) { - return context -> new DateScriptBlockDocValuesReader(factory.newInstance(context)); + static class DateScriptBlockLoader extends DocValuesBlockLoader { + private final DateFieldScript.LeafFactory factory; + + DateScriptBlockLoader(DateFieldScript.LeafFactory factory) { + this.factory = factory; + } + + @Override + public Builder builder(BlockFactory factory, int expectedCount) { + return factory.longs(expectedCount); + } + + @Override + public AllReader reader(LeafReaderContext context) throws IOException { + return new DateScriptBlockDocValuesReader(factory.newInstance(context)); + } } private final DateFieldScript script; @@ -26,18 +43,14 @@ public static BlockLoader blockLoader(DateFieldScript.LeafFactory factory) { } @Override - public int docID() { + public int docId() { return docId; } @Override - public BlockLoader.LongBuilder builder(BlockLoader.BuilderFactory factory, int expectedCount) { - return factory.longs(expectedCount); // Note that we don't pre-sort our output so we can't use longsFromDocValues - } - - @Override - public BlockLoader.Block readValues(BlockLoader.BuilderFactory factory, BlockLoader.Docs docs) { - try (BlockLoader.LongBuilder builder = builder(factory, docs.count())) { + public BlockLoader.Block read(BlockLoader.BlockFactory factory, BlockLoader.Docs docs) throws IOException { + // Note that we don't sort the values sort, so we can't use factory.longsFromDocValues + try (BlockLoader.LongBuilder builder = factory.longs(docs.count())) { for (int i = 0; i < docs.count(); i++) { read(docs.get(i), builder); } @@ -46,7 +59,7 @@ public BlockLoader.Block readValues(BlockLoader.BuilderFactory factory, BlockLoa } @Override - public void readValuesFromSingleDoc(int docId, BlockLoader.Builder builder) { + public void read(int docId, BlockLoader.StoredFields storedFields, BlockLoader.Builder builder) throws IOException { this.docId = docId; read(docId, (BlockLoader.LongBuilder) builder); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DateScriptFieldType.java b/server/src/main/java/org/elasticsearch/index/mapper/DateScriptFieldType.java index 8252d571dce68..238f7488f6b54 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DateScriptFieldType.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DateScriptFieldType.java @@ -181,7 +181,7 @@ public DocValueFormat docValueFormat(@Nullable String format, ZoneId timeZone) { @Override public BlockLoader blockLoader(BlockLoaderContext blContext) { - return DateScriptBlockDocValuesReader.blockLoader(leafFactory(blContext.lookup())); + return new DateScriptBlockDocValuesReader.DateScriptBlockLoader(leafFactory(blContext.lookup())); } @Override diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DoubleScriptBlockDocValuesReader.java b/server/src/main/java/org/elasticsearch/index/mapper/DoubleScriptBlockDocValuesReader.java index 4e317a3ed11cb..a98f5ff661a78 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DoubleScriptBlockDocValuesReader.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DoubleScriptBlockDocValuesReader.java @@ -8,14 +8,31 @@ package org.elasticsearch.index.mapper; +import org.apache.lucene.index.LeafReaderContext; import org.elasticsearch.script.DoubleFieldScript; +import java.io.IOException; + /** * {@link BlockDocValuesReader} implementation for {@code double} scripts. */ public class DoubleScriptBlockDocValuesReader extends BlockDocValuesReader { - public static BlockLoader blockLoader(DoubleFieldScript.LeafFactory factory) { - return context -> new DoubleScriptBlockDocValuesReader(factory.newInstance(context)); + static class DoubleScriptBlockLoader extends DocValuesBlockLoader { + private final DoubleFieldScript.LeafFactory factory; + + DoubleScriptBlockLoader(DoubleFieldScript.LeafFactory factory) { + this.factory = factory; + } + + @Override + public Builder builder(BlockFactory factory, int expectedCount) { + return factory.doubles(expectedCount); + } + + @Override + public AllReader reader(LeafReaderContext context) throws IOException { + return new DoubleScriptBlockDocValuesReader(factory.newInstance(context)); + } } private final DoubleFieldScript script; @@ -26,18 +43,14 @@ public static BlockLoader blockLoader(DoubleFieldScript.LeafFactory factory) { } @Override - public int docID() { + public int docId() { return docId; } @Override - public BlockLoader.DoubleBuilder builder(BlockLoader.BuilderFactory factory, int expectedCount) { - return factory.doubles(expectedCount); // Note that we don't pre-sort our output so we can't use doublesFromDocValues - } - - @Override - public BlockLoader.Block readValues(BlockLoader.BuilderFactory factory, BlockLoader.Docs docs) { - try (BlockLoader.DoubleBuilder builder = builder(factory, docs.count())) { + public BlockLoader.Block read(BlockLoader.BlockFactory factory, BlockLoader.Docs docs) throws IOException { + // Note that we don't sort the values sort, so we can't use factory.doublesFromDocValues + try (BlockLoader.DoubleBuilder builder = factory.doubles(docs.count())) { for (int i = 0; i < docs.count(); i++) { read(docs.get(i), builder); } @@ -46,7 +59,7 @@ public BlockLoader.Block readValues(BlockLoader.BuilderFactory factory, BlockLoa } @Override - public void readValuesFromSingleDoc(int docId, BlockLoader.Builder builder) { + public void read(int docId, BlockLoader.StoredFields storedFields, BlockLoader.Builder builder) throws IOException { this.docId = docId; read(docId, (BlockLoader.DoubleBuilder) builder); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DoubleScriptFieldType.java b/server/src/main/java/org/elasticsearch/index/mapper/DoubleScriptFieldType.java index ef5c112ef212a..c3f7e782c219a 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DoubleScriptFieldType.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DoubleScriptFieldType.java @@ -107,7 +107,7 @@ public DocValueFormat docValueFormat(String format, ZoneId timeZone) { @Override public BlockLoader blockLoader(BlockLoaderContext blContext) { - return DoubleScriptBlockDocValuesReader.blockLoader(leafFactory(blContext.lookup())); + return new DoubleScriptBlockDocValuesReader.DoubleScriptBlockLoader(leafFactory(blContext.lookup())); } @Override diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IndexFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IndexFieldMapper.java index 5f987fd96ca66..1b2667fe9d2ea 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IndexFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IndexFieldMapper.java @@ -80,42 +80,7 @@ public IndexFieldData.Builder fielddataBuilder(FieldDataContext fieldDataContext @Override public BlockLoader blockLoader(BlockLoaderContext blContext) { - // TODO build a constant block directly - BytesRef bytes = new BytesRef(blContext.indexName()); - return context -> new BlockDocValuesReader() { - private int docId; - - @Override - public int docID() { - return docId; - } - - @Override - public BlockLoader.BytesRefBuilder builder(BlockLoader.BuilderFactory factory, int expectedCount) { - return factory.bytesRefs(expectedCount); - } - - @Override - public BlockLoader.Block readValues(BlockLoader.BuilderFactory factory, BlockLoader.Docs docs) { - try (BlockLoader.BytesRefBuilder builder = builder(factory, docs.count())) { - for (int i = 0; i < docs.count(); i++) { - builder.appendBytesRef(bytes); - } - return builder.build(); - } - } - - @Override - public void readValuesFromSingleDoc(int docId, BlockLoader.Builder builder) { - this.docId = docId; - ((BlockLoader.BytesRefBuilder) builder).appendBytesRef(bytes); - } - - @Override - public String toString() { - return "Index"; - } - }; + return BlockLoader.constantBytes(new BytesRef(blContext.indexName())); } @Override diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java index 80fd384f15fb7..56a50c2dee0aa 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java @@ -408,7 +408,7 @@ public static Query rangeQuery( @Override public BlockLoader blockLoader(BlockLoaderContext blContext) { if (hasDocValues()) { - return BlockDocValuesReader.bytesRefsFromOrds(name()); + return new BlockDocValuesReader.BytesRefsFromOrdsBlockLoader(name()); } return null; } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IpScriptBlockDocValuesReader.java b/server/src/main/java/org/elasticsearch/index/mapper/IpScriptBlockDocValuesReader.java index 23229a6533cdb..ff063555ff05d 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IpScriptBlockDocValuesReader.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IpScriptBlockDocValuesReader.java @@ -8,14 +8,31 @@ package org.elasticsearch.index.mapper; +import org.apache.lucene.index.LeafReaderContext; import org.elasticsearch.script.IpFieldScript; +import java.io.IOException; + /** * {@link BlockDocValuesReader} implementation for keyword scripts. */ public class IpScriptBlockDocValuesReader extends BlockDocValuesReader { - public static BlockLoader blockLoader(IpFieldScript.LeafFactory factory) { - return context -> new IpScriptBlockDocValuesReader(factory.newInstance(context)); + static class IpScriptBlockLoader extends DocValuesBlockLoader { + private final IpFieldScript.LeafFactory factory; + + IpScriptBlockLoader(IpFieldScript.LeafFactory factory) { + this.factory = factory; + } + + @Override + public Builder builder(BlockFactory factory, int expectedCount) { + return factory.bytesRefs(expectedCount); + } + + @Override + public AllReader reader(LeafReaderContext context) throws IOException { + return new IpScriptBlockDocValuesReader(factory.newInstance(context)); + } } private final IpFieldScript script; @@ -26,18 +43,14 @@ public static BlockLoader blockLoader(IpFieldScript.LeafFactory factory) { } @Override - public int docID() { + public int docId() { return docId; } @Override - public BlockLoader.BytesRefBuilder builder(BlockLoader.BuilderFactory factory, int expectedCount) { - return factory.bytesRefs(expectedCount); // Note that we don't pre-sort our output so we can't use bytesRefsFromDocValues - } - - @Override - public BlockLoader.Block readValues(BlockLoader.BuilderFactory factory, BlockLoader.Docs docs) { - try (BlockLoader.BytesRefBuilder builder = builder(factory, docs.count())) { + public BlockLoader.Block read(BlockLoader.BlockFactory factory, BlockLoader.Docs docs) throws IOException { + // Note that we don't pre-sort our output so we can't use bytesRefsFromDocValues + try (BlockLoader.BytesRefBuilder builder = factory.bytesRefs(docs.count())) { for (int i = 0; i < docs.count(); i++) { read(docs.get(i), builder); } @@ -46,7 +59,7 @@ public BlockLoader.Block readValues(BlockLoader.BuilderFactory factory, BlockLoa } @Override - public void readValuesFromSingleDoc(int docId, BlockLoader.Builder builder) { + public void read(int docId, BlockLoader.StoredFields storedFields, BlockLoader.Builder builder) throws IOException { this.docId = docId; read(docId, (BlockLoader.BytesRefBuilder) builder); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IpScriptFieldType.java b/server/src/main/java/org/elasticsearch/index/mapper/IpScriptFieldType.java index 0e56b30e2d5d9..4a64184d5d164 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IpScriptFieldType.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IpScriptFieldType.java @@ -211,6 +211,6 @@ private Query cidrQuery(String term, SearchExecutionContext context) { @Override public BlockLoader blockLoader(BlockLoaderContext blContext) { - return IpScriptBlockDocValuesReader.blockLoader(leafFactory(blContext.lookup())); + return new IpScriptBlockDocValuesReader.IpScriptBlockLoader(leafFactory(blContext.lookup())); } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java index f15bb0069570f..caac5b7f3bfe0 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java @@ -580,7 +580,7 @@ NamedAnalyzer normalizer() { @Override public BlockLoader blockLoader(BlockLoaderContext blContext) { if (hasDocValues()) { - return BlockDocValuesReader.bytesRefsFromOrds(name()); + return new BlockDocValuesReader.BytesRefsFromOrdsBlockLoader(name()); } if (isSyntheticSource) { if (false == isStored()) { @@ -590,9 +590,9 @@ public BlockLoader blockLoader(BlockLoaderContext blContext) { + "] is only supported in synthetic _source index if it creates doc values or stored fields" ); } - return BlockStoredFieldsReader.bytesRefsFromBytesRefs(name()); + return new BlockStoredFieldsReader.BytesFromBytesRefsBlockLoader(name()); } - return BlockSourceReader.bytesRefs(sourceValueFetcher(blContext.sourcePaths(name()))); + return new BlockSourceReader.BytesRefsBlockLoader(sourceValueFetcher(blContext.sourcePaths(name()))); } @Override diff --git a/server/src/main/java/org/elasticsearch/index/mapper/KeywordScriptBlockDocValuesReader.java b/server/src/main/java/org/elasticsearch/index/mapper/KeywordScriptBlockDocValuesReader.java index 6afbcae50d31f..df5ba51755c2a 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/KeywordScriptBlockDocValuesReader.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/KeywordScriptBlockDocValuesReader.java @@ -8,15 +8,32 @@ package org.elasticsearch.index.mapper; +import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.util.BytesRefBuilder; import org.elasticsearch.script.StringFieldScript; +import java.io.IOException; + /** * {@link BlockDocValuesReader} implementation for keyword scripts. */ public class KeywordScriptBlockDocValuesReader extends BlockDocValuesReader { - public static BlockLoader blockLoader(StringFieldScript.LeafFactory factory) { - return context -> new KeywordScriptBlockDocValuesReader(factory.newInstance(context)); + static class KeywordScriptBlockLoader extends DocValuesBlockLoader { + private final StringFieldScript.LeafFactory factory; + + KeywordScriptBlockLoader(StringFieldScript.LeafFactory factory) { + this.factory = factory; + } + + @Override + public Builder builder(BlockFactory factory, int expectedCount) { + return factory.bytesRefs(expectedCount); + } + + @Override + public AllReader reader(LeafReaderContext context) throws IOException { + return new KeywordScriptBlockDocValuesReader(factory.newInstance(context)); + } } private final BytesRefBuilder bytesBuild = new BytesRefBuilder(); @@ -28,18 +45,14 @@ public static BlockLoader blockLoader(StringFieldScript.LeafFactory factory) { } @Override - public int docID() { + public int docId() { return docId; } @Override - public BlockLoader.BytesRefBuilder builder(BlockLoader.BuilderFactory factory, int expectedCount) { - return factory.bytesRefs(expectedCount); // Note that we don't pre-sort our output so we can't use bytesRefsFromDocValues - } - - @Override - public BlockLoader.Block readValues(BlockLoader.BuilderFactory factory, BlockLoader.Docs docs) { - try (BlockLoader.BytesRefBuilder builder = builder(factory, docs.count())) { + public BlockLoader.Block read(BlockLoader.BlockFactory factory, BlockLoader.Docs docs) throws IOException { + // Note that we don't pre-sort our output so we can't use bytesRefsFromDocValues + try (BlockLoader.BytesRefBuilder builder = factory.bytesRefs(docs.count())) { for (int i = 0; i < docs.count(); i++) { read(docs.get(i), builder); } @@ -48,7 +61,7 @@ public BlockLoader.Block readValues(BlockLoader.BuilderFactory factory, BlockLoa } @Override - public void readValuesFromSingleDoc(int docId, BlockLoader.Builder builder) { + public void read(int docId, BlockLoader.StoredFields storedFields, BlockLoader.Builder builder) throws IOException { this.docId = docId; read(docId, (BlockLoader.BytesRefBuilder) builder); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/KeywordScriptFieldType.java b/server/src/main/java/org/elasticsearch/index/mapper/KeywordScriptFieldType.java index 879a28d4c76c8..188f0ae508fcc 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/KeywordScriptFieldType.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/KeywordScriptFieldType.java @@ -112,7 +112,7 @@ public Object valueForDisplay(Object value) { @Override public BlockLoader blockLoader(BlockLoaderContext blContext) { - return KeywordScriptBlockDocValuesReader.blockLoader(leafFactory(blContext.lookup())); + return new KeywordScriptBlockDocValuesReader.KeywordScriptBlockLoader(leafFactory(blContext.lookup())); } @Override diff --git a/server/src/main/java/org/elasticsearch/index/mapper/LongScriptBlockDocValuesReader.java b/server/src/main/java/org/elasticsearch/index/mapper/LongScriptBlockDocValuesReader.java index 91c099cd2813b..73ad359147571 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/LongScriptBlockDocValuesReader.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/LongScriptBlockDocValuesReader.java @@ -8,14 +8,31 @@ package org.elasticsearch.index.mapper; +import org.apache.lucene.index.LeafReaderContext; import org.elasticsearch.script.LongFieldScript; +import java.io.IOException; + /** * {@link BlockDocValuesReader} implementation for {@code long} scripts. */ public class LongScriptBlockDocValuesReader extends BlockDocValuesReader { - public static BlockLoader blockLoader(LongFieldScript.LeafFactory factory) { - return context -> new LongScriptBlockDocValuesReader(factory.newInstance(context)); + static class LongScriptBlockLoader extends DocValuesBlockLoader { + private final LongFieldScript.LeafFactory factory; + + LongScriptBlockLoader(LongFieldScript.LeafFactory factory) { + this.factory = factory; + } + + @Override + public Builder builder(BlockFactory factory, int expectedCount) { + return factory.longs(expectedCount); + } + + @Override + public AllReader reader(LeafReaderContext context) throws IOException { + return new LongScriptBlockDocValuesReader(factory.newInstance(context)); + } } private final LongFieldScript script; @@ -26,18 +43,14 @@ public static BlockLoader blockLoader(LongFieldScript.LeafFactory factory) { } @Override - public int docID() { + public int docId() { return docId; } @Override - public BlockLoader.LongBuilder builder(BlockLoader.BuilderFactory factory, int expectedCount) { - return factory.longs(expectedCount); // Note that we don't pre-sort our output so we can't use longsFromDocValues - } - - @Override - public BlockLoader.Block readValues(BlockLoader.BuilderFactory factory, BlockLoader.Docs docs) { - try (BlockLoader.LongBuilder builder = builder(factory, docs.count())) { + public BlockLoader.Block read(BlockLoader.BlockFactory factory, BlockLoader.Docs docs) throws IOException { + // Note that we don't pre-sort our output so we can't use longsFromDocValues + try (BlockLoader.LongBuilder builder = factory.longs(docs.count())) { for (int i = 0; i < docs.count(); i++) { read(docs.get(i), builder); } @@ -46,7 +59,7 @@ public BlockLoader.Block readValues(BlockLoader.BuilderFactory factory, BlockLoa } @Override - public void readValuesFromSingleDoc(int docId, BlockLoader.Builder builder) { + public void read(int docId, BlockLoader.StoredFields storedFields, BlockLoader.Builder builder) throws IOException { this.docId = docId; read(docId, (BlockLoader.LongBuilder) builder); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/LongScriptFieldType.java b/server/src/main/java/org/elasticsearch/index/mapper/LongScriptFieldType.java index f89babe32d0a9..f099ee3625922 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/LongScriptFieldType.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/LongScriptFieldType.java @@ -107,7 +107,7 @@ public DocValueFormat docValueFormat(String format, ZoneId timeZone) { @Override public BlockLoader blockLoader(BlockLoaderContext blContext) { - return LongScriptBlockDocValuesReader.blockLoader(leafFactory(blContext.lookup())); + return new LongScriptBlockDocValuesReader.LongScriptBlockLoader(leafFactory(blContext.lookup())); } @Override diff --git a/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java index 84e9e84fb8ceb..091e3c61764b0 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java @@ -440,12 +440,12 @@ protected void writeValue(XContentBuilder b, long value) throws IOException { @Override BlockLoader blockLoaderFromDocValues(String fieldName) { - return BlockDocValuesReader.doubles(fieldName, l -> HalfFloatPoint.sortableShortToHalfFloat((short) l)); + return new BlockDocValuesReader.DoublesBlockLoader(fieldName, l -> HalfFloatPoint.sortableShortToHalfFloat((short) l)); } @Override BlockLoader blockLoaderFromSource(SourceValueFetcher sourceValueFetcher) { - return BlockSourceReader.doubles(sourceValueFetcher); + return new BlockSourceReader.DoublesBlockLoader(sourceValueFetcher); } }, FLOAT("float", NumericType.FLOAT) { @@ -602,12 +602,12 @@ protected void writeValue(XContentBuilder b, long value) throws IOException { @Override BlockLoader blockLoaderFromDocValues(String fieldName) { - return BlockDocValuesReader.doubles(fieldName, l -> NumericUtils.sortableIntToFloat((int) l)); + return new BlockDocValuesReader.DoublesBlockLoader(fieldName, l -> NumericUtils.sortableIntToFloat((int) l)); } @Override BlockLoader blockLoaderFromSource(SourceValueFetcher sourceValueFetcher) { - return BlockSourceReader.doubles(sourceValueFetcher); + return new BlockSourceReader.DoublesBlockLoader(sourceValueFetcher); } }, DOUBLE("double", NumericType.DOUBLE) { @@ -742,12 +742,12 @@ protected void writeValue(XContentBuilder b, long value) throws IOException { @Override BlockLoader blockLoaderFromDocValues(String fieldName) { - return BlockDocValuesReader.doubles(fieldName, NumericUtils::sortableLongToDouble); + return new BlockDocValuesReader.DoublesBlockLoader(fieldName, NumericUtils::sortableLongToDouble); } @Override BlockLoader blockLoaderFromSource(SourceValueFetcher sourceValueFetcher) { - return BlockSourceReader.doubles(sourceValueFetcher); + return new BlockSourceReader.DoublesBlockLoader(sourceValueFetcher); } }, BYTE("byte", NumericType.BYTE) { @@ -845,12 +845,12 @@ SourceLoader.SyntheticFieldLoader syntheticFieldLoader(String fieldName, String @Override BlockLoader blockLoaderFromDocValues(String fieldName) { - return BlockDocValuesReader.ints(fieldName); + return new BlockDocValuesReader.IntsBlockLoader(fieldName); } @Override BlockLoader blockLoaderFromSource(SourceValueFetcher sourceValueFetcher) { - return BlockSourceReader.ints(sourceValueFetcher); + return new BlockSourceReader.IntsBlockLoader(sourceValueFetcher); } }, SHORT("short", NumericType.SHORT) { @@ -944,12 +944,12 @@ SourceLoader.SyntheticFieldLoader syntheticFieldLoader(String fieldName, String @Override BlockLoader blockLoaderFromDocValues(String fieldName) { - return BlockDocValuesReader.ints(fieldName); + return new BlockDocValuesReader.IntsBlockLoader(fieldName); } @Override BlockLoader blockLoaderFromSource(SourceValueFetcher sourceValueFetcher) { - return BlockSourceReader.ints(sourceValueFetcher); + return new BlockSourceReader.IntsBlockLoader(sourceValueFetcher); } }, INTEGER("integer", NumericType.INT) { @@ -1111,12 +1111,12 @@ SourceLoader.SyntheticFieldLoader syntheticFieldLoader(String fieldName, String @Override BlockLoader blockLoaderFromDocValues(String fieldName) { - return BlockDocValuesReader.ints(fieldName); + return new BlockDocValuesReader.IntsBlockLoader(fieldName); } @Override BlockLoader blockLoaderFromSource(SourceValueFetcher sourceValueFetcher) { - return BlockSourceReader.ints(sourceValueFetcher); + return new BlockSourceReader.IntsBlockLoader(sourceValueFetcher); } }, LONG("long", NumericType.LONG) { @@ -1248,12 +1248,12 @@ SourceLoader.SyntheticFieldLoader syntheticFieldLoader(String fieldName, String @Override BlockLoader blockLoaderFromDocValues(String fieldName) { - return BlockDocValuesReader.longs(fieldName); + return new BlockDocValuesReader.LongsBlockLoader(fieldName); } @Override BlockLoader blockLoaderFromSource(SourceValueFetcher sourceValueFetcher) { - return BlockSourceReader.longs(sourceValueFetcher); + return new BlockSourceReader.LongsBlockLoader(sourceValueFetcher); } }; @@ -1656,7 +1656,7 @@ public Function pointReaderIfPossible() { public BlockLoader blockLoader(BlockLoaderContext blContext) { if (indexMode == IndexMode.TIME_SERIES && metricType == TimeSeriesParams.MetricType.COUNTER) { // Counters are not supported by ESQL so we load them in null - return BlockDocValuesReader.nulls(); + return BlockLoader.CONSTANT_NULLS; } if (hasDocValues()) { return type.blockLoaderFromDocValues(name()); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/ProvidedIdFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/ProvidedIdFieldMapper.java index f681d54ebbead..d8a4177ee3211 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/ProvidedIdFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/ProvidedIdFieldMapper.java @@ -119,7 +119,7 @@ public Query termsQuery(Collection values, SearchExecutionContext context) { @Override public BlockLoader blockLoader(BlockLoaderContext blContext) { - return BlockStoredFieldsReader.id(); + return new BlockStoredFieldsReader.IdBlockLoader(); } @Override diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java index 5a0d9c7c0cf79..420f92cfbf847 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java @@ -678,7 +678,7 @@ public TextFieldType( super(name, indexed, stored, false, tsi, meta); fielddata = false; this.isSyntheticSource = isSyntheticSource; - this.syntheticSourceDelegate = syntheticSourceDelegate; + this.syntheticSourceDelegate = syntheticSourceDelegate; // TODO rename to "exactDelegate" or something this.eagerGlobalOrdinals = eagerGlobalOrdinals; this.indexPhrases = indexPhrases; } @@ -939,11 +939,16 @@ public boolean isAggregatable() { @Override public BlockLoader blockLoader(BlockLoaderContext blContext) { if (syntheticSourceDelegate != null) { - return syntheticSourceDelegate.blockLoader(blContext); + return new BlockLoader.Delegating(syntheticSourceDelegate.blockLoader(blContext)) { + @Override + protected String delegatingTo() { + return syntheticSourceDelegate.name(); + } + }; } if (isSyntheticSource) { if (isStored()) { - return BlockStoredFieldsReader.bytesRefsFromStrings(name()); + return new BlockStoredFieldsReader.BytesFromStringsBlockLoader(name()); } /* * We *shouldn't fall to this exception. The mapping should be @@ -957,7 +962,7 @@ public BlockLoader blockLoader(BlockLoaderContext blContext) { + "] is not supported because synthetic _source is enabled and we don't have a way to load the fields" ); } - return BlockSourceReader.bytesRefs(SourceValueFetcher.toString(blContext.sourcePaths(name()))); + return new BlockSourceReader.BytesRefsBlockLoader(SourceValueFetcher.toString(blContext.sourcePaths(name()))); } @Override @@ -1034,6 +1039,10 @@ protected BytesRef storedToBytesRef(Object stored) { public boolean isSyntheticSource() { return isSyntheticSource; } + + KeywordFieldMapper.KeywordFieldType syntheticSourceDelegate() { + return syntheticSourceDelegate; + } } public static class ConstantScoreTextFieldType extends TextFieldType { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapper.java index 9d43ef398feac..9245e78602eb7 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapper.java @@ -89,7 +89,7 @@ public Query termsQuery(Collection values, SearchExecutionContext context) { @Override public BlockLoader blockLoader(BlockLoaderContext blContext) { - return BlockStoredFieldsReader.id(); + return new BlockStoredFieldsReader.IdBlockLoader(); } @Override diff --git a/server/src/main/java/org/elasticsearch/index/mapper/VersionFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/VersionFieldMapper.java index 54a44dd55caa4..8f69f6afe47db 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/VersionFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/VersionFieldMapper.java @@ -56,7 +56,7 @@ public ValueFetcher valueFetcher(SearchExecutionContext context, String format) @Override public BlockLoader blockLoader(BlockLoaderContext blContext) { - return BlockDocValuesReader.longs(name()); + return new BlockDocValuesReader.LongsBlockLoader(name()); } @Override diff --git a/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java b/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java index 8b6f6afb72042..fedd84ee7392b 100644 --- a/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java +++ b/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java @@ -2006,7 +2006,7 @@ private void innerOpenEngineAndTranslog(LongSupplier globalCheckpointSupplier) t assert currentEngineReference.get() == null : "engine is running"; verifyNotClosed(); // we must create a new engine under mutex (see IndexShard#snapshotStoreMetadata). - final Engine newEngine = engineFactory.newReadWriteEngine(config); + final Engine newEngine = createEngine(config); onNewEngine(newEngine); currentEngineReference.set(newEngine); // We set active because we are now writing operations to the engine; this way, @@ -2021,6 +2021,22 @@ private void innerOpenEngineAndTranslog(LongSupplier globalCheckpointSupplier) t checkAndCallWaitForEngineOrClosedShardListeners(); } + // awful hack to work around problem in CloseFollowerIndexIT + static boolean suppressCreateEngineErrors; + + private Engine createEngine(EngineConfig config) { + if (suppressCreateEngineErrors) { + try { + return engineFactory.newReadWriteEngine(config); + } catch (Error e) { + ExceptionsHelper.maybeDieOnAnotherThread(e); + throw new RuntimeException("rethrowing suppressed error", e); + } + } else { + return engineFactory.newReadWriteEngine(config); + } + } + private boolean assertSequenceNumbersInCommit() throws IOException { final SegmentInfos segmentCommitInfos = SegmentInfos.readLatestCommit(store.directory()); final Map userData = segmentCommitInfos.getUserData(); diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestForceMergeAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestForceMergeAction.java index a04e23f289379..4c9ac8fcb9a3c 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestForceMergeAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestForceMergeAction.java @@ -13,7 +13,7 @@ import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeRequest; import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeResponse; import org.elasticsearch.action.support.IndicesOptions; -import org.elasticsearch.action.support.ListenableActionFuture; +import org.elasticsearch.action.support.SubscribableListener; import org.elasticsearch.client.internal.node.NodeClient; import org.elasticsearch.common.Strings; import org.elasticsearch.rest.BaseRestHandler; @@ -65,9 +65,9 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC if (validationException != null) { throw validationException; } - final var responseFuture = new ListenableActionFuture(); - final var task = client.executeLocally(ForceMergeAction.INSTANCE, mergeRequest, responseFuture); - responseFuture.addListener(new LoggingTaskListener<>(task)); + final var responseListener = new SubscribableListener(); + final var task = client.executeLocally(ForceMergeAction.INSTANCE, mergeRequest, responseListener); + responseListener.addListener(new LoggingTaskListener<>(task)); return sendTask(client.getLocalNodeId(), task); } } diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetAliasesAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetAliasesAction.java index b6e1240a3f85a..a8f6fa325b468 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetAliasesAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetAliasesAction.java @@ -21,6 +21,7 @@ import org.elasticsearch.common.regex.Regex; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.core.RestApiVersion; +import org.elasticsearch.core.UpdateForV9; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.RestResponse; @@ -50,6 +51,7 @@ @ServerlessScope(Scope.PUBLIC) public class RestGetAliasesAction extends BaseRestHandler { + @UpdateForV9 // reject the deprecated ?local parameter private static final DeprecationLogger DEPRECATION_LOGGER = DeprecationLogger.getLogger(RestGetAliasesAction.class); @Override diff --git a/server/src/main/java/org/elasticsearch/search/fetch/StoredFieldsSpec.java b/server/src/main/java/org/elasticsearch/search/fetch/StoredFieldsSpec.java index 48aea98887ff0..87cbf9b1d6b85 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/StoredFieldsSpec.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/StoredFieldsSpec.java @@ -38,6 +38,9 @@ public boolean noRequirements() { * Combine these stored field requirements with those from another StoredFieldsSpec */ public StoredFieldsSpec merge(StoredFieldsSpec other) { + if (this == other) { + return this; + } Set mergedFields = new HashSet<>(this.requiredStoredFields); mergedFields.addAll(other.requiredStoredFields); return new StoredFieldsSpec( diff --git a/server/src/main/java/org/elasticsearch/tasks/LoggingTaskListener.java b/server/src/main/java/org/elasticsearch/tasks/LoggingTaskListener.java index c99194d933131..63e17bd62f8ee 100644 --- a/server/src/main/java/org/elasticsearch/tasks/LoggingTaskListener.java +++ b/server/src/main/java/org/elasticsearch/tasks/LoggingTaskListener.java @@ -31,7 +31,6 @@ public LoggingTaskListener(Task task) { @Override public void onResponse(Response response) { logger.info("{} finished with response {}", task.getId(), response); - } @Override diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/alias/get/GetAliasesResponseTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/alias/get/GetAliasesResponseTests.java index 6fde4bed97a17..433563b99ef64 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/alias/get/GetAliasesResponseTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/alias/get/GetAliasesResponseTests.java @@ -15,6 +15,7 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.core.Tuple; +import org.elasticsearch.core.UpdateForV9; import org.elasticsearch.test.AbstractWireSerializingTestCase; import java.util.ArrayList; @@ -24,6 +25,7 @@ import java.util.Map; import java.util.function.Predicate; +@UpdateForV9 // no need to round-trip these objects over the wire any more, we only need a checkEqualsAndHashCode test public class GetAliasesResponseTests extends AbstractWireSerializingTestCase { @Override @@ -33,9 +35,8 @@ protected GetAliasesResponse createTestInstance() { /** * NB prior to 8.12 get-aliases was a TransportMasterNodeReadAction so for BwC we must remain able to write these responses so that - * older nodes can read them until we no longer need to support {@link org.elasticsearch.TransportVersions#CLUSTER_FEATURES_ADDED} and - * earlier. The reader implementation below is the production implementation from earlier versions, but moved here because it is unused - * in production now. + * older nodes can read them until we no longer need to support calling this action remotely. The reader implementation below is the + * production implementation from earlier versions, but moved here because it is unused in production now. */ @Override protected Writeable.Reader instanceReader() { diff --git a/server/src/test/java/org/elasticsearch/cluster/coordination/LinearizabilityCheckerTests.java b/server/src/test/java/org/elasticsearch/cluster/coordination/LinearizabilityCheckerTests.java index 34fe7eae32fcb..83df7e7d18f5c 100644 --- a/server/src/test/java/org/elasticsearch/cluster/coordination/LinearizabilityCheckerTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/coordination/LinearizabilityCheckerTests.java @@ -143,6 +143,27 @@ public void testRegisterWithLinearizableHistory() throws LinearizabilityCheckAbo assertTrue(LinearizabilityChecker.isLinearizable(registerSpec, history)); } + public void testRegisterHistoryVisualisation() { + final History history = new History(); + int write0 = history.invoke(42); // invoke write(42) + history.respond(history.invoke(null), 42); // read, returns 42 + history.respond(write0, null); // write(42) succeeds + + int write1 = history.invoke(24); // invoke write 24 + history.respond(history.invoke(null), 42); // read returns 42 + history.respond(history.invoke(null), 24); // subsequent read returns 24 + history.respond(write1, null); // write(24) succeeds + + assertEquals(""" + Partition 0 + 42 XXX null (0) + null X 42 (1) + 24 XXXXX null (2) + null X 42 (3) + null X 24 (4) + """, LinearizabilityChecker.visualize(registerSpec, history, o -> { throw new AssertionError("history was complete"); })); + } + public void testRegisterWithNonLinearizableHistory() throws LinearizabilityCheckAborted { final History history = new History(); int call0 = history.invoke(42); // 0: invoke write 42 diff --git a/server/src/test/java/org/elasticsearch/common/util/BytesRefArrayTests.java b/server/src/test/java/org/elasticsearch/common/util/BytesRefArrayTests.java index 0ca6bf86ceec7..e7868f442efd0 100644 --- a/server/src/test/java/org/elasticsearch/common/util/BytesRefArrayTests.java +++ b/server/src/test/java/org/elasticsearch/common/util/BytesRefArrayTests.java @@ -17,6 +17,9 @@ import org.elasticsearch.test.ESTestCase; import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.IntStream; import static org.hamcrest.Matchers.equalTo; @@ -115,6 +118,64 @@ public void testLookup() throws IOException { } } + public void testReadWritten() { + testReadWritten(false); + } + + public void testReadWrittenHalfEmpty() { + testReadWritten(true); + } + + private void testReadWritten(boolean halfEmpty) { + List values = new ArrayList<>(); + int bytes = PageCacheRecycler.PAGE_SIZE_IN_BYTES * between(2, 20); + int used = 0; + while (used < bytes) { + String str = halfEmpty && randomBoolean() ? "" : randomAlphaOfLengthBetween(0, 200); + BytesRef v = new BytesRef(str); + used += v.length; + values.add(v); + } + testReadWritten(values, randomBoolean() ? bytes : between(0, bytes)); + } + + public void testReadWrittenRepeated() { + testReadWrittenRepeated(false, between(2, 3000)); + } + + public void testReadWrittenRepeatedPowerOfTwo() { + testReadWrittenRepeated(false, 1024); + } + + public void testReadWrittenRepeatedHalfEmpty() { + testReadWrittenRepeated(true, between(1, 3000)); + } + + public void testReadWrittenRepeatedHalfEmptyPowerOfTwo() { + testReadWrittenRepeated(true, 1024); + } + + public void testReadWrittenRepeated(boolean halfEmpty, int listSize) { + List values = randomList(2, 10, () -> { + String str = halfEmpty && randomBoolean() ? "" : randomAlphaOfLengthBetween(0, 10); + return new BytesRef(str); + }); + testReadWritten(IntStream.range(0, listSize).mapToObj(i -> values).flatMap(List::stream).toList(), 10); + } + + private void testReadWritten(List values, int initialCapacity) { + try (BytesRefArray array = new BytesRefArray(initialCapacity, mockBigArrays())) { + for (BytesRef v : values) { + array.append(v); + } + BytesRef scratch = new BytesRef(); + for (int i = 0; i < values.size(); i++) { + array.get(i, scratch); + assertThat(scratch, equalTo(values.get(i))); + } + } + } + private static BigArrays mockBigArrays() { return new MockBigArrays(new MockPageCacheRecycler(Settings.EMPTY), new NoneCircuitBreakerService()); } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/BooleanScriptFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/BooleanScriptFieldTypeTests.java index 8d5a47f08c663..d8f063ece35c0 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/BooleanScriptFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/BooleanScriptFieldTypeTests.java @@ -417,8 +417,8 @@ public void testBlockLoader() throws IOException { try (DirectoryReader reader = iw.getReader()) { BooleanScriptFieldType fieldType = build("xor_param", Map.of("param", false), OnScriptError.FAIL); List expected = List.of(false, true); - assertThat(blockLoaderReadValues(reader, fieldType), equalTo(expected)); - assertThat(blockLoaderReadValuesFromSingleDoc(reader, fieldType), equalTo(expected)); + assertThat(blockLoaderReadValuesFromColumnAtATimeReader(reader, fieldType), equalTo(expected)); + assertThat(blockLoaderReadValuesFromRowStrideReader(reader, fieldType), equalTo(expected)); } } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DateScriptFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DateScriptFieldTypeTests.java index d1652b9f57716..eb3daf472ea2e 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DateScriptFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DateScriptFieldTypeTests.java @@ -477,8 +477,11 @@ public void testBlockLoader() throws IOException { iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"timestamp\": [1595432181355]}")))); try (DirectoryReader reader = iw.getReader()) { DateScriptFieldType fieldType = build("add_days", Map.of("days", 1), OnScriptError.FAIL); - assertThat(blockLoaderReadValues(reader, fieldType), equalTo(List.of(1595518581354L, 1595518581355L))); - assertThat(blockLoaderReadValuesFromSingleDoc(reader, fieldType), equalTo(List.of(1595518581354L, 1595518581355L))); + assertThat( + blockLoaderReadValuesFromColumnAtATimeReader(reader, fieldType), + equalTo(List.of(1595518581354L, 1595518581355L)) + ); + assertThat(blockLoaderReadValuesFromRowStrideReader(reader, fieldType), equalTo(List.of(1595518581354L, 1595518581355L))); } } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DoubleScriptFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DoubleScriptFieldTypeTests.java index 0f05dad8098f4..d37e42e04edca 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DoubleScriptFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DoubleScriptFieldTypeTests.java @@ -236,8 +236,8 @@ public void testBlockLoader() throws IOException { iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": [2]}")))); try (DirectoryReader reader = iw.getReader()) { DoubleScriptFieldType fieldType = build("add_param", Map.of("param", 1), OnScriptError.FAIL); - assertThat(blockLoaderReadValues(reader, fieldType), equalTo(List.of(2d, 3d))); - assertThat(blockLoaderReadValuesFromSingleDoc(reader, fieldType), equalTo(List.of(2d, 3d))); + assertThat(blockLoaderReadValuesFromColumnAtATimeReader(reader, fieldType), equalTo(List.of(2d, 3d))); + assertThat(blockLoaderReadValuesFromRowStrideReader(reader, fieldType), equalTo(List.of(2d, 3d))); } } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IpScriptFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IpScriptFieldTypeTests.java index 56ca5f3dae89f..cd19bb50b842c 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/IpScriptFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/IpScriptFieldTypeTests.java @@ -256,8 +256,8 @@ public void testBlockLoader() throws IOException { new BytesRef(InetAddressPoint.encode(InetAddresses.forString("192.168.0.1"))), new BytesRef(InetAddressPoint.encode(InetAddresses.forString("192.168.1.1"))) ); - assertThat(blockLoaderReadValues(reader, fieldType), equalTo(expected)); - assertThat(blockLoaderReadValuesFromSingleDoc(reader, fieldType), equalTo(expected)); + assertThat(blockLoaderReadValuesFromColumnAtATimeReader(reader, fieldType), equalTo(expected)); + assertThat(blockLoaderReadValuesFromRowStrideReader(reader, fieldType), equalTo(expected)); } } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/KeywordScriptFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/KeywordScriptFieldTypeTests.java index 65f4c2e3ea6eb..ce705f2e9ae8b 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/KeywordScriptFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/KeywordScriptFieldTypeTests.java @@ -382,9 +382,12 @@ public void testBlockLoader() throws IOException { iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": [2]}")))); try (DirectoryReader reader = iw.getReader()) { KeywordScriptFieldType fieldType = build("append_param", Map.of("param", "-Suffix"), OnScriptError.FAIL); - assertThat(blockLoaderReadValues(reader, fieldType), equalTo(List.of(new BytesRef("1-Suffix"), new BytesRef("2-Suffix")))); assertThat( - blockLoaderReadValuesFromSingleDoc(reader, fieldType), + blockLoaderReadValuesFromColumnAtATimeReader(reader, fieldType), + equalTo(List.of(new BytesRef("1-Suffix"), new BytesRef("2-Suffix"))) + ); + assertThat( + blockLoaderReadValuesFromRowStrideReader(reader, fieldType), equalTo(List.of(new BytesRef("1-Suffix"), new BytesRef("2-Suffix"))) ); } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/LongScriptFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/LongScriptFieldTypeTests.java index 1688cab24af3e..fd20b6c71e984 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/LongScriptFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/LongScriptFieldTypeTests.java @@ -269,8 +269,8 @@ public void testBlockLoader() throws IOException { iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": [2]}")))); try (DirectoryReader reader = iw.getReader()) { LongScriptFieldType fieldType = build("add_param", Map.of("param", 1), OnScriptError.FAIL); - assertThat(blockLoaderReadValues(reader, fieldType), equalTo(List.of(2L, 3L))); - assertThat(blockLoaderReadValuesFromSingleDoc(reader, fieldType), equalTo(List.of(2L, 3L))); + assertThat(blockLoaderReadValuesFromColumnAtATimeReader(reader, fieldType), equalTo(List.of(2L, 3L))); + assertThat(blockLoaderReadValuesFromRowStrideReader(reader, fieldType), equalTo(List.of(2L, 3L))); } } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java index bbfeaaa8b9d69..b2a729d6868d2 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java @@ -1324,4 +1324,10 @@ public void testEmpty() throws Exception { assertFalse(dv.advanceExact(3)); }); } + + @Override + protected boolean supportsColumnAtATimeReader(MappedFieldType ft) { + TextFieldMapper.TextFieldType text = (TextFieldType) ft; + return text.syntheticSourceDelegate() != null && text.syntheticSourceDelegate().hasDocValues(); + } } diff --git a/test/framework/src/main/java/org/elasticsearch/cluster/coordination/LinearizabilityChecker.java b/test/framework/src/main/java/org/elasticsearch/cluster/coordination/LinearizabilityChecker.java index 223b0dc5a546b..6c43eff24be21 100644 --- a/test/framework/src/main/java/org/elasticsearch/cluster/coordination/LinearizabilityChecker.java +++ b/test/framework/src/main/java/org/elasticsearch/cluster/coordination/LinearizabilityChecker.java @@ -17,6 +17,9 @@ import org.elasticsearch.threadpool.Scheduler; import org.elasticsearch.threadpool.ThreadPool; +import java.io.IOException; +import java.io.StringWriter; +import java.io.Writer; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -35,7 +38,6 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.function.BooleanSupplier; -import java.util.function.Consumer; import java.util.function.Function; /** @@ -308,25 +310,38 @@ private static boolean isLinearizable(SequentialSpec spec, List history, * Return a visual representation of the history */ public static String visualize(SequentialSpec spec, History history, Function missingResponseGenerator) { + final var writer = new StringWriter(); + writeVisualisation(spec, history, missingResponseGenerator, writer); + return writer.toString(); + } + + /** + * Write a visual representation of the history to the given writer + */ + public static void writeVisualisation( + SequentialSpec spec, + History history, + Function missingResponseGenerator, + Writer writer + ) { history = history.clone(); history.complete(missingResponseGenerator); final Collection> partitions = spec.partition(history.copyEvents()); - StringBuilder builder = new StringBuilder(); - partitions.forEach(new Consumer>() { + try { int index = 0; - - @Override - public void accept(List events) { - builder.append("Partition ").append(index++).append("\n"); - builder.append(visualizePartition(events)); + for (List partition : partitions) { + writer.write("Partition "); + writer.write(Integer.toString(index++)); + writer.append('\n'); + visualizePartition(partition, writer); } - }); - - return builder.toString(); + } catch (IOException e) { + logger.error("unexpected writeVisualisation failure", e); + assert false : e; // not really doing any IO + } } - private static String visualizePartition(List events) { - StringBuilder builder = new StringBuilder(); + private static void visualizePartition(List events, Writer writer) throws IOException { Entry entry = createLinkedEntries(events).next; Map, Integer> eventToPosition = new HashMap<>(); for (Event event : events) { @@ -334,28 +349,30 @@ private static String visualizePartition(List events) { } while (entry != null) { if (entry.match != null) { - builder.append(visualizeEntry(entry, eventToPosition)).append("\n"); + visualizeEntry(entry, eventToPosition, writer); + writer.append('\n'); } entry = entry.next; } - return builder.toString(); } - private static String visualizeEntry(Entry entry, Map, Integer> eventToPosition) { + private static void visualizeEntry(Entry entry, Map, Integer> eventToPosition, Writer writer) + throws IOException { + String input = String.valueOf(entry.event.value); String output = String.valueOf(entry.match.event.value); int id = entry.event.id; int beginIndex = eventToPosition.get(Tuple.tuple(EventType.INVOCATION, id)); int endIndex = eventToPosition.get(Tuple.tuple(EventType.RESPONSE, id)); input = input.substring(0, Math.min(beginIndex + 25, input.length())); - return Strings.padStart(input, beginIndex + 25, ' ') - + " " - + Strings.padStart("", endIndex - beginIndex, 'X') - + " " - + output - + " (" - + entry.event.id - + ")"; + writer.write(Strings.padStart(input, beginIndex + 25, ' ')); + writer.write(" "); + writer.write(Strings.padStart("", endIndex - beginIndex, 'X')); + writer.write(" "); + writer.write(output); + writer.write(" ("); + writer.write(Integer.toString(entry.event.id)); + writer.write(")"); } /** diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/AbstractScriptFieldTypeTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/AbstractScriptFieldTypeTestCase.java index 56ad35bee83d5..7eb2511f58206 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/AbstractScriptFieldTypeTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/AbstractScriptFieldTypeTestCase.java @@ -383,11 +383,12 @@ public final void testCacheable() throws IOException { } } - protected final List blockLoaderReadValues(DirectoryReader reader, MappedFieldType fieldType) throws IOException { + protected final List blockLoaderReadValuesFromColumnAtATimeReader(DirectoryReader reader, MappedFieldType fieldType) + throws IOException { BlockLoader loader = fieldType.blockLoader(blContext()); List all = new ArrayList<>(); for (LeafReaderContext ctx : reader.leaves()) { - TestBlock block = (TestBlock) loader.reader(ctx).readValues(TestBlock.FACTORY, TestBlock.docs(ctx)); + TestBlock block = (TestBlock) loader.columnAtATimeReader(ctx).read(TestBlock.FACTORY, TestBlock.docs(ctx)); for (int i = 0; i < block.size(); i++) { all.add(block.get(i)); } @@ -395,15 +396,17 @@ protected final List blockLoaderReadValues(DirectoryReader reader, Mappe return all; } - protected final List blockLoaderReadValuesFromSingleDoc(DirectoryReader reader, MappedFieldType fieldType) throws IOException { + protected final List blockLoaderReadValuesFromRowStrideReader(DirectoryReader reader, MappedFieldType fieldType) + throws IOException { BlockLoader loader = fieldType.blockLoader(blContext()); List all = new ArrayList<>(); for (LeafReaderContext ctx : reader.leaves()) { - BlockDocValuesReader blockReader = loader.reader(ctx); - TestBlock block = (TestBlock) blockReader.builder(TestBlock.FACTORY, ctx.reader().numDocs()); + BlockLoader.RowStrideReader blockReader = loader.rowStrideReader(ctx); + BlockLoader.Builder builder = loader.builder(TestBlock.FACTORY, ctx.reader().numDocs()); for (int i = 0; i < ctx.reader().numDocs(); i++) { - blockReader.readValuesFromSingleDoc(i, block); + blockReader.read(i, null, builder); } + TestBlock block = (TestBlock) builder.build(); for (int i = 0; i < block.size(); i++) { all.add(block.get(i)); } diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java index e34072fbf1668..d68324ff902e2 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java @@ -31,7 +31,6 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.core.CheckedConsumer; -import org.elasticsearch.core.CheckedFunction; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.IndexVersions; @@ -1240,19 +1239,19 @@ public final void testSyntheticEmptyListNoDocValuesLoader() throws IOException { assertNoDocValueLoader(b -> b.startArray("field").endArray()); } - public final void testBlockLoaderReadValues() throws IOException { - testBlockLoader(blockReader -> (TestBlock) blockReader.readValues(TestBlock.FACTORY, TestBlock.docs(0))); + public final void testBlockLoaderFromColumnReader() throws IOException { + testBlockLoader(true); } - public final void testBlockLoaderReadValuesFromSingleDoc() throws IOException { - testBlockLoader(blockReader -> { - TestBlock block = (TestBlock) blockReader.builder(TestBlock.FACTORY, 1); - blockReader.readValuesFromSingleDoc(0, block); - return block; - }); + public final void testBlockLoaderFromRowStrideReader() throws IOException { + testBlockLoader(false); + } + + protected boolean supportsColumnAtATimeReader(MappedFieldType ft) { + return ft.hasDocValues(); } - private void testBlockLoader(CheckedFunction body) throws IOException { + private void testBlockLoader(boolean columnReader) throws IOException { SyntheticSourceExample example = syntheticSourceSupport(false).example(5); MapperService mapper = createMapperService(syntheticSourceMapping(b -> { b.startObject("field"); @@ -1289,7 +1288,25 @@ public Set sourcePaths(String name) { iw.addDocument(doc); iw.close(); try (DirectoryReader reader = DirectoryReader.open(directory)) { - TestBlock block = body.apply(loader.reader(reader.leaves().get(0))); + LeafReaderContext ctx = reader.leaves().get(0); + TestBlock block; + if (columnReader) { + if (supportsColumnAtATimeReader(mapper.fieldType("field"))) { + block = (TestBlock) loader.columnAtATimeReader(ctx).read(TestBlock.FACTORY, TestBlock.docs(0)); + } else { + assertNull(loader.columnAtATimeReader(ctx)); + return; + } + } else { + BlockLoaderStoredFieldsFromLeafLoader storedFieldsLoader = new BlockLoaderStoredFieldsFromLeafLoader( + StoredFieldLoader.fromSpec(loader.rowStrideStoredFieldSpec()).getLoader(ctx, null), + loader.rowStrideStoredFieldSpec().requiresSource() + ); + storedFieldsLoader.advanceTo(0); + BlockLoader.Builder builder = loader.builder(TestBlock.FACTORY, 1); + loader.rowStrideReader(ctx).read(0, storedFieldsLoader, builder); + block = (TestBlock) builder.build(); + } Object inBlock = block.get(0); if (inBlock != null) { if (inBlock instanceof List l) { @@ -1319,7 +1336,7 @@ public Set sourcePaths(String name) { } /** - * Matcher for {@link #testBlockLoaderReadValues} and {@link #testBlockLoaderReadValuesFromSingleDoc}. + * Matcher for {@link #testBlockLoaderFromColumnReader} and {@link #testBlockLoaderFromRowStrideReader}. */ protected Matcher blockItemMatcher(Object expected) { return equalTo(expected); diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/TestBlock.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/TestBlock.java index 298acb9519532..30dece5767b61 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/TestBlock.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/TestBlock.java @@ -11,7 +11,6 @@ import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.SortedDocValues; import org.apache.lucene.util.BytesRef; -import org.elasticsearch.core.Nullable; import java.io.IOException; import java.io.UncheckedIOException; @@ -21,74 +20,130 @@ import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; -public class TestBlock - implements - BlockLoader.BooleanBuilder, - BlockLoader.BytesRefBuilder, - BlockLoader.DoubleBuilder, - BlockLoader.IntBuilder, - BlockLoader.LongBuilder, - BlockLoader.SingletonOrdinalsBuilder, - BlockLoader.Block { - public static BlockLoader.BuilderFactory FACTORY = new BlockLoader.BuilderFactory() { +public class TestBlock implements BlockLoader.Block { + public static BlockLoader.BlockFactory FACTORY = new BlockLoader.BlockFactory() { @Override public BlockLoader.BooleanBuilder booleansFromDocValues(int expectedCount) { - return new TestBlock(null); + return booleans(expectedCount); } @Override public BlockLoader.BooleanBuilder booleans(int expectedCount) { - return new TestBlock(null); + class BooleansBuilder extends TestBlock.Builder implements BlockLoader.BooleanBuilder { + @Override + public BooleansBuilder appendBoolean(boolean value) { + add(value); + return this; + } + } + return new BooleansBuilder(); } @Override public BlockLoader.BytesRefBuilder bytesRefsFromDocValues(int expectedCount) { - return new TestBlock(null); + return bytesRefs(expectedCount); } @Override public BlockLoader.BytesRefBuilder bytesRefs(int expectedCount) { - return new TestBlock(null); + class BytesRefsBuilder extends TestBlock.Builder implements BlockLoader.BytesRefBuilder { + @Override + public BytesRefsBuilder appendBytesRef(BytesRef value) { + add(BytesRef.deepCopyOf(value)); + return this; + } + } + return new BytesRefsBuilder(); } @Override public BlockLoader.DoubleBuilder doublesFromDocValues(int expectedCount) { - return new TestBlock(null); + return doubles(expectedCount); } @Override public BlockLoader.DoubleBuilder doubles(int expectedCount) { - return new TestBlock(null); + class DoublesBuilder extends TestBlock.Builder implements BlockLoader.DoubleBuilder { + @Override + public DoublesBuilder appendDouble(double value) { + add(value); + return this; + } + } + return new DoublesBuilder(); } @Override public BlockLoader.IntBuilder intsFromDocValues(int expectedCount) { - return new TestBlock(null); + return ints(expectedCount); } @Override public BlockLoader.IntBuilder ints(int expectedCount) { - return new TestBlock(null); + class IntsBuilder extends TestBlock.Builder implements BlockLoader.IntBuilder { + @Override + public IntsBuilder appendInt(int value) { + add(value); + return this; + } + } + return new IntsBuilder(); } @Override public BlockLoader.LongBuilder longsFromDocValues(int expectedCount) { - return new TestBlock(null); + return longs(expectedCount); } @Override public BlockLoader.LongBuilder longs(int expectedCount) { - return new TestBlock(null); + class LongsBuilder extends TestBlock.Builder implements BlockLoader.LongBuilder { + @Override + public LongsBuilder appendLong(long value) { + add(value); + return this; + } + } + return new LongsBuilder(); } @Override public BlockLoader.Builder nulls(int expectedCount) { - return new TestBlock(null); + return longs(expectedCount); + } + + @Override + public BlockLoader.Block constantNulls(int size) { + BlockLoader.LongBuilder builder = longs(size); + for (int i = 0; i < size; i++) { + builder.appendNull(); + } + return builder.build(); + } + + @Override + public BlockLoader.Block constantBytes(BytesRef value, int size) { + BlockLoader.BytesRefBuilder builder = bytesRefs(size); + for (int i = 0; i < size; i++) { + builder.appendBytesRef(value); + } + return builder.build(); } @Override public BlockLoader.SingletonOrdinalsBuilder singletonOrdinalsBuilder(SortedDocValues ordinals, int count) { - return new TestBlock(ordinals); + class SingletonOrdsBuilder extends TestBlock.Builder implements BlockLoader.SingletonOrdinalsBuilder { + @Override + public SingletonOrdsBuilder appendOrd(int value) { + try { + add(ordinals.lookupOrd(value)); + return this; + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + } + return new SingletonOrdsBuilder(); } }; @@ -120,13 +175,10 @@ public int get(int i) { }; } - private final SortedDocValues sortedDocValues; - private final List values = new ArrayList<>(); - - private List currentPosition = null; + private final List values; - private TestBlock(@Nullable SortedDocValues sortedDocValues) { - this.sortedDocValues = sortedDocValues; + private TestBlock(List values) { + this.values = values; } public Object get(int i) { @@ -138,73 +190,49 @@ public int size() { } @Override - public TestBlock appendNull() { - assertNull(currentPosition); - values.add(null); - return this; - } - - @Override - public TestBlock beginPositionEntry() { - assertNull(currentPosition); - currentPosition = new ArrayList<>(); - values.add(currentPosition); - return this; - } - - @Override - public TestBlock endPositionEntry() { - assertNotNull(currentPosition); - currentPosition = null; - return this; - } - - @Override - public TestBlock appendBoolean(boolean value) { - return add(value); + public void close() { + // TODO assert that we close the test blocks } - @Override - public TestBlock appendBytesRef(BytesRef value) { - return add(BytesRef.deepCopyOf(value)); - } + private abstract static class Builder implements BlockLoader.Builder { + private final List values = new ArrayList<>(); - @Override - public TestBlock appendDouble(double value) { - return add(value); - } + private List currentPosition = null; - @Override - public TestBlock appendInt(int value) { - return add(value); - } + @Override + public Builder appendNull() { + assertNull(currentPosition); + values.add(null); + return this; + } - @Override - public TestBlock appendLong(long value) { - return add(value); - } + @Override + public Builder beginPositionEntry() { + assertNull(currentPosition); + currentPosition = new ArrayList<>(); + values.add(currentPosition); + return this; + } - @Override - public TestBlock appendOrd(int value) { - try { - return add(sortedDocValues.lookupOrd(value)); - } catch (IOException e) { - throw new UncheckedIOException(e); + @Override + public Builder endPositionEntry() { + assertNotNull(currentPosition); + currentPosition = null; + return this; } - } - @Override - public TestBlock build() { - return this; - } + protected void add(Object value) { + (currentPosition == null ? values : currentPosition).add(value); + } - private TestBlock add(Object value) { - (currentPosition == null ? values : currentPosition).add(value); - return this; - } + @Override + public TestBlock build() { + return new TestBlock(values); + } - @Override - public void close() { - // TODO assert that we close the test blocks + @Override + public void close() { + // TODO assert that we close the test block builders + } } } diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java index aff8a20aa88b6..3327137cef7b7 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java @@ -112,6 +112,7 @@ import static java.util.Collections.sort; import static java.util.Collections.unmodifiableList; +import static org.elasticsearch.client.RestClient.IGNORE_RESPONSE_CODES_PARAM; import static org.elasticsearch.core.Strings.format; import static org.hamcrest.Matchers.anyOf; import static org.hamcrest.Matchers.containsString; @@ -1157,7 +1158,7 @@ private void wipeRollupJobs() throws IOException { @SuppressWarnings("unchecked") String jobId = (String) ((Map) jobConfig.get("config")).get("id"); Request request = new Request("POST", "/_rollup/job/" + jobId + "/_stop"); - request.addParameter("ignore", "404"); + setIgnoredErrorResponseCodes(request, RestStatus.NOT_FOUND); request.addParameter("wait_for_completion", "true"); request.addParameter("timeout", "10s"); logger.debug("stopping rollup job [{}]", jobId); @@ -1168,7 +1169,7 @@ private void wipeRollupJobs() throws IOException { @SuppressWarnings("unchecked") String jobId = (String) ((Map) jobConfig.get("config")).get("id"); Request request = new Request("DELETE", "/_rollup/job/" + jobId); - request.addParameter("ignore", "404"); // Ignore 404s because they imply someone was racing us to delete this + setIgnoredErrorResponseCodes(request, RestStatus.NOT_FOUND); // 404s imply someone was racing us to delete this logger.debug("deleting rollup job [{}]", jobId); adminClient().performRequest(request); } @@ -1485,8 +1486,9 @@ private static Set runningTasks(Response response) throws IOException { return runningTasks; } - public static void assertOK(Response response) { + public static Response assertOK(Response response) { assertThat(response.getStatusLine().getStatusCode(), anyOf(equalTo(200), equalTo(201))); + return response; } public static ObjectPath assertOKAndCreateObjectPath(Response response) throws IOException { @@ -1845,7 +1847,7 @@ protected static void deleteSnapshot(RestClient restClient, String repository, S throws IOException { final Request request = new Request(HttpDelete.METHOD_NAME, "_snapshot/" + repository + '/' + snapshot); if (ignoreMissing) { - request.addParameter("ignore", "404"); + setIgnoredErrorResponseCodes(request, RestStatus.NOT_FOUND); } final Response response = restClient.performRequest(request); assertThat(response.getStatusLine().getStatusCode(), ignoreMissing ? anyOf(equalTo(200), equalTo(404)) : equalTo(200)); @@ -2243,4 +2245,11 @@ protected Map getHistoricalFeatures() { return historicalFeatures; } + + public static void setIgnoredErrorResponseCodes(Request request, RestStatus... restStatuses) { + request.addParameter( + IGNORE_RESPONSE_CODES_PARAM, + Arrays.stream(restStatuses).map(restStatus -> Integer.toString(restStatus.getStatus())).collect(Collectors.joining(",")) + ); + } } diff --git a/test/yaml-rest-runner/src/main/java/org/elasticsearch/test/rest/yaml/restspec/ClientYamlSuiteRestSpec.java b/test/yaml-rest-runner/src/main/java/org/elasticsearch/test/rest/yaml/restspec/ClientYamlSuiteRestSpec.java index be34ee9be0ea1..8662d886cce89 100644 --- a/test/yaml-rest-runner/src/main/java/org/elasticsearch/test/rest/yaml/restspec/ClientYamlSuiteRestSpec.java +++ b/test/yaml-rest-runner/src/main/java/org/elasticsearch/test/rest/yaml/restspec/ClientYamlSuiteRestSpec.java @@ -24,6 +24,8 @@ import java.util.Set; import java.util.stream.Stream; +import static org.elasticsearch.client.RestClient.IGNORE_RESPONSE_CODES_PARAM; + /** * Holds the specification used to turn {@code do} actions in the YAML suite into REST api calls. */ @@ -69,7 +71,7 @@ public boolean isGlobalParameter(String param) { * that they influence the client behaviour and don't get sent to Elasticsearch */ public boolean isClientParameter(String name) { - return "ignore".equals(name); + return IGNORE_RESPONSE_CODES_PARAM.equals(name); } /** diff --git a/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/index/shard/CloseFollowerIndexErrorSuppressionHelper.java b/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/index/shard/CloseFollowerIndexErrorSuppressionHelper.java new file mode 100644 index 0000000000000..89ba41317e0e3 --- /dev/null +++ b/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/index/shard/CloseFollowerIndexErrorSuppressionHelper.java @@ -0,0 +1,14 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.index.shard; + +public class CloseFollowerIndexErrorSuppressionHelper { + public static void setSuppressCreateEngineErrors(boolean value) { + IndexShard.suppressCreateEngineErrors = value; + } +} diff --git a/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/CloseFollowerIndexIT.java b/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/CloseFollowerIndexIT.java index 64ebb20538832..8e597c3992528 100644 --- a/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/CloseFollowerIndexIT.java +++ b/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/CloseFollowerIndexIT.java @@ -18,6 +18,7 @@ import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.engine.ReadOnlyEngine; +import org.elasticsearch.index.shard.CloseFollowerIndexErrorSuppressionHelper; import org.elasticsearch.xcontent.XContentType; import org.elasticsearch.xpack.CcrIntegTestCase; import org.elasticsearch.xpack.core.ccr.action.PutFollowAction; @@ -39,13 +40,16 @@ public class CloseFollowerIndexIT extends CcrIntegTestCase { @Before public void wrapUncaughtExceptionHandler() { + CloseFollowerIndexErrorSuppressionHelper.setSuppressCreateEngineErrors(true); uncaughtExceptionHandler = Thread.getDefaultUncaughtExceptionHandler(); AccessController.doPrivileged((PrivilegedAction) () -> { Thread.setDefaultUncaughtExceptionHandler((t, e) -> { - if (t.getThreadGroup().getName().contains(getTestClass().getSimpleName())) { + if (t.getThreadGroup().getName().contains(getTestClass().getSimpleName()) + && t.getName().equals("elasticsearch-error-rethrower")) { for (StackTraceElement element : e.getStackTrace()) { if (element.getClassName().equals(ReadOnlyEngine.class.getName())) { if (element.getMethodName().equals("assertMaxSeqNoEqualsToGlobalCheckpoint")) { + logger.error("HACK: suppressing uncaught exception thrown from assertMaxSeqNoEqualsToGlobalCheckpoint", e); return; } } @@ -59,6 +63,7 @@ public void wrapUncaughtExceptionHandler() { @After public void restoreUncaughtExceptionHandler() { + CloseFollowerIndexErrorSuppressionHelper.setSuppressCreateEngineErrors(false); AccessController.doPrivileged((PrivilegedAction) () -> { Thread.setDefaultUncaughtExceptionHandler(uncaughtExceptionHandler); return null; diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/action/GetCheckpointAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/action/GetCheckpointAction.java index 7ac27d79d3cb8..e492a98748af2 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/action/GetCheckpointAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/action/GetCheckpointAction.java @@ -18,6 +18,7 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.core.TimeValue; +import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.tasks.CancellableTask; import org.elasticsearch.tasks.TaskId; @@ -48,12 +49,21 @@ public static class Request extends ActionRequest implements IndicesRequest.Repl private String[] indices; private final IndicesOptions indicesOptions; + private final QueryBuilder query; + private final String cluster; private final TimeValue timeout; public Request(StreamInput in) throws IOException { super(in); indices = in.readStringArray(); indicesOptions = IndicesOptions.readIndicesOptions(in); + if (in.getTransportVersion().onOrAfter(TransportVersions.TRANSFORM_GET_CHECKPOINT_QUERY_AND_CLUSTER_ADDED)) { + query = in.readOptionalNamedWriteable(QueryBuilder.class); + cluster = in.readOptionalString(); + } else { + query = null; + cluster = null; + } if (in.getTransportVersion().onOrAfter(TransportVersions.TRANSFORM_GET_CHECKPOINT_TIMEOUT_ADDED)) { timeout = in.readOptionalTimeValue(); } else { @@ -61,9 +71,11 @@ public Request(StreamInput in) throws IOException { } } - public Request(String[] indices, IndicesOptions indicesOptions, TimeValue timeout) { + public Request(String[] indices, IndicesOptions indicesOptions, QueryBuilder query, String cluster, TimeValue timeout) { this.indices = indices != null ? indices : Strings.EMPTY_ARRAY; this.indicesOptions = indicesOptions; + this.query = query; + this.cluster = cluster; this.timeout = timeout; } @@ -82,6 +94,14 @@ public IndicesOptions indicesOptions() { return indicesOptions; } + public QueryBuilder getQuery() { + return query; + } + + public String getCluster() { + return cluster; + } + public TimeValue getTimeout() { return timeout; } @@ -98,12 +118,14 @@ public boolean equals(Object obj) { return Arrays.equals(indices, that.indices) && Objects.equals(indicesOptions, that.indicesOptions) + && Objects.equals(query, that.query) + && Objects.equals(cluster, that.cluster) && Objects.equals(timeout, that.timeout); } @Override public int hashCode() { - return Objects.hash(Arrays.hashCode(indices), indicesOptions, timeout); + return Objects.hash(Arrays.hashCode(indices), indicesOptions, query, cluster, timeout); } @Override @@ -111,6 +133,10 @@ public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); out.writeStringArray(indices); indicesOptions.writeIndicesOptions(out); + if (out.getTransportVersion().onOrAfter(TransportVersions.TRANSFORM_GET_CHECKPOINT_QUERY_AND_CLUSTER_ADDED)) { + out.writeOptionalNamedWriteable(query); + out.writeOptionalString(cluster); + } if (out.getTransportVersion().onOrAfter(TransportVersions.TRANSFORM_GET_CHECKPOINT_TIMEOUT_ADDED)) { out.writeOptionalTimeValue(timeout); } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/action/GetCheckpointActionRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/action/GetCheckpointActionRequestTests.java index 43ec0a0f1b4f5..e96a7741b4f52 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/action/GetCheckpointActionRequestTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/action/GetCheckpointActionRequestTests.java @@ -11,9 +11,10 @@ import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.common.io.stream.Writeable.Reader; import org.elasticsearch.core.TimeValue; +import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.tasks.CancellableTask; import org.elasticsearch.tasks.TaskId; -import org.elasticsearch.test.AbstractWireSerializingTestCase; import org.elasticsearch.xpack.core.transform.action.GetCheckpointAction.Request; import java.util.ArrayList; @@ -26,7 +27,7 @@ import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; -public class GetCheckpointActionRequestTests extends AbstractWireSerializingTestCase { +public class GetCheckpointActionRequestTests extends AbstractWireSerializingTransformTestCase { @Override protected Request createTestInstance() { @@ -42,9 +43,11 @@ protected Reader instanceReader() { protected Request mutateInstance(Request instance) { List indices = instance.indices() != null ? new ArrayList<>(Arrays.asList(instance.indices())) : new ArrayList<>(); IndicesOptions indicesOptions = instance.indicesOptions(); + QueryBuilder query = instance.getQuery(); + String cluster = instance.getCluster(); TimeValue timeout = instance.getTimeout(); - switch (between(0, 2)) { + switch (between(0, 4)) { case 0: indices.add(randomAlphaOfLengthBetween(1, 20)); break; @@ -58,13 +61,19 @@ protected Request mutateInstance(Request instance) { ); break; case 2: + query = query != null ? null : QueryBuilders.matchAllQuery(); + break; + case 3: + cluster = cluster != null ? null : randomAlphaOfLengthBetween(1, 10); + break; + case 4: timeout = timeout != null ? null : TimeValue.timeValueSeconds(randomIntBetween(1, 300)); break; default: throw new AssertionError("Illegal randomization branch"); } - return new Request(indices.toArray(new String[0]), indicesOptions, timeout); + return new Request(indices.toArray(new String[0]), indicesOptions, query, cluster, timeout); } public void testCreateTask() { @@ -74,7 +83,7 @@ public void testCreateTask() { } public void testCreateTaskWithNullIndices() { - Request request = new Request(null, null, null); + Request request = new Request(null, null, null, null, null); CancellableTask task = request.createTask(123, "type", "action", new TaskId("dummy-node:456"), Map.of()); assertThat(task.getDescription(), is(equalTo("get_checkpoint[0]"))); } @@ -89,6 +98,8 @@ private static Request randomRequest(Integer numIndices) { Boolean.toString(randomBoolean()), SearchRequest.DEFAULT_INDICES_OPTIONS ), + randomBoolean() ? QueryBuilders.matchAllQuery() : null, + randomBoolean() ? randomAlphaOfLengthBetween(1, 10) : null, randomBoolean() ? TimeValue.timeValueSeconds(randomIntBetween(1, 300)) : null ); } diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BooleanArrayBlock.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BooleanArrayBlock.java index eb9364c57e755..ed38d3139dd4a 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BooleanArrayBlock.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BooleanArrayBlock.java @@ -135,11 +135,7 @@ public String toString() { } @Override - public void close() { - if (released) { - throw new IllegalStateException("can't release already released block [" + this + "]"); - } - released = true; + public void closeInternal() { blockFactory.adjustBreaker(-ramBytesUsed(), true); } } diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BooleanVectorBlock.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BooleanVectorBlock.java index 1a7a5b4aa6e7e..c5c3a24736c16 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BooleanVectorBlock.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BooleanVectorBlock.java @@ -17,6 +17,9 @@ public final class BooleanVectorBlock extends AbstractVectorBlock implements Boo private final BooleanVector vector; + /** + * @param vector considered owned by the current block; must not be used in any other {@code Block} + */ BooleanVectorBlock(BooleanVector vector) { super(vector.getPositionCount(), vector.blockFactory()); this.vector = vector; @@ -72,15 +75,12 @@ public String toString() { @Override public boolean isReleased() { - return released || vector.isReleased(); + return super.isReleased() || vector.isReleased(); } @Override - public void close() { - if (released || vector.isReleased()) { - throw new IllegalStateException("can't release already released block [" + this + "]"); - } - released = true; + public void closeInternal() { + assert (vector.isReleased() == false) : "can't release block [" + this + "] containing already released vector"; Releasables.closeExpectNoException(vector); } } diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BytesRefArrayBlock.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BytesRefArrayBlock.java index b2729ed370b32..6aef8fa54b134 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BytesRefArrayBlock.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BytesRefArrayBlock.java @@ -139,11 +139,7 @@ public String toString() { } @Override - public void close() { - if (released) { - throw new IllegalStateException("can't release already released block [" + this + "]"); - } - released = true; + public void closeInternal() { blockFactory.adjustBreaker(-ramBytesUsed() + values.bigArraysRamBytesUsed(), true); Releasables.closeExpectNoException(values); } diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BytesRefVectorBlock.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BytesRefVectorBlock.java index 5b0f2f2331fbe..d8c2c615a3dfb 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BytesRefVectorBlock.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BytesRefVectorBlock.java @@ -18,6 +18,9 @@ public final class BytesRefVectorBlock extends AbstractVectorBlock implements By private final BytesRefVector vector; + /** + * @param vector considered owned by the current block; must not be used in any other {@code Block} + */ BytesRefVectorBlock(BytesRefVector vector) { super(vector.getPositionCount(), vector.blockFactory()); this.vector = vector; @@ -73,15 +76,12 @@ public String toString() { @Override public boolean isReleased() { - return released || vector.isReleased(); + return super.isReleased() || vector.isReleased(); } @Override - public void close() { - if (released || vector.isReleased()) { - throw new IllegalStateException("can't release already released block [" + this + "]"); - } - released = true; + public void closeInternal() { + assert (vector.isReleased() == false) : "can't release block [" + this + "] containing already released vector"; Releasables.closeExpectNoException(vector); } } diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/DoubleArrayBlock.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/DoubleArrayBlock.java index b6b1fae0ded03..6a5af2d7ca6de 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/DoubleArrayBlock.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/DoubleArrayBlock.java @@ -135,11 +135,7 @@ public String toString() { } @Override - public void close() { - if (released) { - throw new IllegalStateException("can't release already released block [" + this + "]"); - } - released = true; + public void closeInternal() { blockFactory.adjustBreaker(-ramBytesUsed(), true); } } diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/DoubleVectorBlock.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/DoubleVectorBlock.java index d05be62744bc8..ac4c826b5f2d2 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/DoubleVectorBlock.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/DoubleVectorBlock.java @@ -17,6 +17,9 @@ public final class DoubleVectorBlock extends AbstractVectorBlock implements Doub private final DoubleVector vector; + /** + * @param vector considered owned by the current block; must not be used in any other {@code Block} + */ DoubleVectorBlock(DoubleVector vector) { super(vector.getPositionCount(), vector.blockFactory()); this.vector = vector; @@ -72,15 +75,12 @@ public String toString() { @Override public boolean isReleased() { - return released || vector.isReleased(); + return super.isReleased() || vector.isReleased(); } @Override - public void close() { - if (released || vector.isReleased()) { - throw new IllegalStateException("can't release already released block [" + this + "]"); - } - released = true; + public void closeInternal() { + assert (vector.isReleased() == false) : "can't release block [" + this + "] containing already released vector"; Releasables.closeExpectNoException(vector); } } diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/IntArrayBlock.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/IntArrayBlock.java index 31f71d292f95d..284520a5f3bd6 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/IntArrayBlock.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/IntArrayBlock.java @@ -135,11 +135,7 @@ public String toString() { } @Override - public void close() { - if (released) { - throw new IllegalStateException("can't release already released block [" + this + "]"); - } - released = true; + public void closeInternal() { blockFactory.adjustBreaker(-ramBytesUsed(), true); } } diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/IntVectorBlock.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/IntVectorBlock.java index 472475d0662d7..60280ebb13064 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/IntVectorBlock.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/IntVectorBlock.java @@ -17,6 +17,9 @@ public final class IntVectorBlock extends AbstractVectorBlock implements IntBloc private final IntVector vector; + /** + * @param vector considered owned by the current block; must not be used in any other {@code Block} + */ IntVectorBlock(IntVector vector) { super(vector.getPositionCount(), vector.blockFactory()); this.vector = vector; @@ -72,15 +75,12 @@ public String toString() { @Override public boolean isReleased() { - return released || vector.isReleased(); + return super.isReleased() || vector.isReleased(); } @Override - public void close() { - if (released || vector.isReleased()) { - throw new IllegalStateException("can't release already released block [" + this + "]"); - } - released = true; + public void closeInternal() { + assert (vector.isReleased() == false) : "can't release block [" + this + "] containing already released vector"; Releasables.closeExpectNoException(vector); } } diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/LongArrayBlock.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/LongArrayBlock.java index 8a71703441ebb..fccad0ec1f09b 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/LongArrayBlock.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/LongArrayBlock.java @@ -135,11 +135,7 @@ public String toString() { } @Override - public void close() { - if (released) { - throw new IllegalStateException("can't release already released block [" + this + "]"); - } - released = true; + public void closeInternal() { blockFactory.adjustBreaker(-ramBytesUsed(), true); } } diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/LongVectorBlock.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/LongVectorBlock.java index b94cd4e875dc3..c9b65ba3e9029 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/LongVectorBlock.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/LongVectorBlock.java @@ -17,6 +17,9 @@ public final class LongVectorBlock extends AbstractVectorBlock implements LongBl private final LongVector vector; + /** + * @param vector considered owned by the current block; must not be used in any other {@code Block} + */ LongVectorBlock(LongVector vector) { super(vector.getPositionCount(), vector.blockFactory()); this.vector = vector; @@ -72,15 +75,12 @@ public String toString() { @Override public boolean isReleased() { - return released || vector.isReleased(); + return super.isReleased() || vector.isReleased(); } @Override - public void close() { - if (released || vector.isReleased()) { - throw new IllegalStateException("can't release already released block [" + this + "]"); - } - released = true; + public void closeInternal() { + assert (vector.isReleased() == false) : "can't release block [" + this + "] containing already released vector"; Releasables.closeExpectNoException(vector); } } diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/AbstractBlock.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/AbstractBlock.java index cbe74c814594d..39f17cfecab1a 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/AbstractBlock.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/AbstractBlock.java @@ -12,7 +12,7 @@ import java.util.BitSet; abstract class AbstractBlock implements Block { - + private int references = 1; private final int positionCount; @Nullable @@ -23,8 +23,6 @@ abstract class AbstractBlock implements Block { protected final BlockFactory blockFactory; - protected boolean released = false; - /** * @param positionCount the number of values in this block */ @@ -99,6 +97,54 @@ public BlockFactory blockFactory() { @Override public boolean isReleased() { - return released; + return hasReferences() == false; + } + + @Override + public final void incRef() { + if (isReleased()) { + throw new IllegalStateException("can't increase refCount on already released block [" + this + "]"); + } + references++; + } + + @Override + public final boolean tryIncRef() { + if (isReleased()) { + return false; + } + references++; + return true; + } + + @Override + public final boolean decRef() { + if (isReleased()) { + throw new IllegalStateException("can't release already released block [" + this + "]"); + } + + references--; + + if (references <= 0) { + closeInternal(); + return true; + } + return false; + } + + @Override + public final boolean hasReferences() { + return references >= 1; } + + @Override + public final void close() { + decRef(); + } + + /** + * This is called when the number of references reaches zero. + * It must release any resources held by the block (adjusting circuit breakers if needed). + */ + protected abstract void closeInternal(); } diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/Block.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/Block.java index 75b02ff911df7..ee9889d7d3be8 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/Block.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/Block.java @@ -11,6 +11,7 @@ import org.elasticsearch.common.io.stream.NamedWriteable; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.core.Nullable; +import org.elasticsearch.core.RefCounted; import org.elasticsearch.core.Releasable; import org.elasticsearch.core.Releasables; import org.elasticsearch.index.mapper.BlockLoader; @@ -23,12 +24,19 @@ * position. * *

Blocks can represent various shapes of underlying data. A Block can represent either sparse - * or dense data. A Block can represent either single or multi valued data. A Block that represents + * or dense data. A Block can represent either single or multivalued data. A Block that represents * dense single-valued data can be viewed as a {@link Vector}. * - *

Block are immutable and can be passed between threads. + *

Blocks are reference counted; to make a shallow copy of a block (e.g. if a {@link Page} contains + * the same column twice), use {@link Block#incRef()}. Before a block is garbage collected, + * {@link Block#close()} must be called to release a block's resources; it must also be called one + * additional time for each time {@link Block#incRef()} was called. Calls to {@link Block#decRef()} and + * {@link Block#close()} are equivalent. + * + *

Block are immutable and can be passed between threads as long as no two threads hold a reference to + * the same block at the same time. */ -public interface Block extends Accountable, BlockLoader.Block, NamedWriteable, Releasable { +public interface Block extends Accountable, BlockLoader.Block, NamedWriteable, RefCounted, Releasable { /** * {@return an efficient dense single-value view of this block}. @@ -57,14 +65,15 @@ public interface Block extends Accountable, BlockLoader.Block, NamedWriteable, R /** The block factory associated with this block. */ BlockFactory blockFactory(); - /** Tells if this block has been released. A block is released by calling its {@link Block#close()} method. */ + /** + * Tells if this block has been released. A block is released by calling its {@link Block#close()} or {@link Block#decRef()} methods. + * @return true iff the block's reference count is zero. + * */ boolean isReleased(); /** - * Returns true if the value stored at the given position is null, false otherwise. - * * @param position the position - * @return true or false + * @return true if the value stored at the given position is null, false otherwise */ boolean isNull(int position); @@ -91,6 +100,7 @@ public interface Block extends Accountable, BlockLoader.Block, NamedWriteable, R /** * Creates a new block that only exposes the positions provided. Materialization of the selected positions is avoided. + * The new block may hold a reference to this block, increasing this block's reference count. * @param positions the positions to retain * @return a filtered block */ @@ -137,6 +147,7 @@ default boolean mvSortedAscending() { * Expand multivalued fields into one row per value. Returns the * block if there aren't any multivalued fields to expand. */ + // TODO: We should use refcounting instead of either deep copies or returning the same identical block. Block expand(); /** @@ -231,7 +242,7 @@ static Block[] buildAll(Block.Builder... builders) { /** * A reference to a {@link Block}. This is {@link Releasable} and - * {@link Ref#close closing} it will {@link Block#close release} + * {@link Ref#close closing} it will {@link Block#close() release} * the underlying {@link Block} if it wasn't borrowed from a {@link Page}. * * The usual way to use this is: @@ -248,6 +259,7 @@ static Block[] buildAll(Block.Builder... builders) { * @param block the block referenced * @param containedIn the page containing it or null, if it is "free floating". */ + // We probably want to remove this; instead, we could incRef and decRef consistently in the EvalOperator. record Ref(Block block, @Nullable Page containedIn) implements Releasable { /** * Create a "free floating" {@link Ref}. diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/ConstantNullBlock.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/ConstantNullBlock.java index 9437bdd35e21f..5823a4b98d52c 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/ConstantNullBlock.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/ConstantNullBlock.java @@ -126,11 +126,7 @@ public String toString() { } @Override - public void close() { - if (isReleased()) { - throw new IllegalStateException("can't release already released block [" + this + "]"); - } - released = true; + public void closeInternal() { blockFactory.adjustBreaker(-ramBytesUsed(), true); } diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/DocBlock.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/DocBlock.java index ed7e317bfc4c7..9dc27196bd128 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/DocBlock.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/DocBlock.java @@ -71,11 +71,13 @@ public long ramBytesUsed() { } @Override - public void close() { - if (released) { - throw new IllegalStateException("can't release already released block [" + this + "]"); - } - released = true; + public boolean isReleased() { + return super.isReleased() || vector.isReleased(); + } + + @Override + public void closeInternal() { + assert (vector.isReleased() == false) : "can't release block [" + this + "] containing already released vector"; Releasables.closeExpectNoException(vector); } diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/DocVector.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/DocVector.java index b6ba42f953609..24c656404e89f 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/DocVector.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/DocVector.java @@ -225,6 +225,7 @@ public long ramBytesUsed() { @Override public void close() { + released = true; Releasables.closeExpectNoException(shards.asBlock(), segments.asBlock(), docs.asBlock()); // Ugh! we always close blocks } } diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/Page.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/Page.java index 451a0b540f308..94f27e9e55f33 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/Page.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/Page.java @@ -14,8 +14,6 @@ import java.io.IOException; import java.util.Arrays; -import java.util.Collections; -import java.util.IdentityHashMap; import java.util.Objects; /** @@ -235,39 +233,7 @@ public void releaseBlocks() { blocksReleased = true; - // blocks can be used as multiple columns - var map = new IdentityHashMap(mapSize(blocks.length)); - for (Block b : blocks) { - if (map.putIfAbsent(b, Boolean.TRUE) == null) { - Releasables.closeExpectNoException(b); - } - } - } - - /** - * Returns a Page from the given blocks and closes all blocks that are not included, from the current Page. - * That is, allows clean-up of the current page _after_ external manipulation of the blocks. - * The current page should no longer be used and be considered closed. - */ - public Page newPageAndRelease(Block... keep) { - if (blocksReleased) { - throw new IllegalStateException("can't create new page from already released page"); - } - - blocksReleased = true; - - var newPage = new Page(positionCount, keep); - var set = Collections.newSetFromMap(new IdentityHashMap(mapSize(keep.length))); - set.addAll(Arrays.asList(keep)); - - // close blocks that have been left out - for (Block b : blocks) { - if (set.contains(b) == false) { - Releasables.closeExpectNoException(b); - } - } - - return newPage; + Releasables.closeExpectNoException(blocks); } static int mapSize(int expectedSize) { diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-ArrayBlock.java.st b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-ArrayBlock.java.st index 49a4c43709cde..86a8dfc78450d 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-ArrayBlock.java.st +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-ArrayBlock.java.st @@ -168,11 +168,7 @@ $endif$ } @Override - public void close() { - if (released) { - throw new IllegalStateException("can't release already released block [" + this + "]"); - } - released = true; + public void closeInternal() { $if(BytesRef)$ blockFactory.adjustBreaker(-ramBytesUsed() + values.bigArraysRamBytesUsed(), true); Releasables.closeExpectNoException(values); diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-VectorBlock.java.st b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-VectorBlock.java.st index 3ef4251f80684..89bc84d551b63 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-VectorBlock.java.st +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-VectorBlock.java.st @@ -20,6 +20,9 @@ public final class $Type$VectorBlock extends AbstractVectorBlock implements $Typ private final $Type$Vector vector; + /** + * @param vector considered owned by the current block; must not be used in any other {@code Block} + */ $Type$VectorBlock($Type$Vector vector) { super(vector.getPositionCount(), vector.blockFactory()); this.vector = vector; @@ -80,15 +83,12 @@ $endif$ @Override public boolean isReleased() { - return released || vector.isReleased(); + return super.isReleased() || vector.isReleased(); } @Override - public void close() { - if (released || vector.isReleased()) { - throw new IllegalStateException("can't release already released block [" + this + "]"); - } - released = true; + public void closeInternal() { + assert (vector.isReleased() == false) : "can't release block [" + this + "] containing already released vector"; Releasables.closeExpectNoException(vector); } } diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/BlockReaderFactories.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/BlockReaderFactories.java index a0d08bc798fbb..a730931208663 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/BlockReaderFactories.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/BlockReaderFactories.java @@ -7,17 +7,13 @@ package org.elasticsearch.compute.lucene; -import org.apache.lucene.index.IndexReader; -import org.apache.lucene.index.SortedSetDocValues; import org.elasticsearch.common.logging.HeaderWarning; -import org.elasticsearch.index.mapper.BlockDocValuesReader; import org.elasticsearch.index.mapper.BlockLoader; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.search.internal.SearchContext; import org.elasticsearch.search.lookup.SearchLookup; -import java.io.IOException; import java.util.ArrayList; import java.util.List; import java.util.Set; @@ -36,23 +32,19 @@ private BlockReaderFactories() {} * @param asUnsupportedSource should the field be loaded as "unsupported"? * These will always have {@code null} values */ - public static List factories( - List searchContexts, - String fieldName, - boolean asUnsupportedSource - ) { - List factories = new ArrayList<>(searchContexts.size()); + public static List loaders(List searchContexts, String fieldName, boolean asUnsupportedSource) { + List loaders = new ArrayList<>(searchContexts.size()); for (SearchContext searchContext : searchContexts) { SearchExecutionContext ctx = searchContext.getSearchExecutionContext(); if (asUnsupportedSource) { - factories.add(loaderToFactory(ctx.getIndexReader(), BlockDocValuesReader.nulls())); + loaders.add(BlockLoader.CONSTANT_NULLS); continue; } MappedFieldType fieldType = ctx.getFieldType(fieldName); if (fieldType == null) { // the field does not exist in this context - factories.add(loaderToFactory(ctx.getIndexReader(), BlockDocValuesReader.nulls())); + loaders.add(BlockLoader.CONSTANT_NULLS); continue; } BlockLoader loader = fieldType.blockLoader(new MappedFieldType.BlockLoaderContext() { @@ -73,36 +65,12 @@ public Set sourcePaths(String name) { }); if (loader == null) { HeaderWarning.addWarning("Field [{}] cannot be retrieved, it is unsupported or not indexed; returning null", fieldName); - factories.add(loaderToFactory(ctx.getIndexReader(), BlockDocValuesReader.nulls())); + loaders.add(BlockLoader.CONSTANT_NULLS); continue; } - factories.add(loaderToFactory(ctx.getIndexReader(), loader)); + loaders.add(loader); } - return factories; - } - - /** - * Converts a {@link BlockLoader}, something defined in core elasticsearch at - * the field level, into a {@link BlockDocValuesReader.Factory} which can be - * used inside ESQL. - */ - public static BlockDocValuesReader.Factory loaderToFactory(IndexReader reader, BlockLoader loader) { - return new BlockDocValuesReader.Factory() { - @Override - public BlockDocValuesReader build(int segment) throws IOException { - return loader.reader(reader.leaves().get(segment)); - } - - @Override - public boolean supportsOrdinals() { - return loader.supportsOrdinals(); - } - - @Override - public SortedSetDocValues ordinals(int segment) throws IOException { - return loader.ordinals(reader.leaves().get(segment)); - } - }; + return loaders; } } diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/LuceneTopNSourceOperator.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/LuceneTopNSourceOperator.java index 4ce0af3bd0ffe..9624fa48ef20d 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/LuceneTopNSourceOperator.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/LuceneTopNSourceOperator.java @@ -219,7 +219,7 @@ private Page emit(boolean startEmitting) { page = new Page(size, new DocVector(shard.asVector(), segments, docs, null).asBlock()); } finally { if (page == null) { - Releasables.close(shard, segments, docs); + Releasables.closeExpectNoException(shard, segments, docs); } } pagesEmitted++; diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/ValuesSourceReaderOperator.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/ValuesSourceReaderOperator.java index 61c1bd9730e02..8d7a9df523c3d 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/ValuesSourceReaderOperator.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/ValuesSourceReaderOperator.java @@ -7,13 +7,17 @@ package org.elasticsearch.compute.lucene; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.SortedDocValues; +import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.compute.data.Block; import org.elasticsearch.compute.data.BlockFactory; +import org.elasticsearch.compute.data.BytesRefBlock; import org.elasticsearch.compute.data.DocBlock; import org.elasticsearch.compute.data.DocVector; import org.elasticsearch.compute.data.ElementType; @@ -23,75 +27,69 @@ import org.elasticsearch.compute.operator.AbstractPageMappingOperator; import org.elasticsearch.compute.operator.DriverContext; import org.elasticsearch.compute.operator.Operator; -import org.elasticsearch.index.mapper.BlockDocValuesReader; +import org.elasticsearch.core.Releasable; +import org.elasticsearch.core.Releasables; +import org.elasticsearch.index.fieldvisitor.StoredFieldLoader; import org.elasticsearch.index.mapper.BlockLoader; -import org.elasticsearch.search.aggregations.support.ValuesSource; +import org.elasticsearch.index.mapper.BlockLoaderStoredFieldsFromLeafLoader; +import org.elasticsearch.search.fetch.StoredFieldsSpec; import org.elasticsearch.xcontent.XContentBuilder; import java.io.IOException; import java.io.UncheckedIOException; +import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.TreeMap; +import java.util.stream.Collectors; /** * Operator that extracts doc_values from a Lucene index out of pages that have been produced by {@link LuceneSourceOperator} - * and outputs them to a new column. The operator leverages the {@link ValuesSource} infrastructure for extracting - * field values. This allows for a more uniform way of extracting data compared to deciding the correct doc_values - * loader for different field types. + * and outputs them to a new column. */ public class ValuesSourceReaderOperator extends AbstractPageMappingOperator { /** - * Creates a new extractor that uses ValuesSources load data - * @param sources the value source, type and index readers to use for extraction + * Creates a factory for {@link ValuesSourceReaderOperator}. + * @param fields fields to load * @param docChannel the channel containing the shard, leaf/segment and doc id - * @param field the lucene field being loaded */ - public record ValuesSourceReaderOperatorFactory(List sources, int docChannel, String field) - implements - OperatorFactory { + public record Factory(List fields, List readers, int docChannel) implements OperatorFactory { @Override public Operator get(DriverContext driverContext) { - return new ValuesSourceReaderOperator(driverContext.blockFactory(), sources, docChannel, field); + return new ValuesSourceReaderOperator(driverContext.blockFactory(), fields, readers, docChannel); } @Override public String describe() { - return "ValuesSourceReaderOperator[field = " + field + "]"; + return "ValuesSourceReaderOperator[field = " + fields.stream().map(f -> f.name).collect(Collectors.joining(", ")) + "]"; } } - /** - * A list, one entry per shard, of factories for {@link BlockDocValuesReader}s - * which perform the actual reading. - */ - private final List factories; + private final List fields; + private final List readers; private final int docChannel; - private final String field; private final ComputeBlockLoaderFactory blockFactory; - private BlockDocValuesReader lastReader; - private int lastShard = -1; - private int lastSegment = -1; - private final Map readersBuilt = new TreeMap<>(); + /** + * Configuration for a field to load. + * + * {@code blockLoaders} is a list, one entry per shard, of + * {@link BlockLoader}s which load the actual blocks. + */ + public record FieldInfo(String name, List blockLoaders) {} + /** * Creates a new extractor - * @param factories builds {@link BlockDocValuesReader} + * @param fields fields to load * @param docChannel the channel containing the shard, leaf/segment and doc id - * @param field the lucene field being loaded */ - public ValuesSourceReaderOperator( - BlockFactory blockFactory, - List factories, - int docChannel, - String field - ) { - this.factories = factories; + public ValuesSourceReaderOperator(BlockFactory blockFactory, List fields, List readers, int docChannel) { + this.fields = fields.stream().map(f -> new FieldWork(f)).toList(); + this.readers = readers; this.docChannel = docChannel; - this.field = field; this.blockFactory = new ComputeBlockLoaderFactory(blockFactory); } @@ -99,21 +97,31 @@ public ValuesSourceReaderOperator( protected Page process(Page page) { DocVector docVector = page.getBlock(docChannel).asVector(); + Block[] blocks = new Block[fields.size()]; + boolean success = false; try { if (docVector.singleSegmentNonDecreasing()) { - return page.appendBlock(loadFromSingleLeaf(docVector)); + loadFromSingleLeaf(blocks, docVector); + } else { + loadFromManyLeaves(blocks, docVector); } - return page.appendBlock(loadFromManyLeaves(docVector)); + success = true; } catch (IOException e) { throw new UncheckedIOException(e); + } finally { + if (success == false) { + Releasables.closeExpectNoException(blocks); + } } + return page.appendBlocks(blocks); } - private Block loadFromSingleLeaf(DocVector docVector) throws IOException { - setupReader(docVector.shards().getInt(0), docVector.segments().getInt(0), docVector.docs().getInt(0)); - return ((Block) lastReader.readValues(blockFactory, new BlockLoader.Docs() { - private final IntVector docs = docVector.docs(); - + private void loadFromSingleLeaf(Block[] blocks, DocVector docVector) throws IOException { + int shard = docVector.shards().getInt(0); + int segment = docVector.segments().getInt(0); + int firstDoc = docVector.docs().getInt(0); + IntVector docs = docVector.docs(); + BlockLoader.Docs loaderDocs = new BlockLoader.Docs() { @Override public int count() { return docs.getPositionCount(); @@ -123,44 +131,209 @@ public int count() { public int get(int i) { return docs.getInt(i); } - })); + }; + StoredFieldsSpec storedFieldsSpec = StoredFieldsSpec.NO_REQUIREMENTS; + List rowStrideReaders = new ArrayList<>(fields.size()); + try { + for (int b = 0; b < fields.size(); b++) { + FieldWork field = fields.get(b); + BlockLoader.ColumnAtATimeReader columnAtATime = field.columnAtATime.reader(shard, segment, firstDoc); + if (columnAtATime != null) { + blocks[b] = (Block) columnAtATime.read(blockFactory, loaderDocs); + } else { + BlockLoader.RowStrideReader rowStride = field.rowStride.reader(shard, segment, firstDoc); + rowStrideReaders.add( + new RowStrideReaderWork( + rowStride, + (Block.Builder) field.info.blockLoaders.get(shard).builder(blockFactory, docs.getPositionCount()), + b + ) + ); + storedFieldsSpec = storedFieldsSpec.merge(field.info.blockLoaders.get(shard).rowStrideStoredFieldSpec()); + } + } + + if (rowStrideReaders.isEmpty()) { + return; + } + if (storedFieldsSpec.equals(StoredFieldsSpec.NO_REQUIREMENTS)) { + throw new IllegalStateException( + "found row stride readers [" + rowStrideReaders + "] without stored fields [" + storedFieldsSpec + "]" + ); + } + BlockLoaderStoredFieldsFromLeafLoader storedFields = new BlockLoaderStoredFieldsFromLeafLoader( + // TODO enable the optimization by passing non-null to docs if correct + StoredFieldLoader.fromSpec(storedFieldsSpec).getLoader(ctx(shard, segment), null), + storedFieldsSpec.requiresSource() + ); + trackStoredFields(storedFieldsSpec); // TODO when optimization is enabled add it to tracking + for (int p = 0; p < docs.getPositionCount(); p++) { + int doc = docs.getInt(p); + if (storedFields != null) { + storedFields.advanceTo(doc); + } + for (int r = 0; r < rowStrideReaders.size(); r++) { + RowStrideReaderWork work = rowStrideReaders.get(r); + work.reader.read(doc, storedFields, work.builder); + } + } + for (int r = 0; r < rowStrideReaders.size(); r++) { + RowStrideReaderWork work = rowStrideReaders.get(r); + blocks[work.offset] = work.builder.build(); + } + } finally { + Releasables.close(rowStrideReaders); + } } - private Block loadFromManyLeaves(DocVector docVector) throws IOException { + private void loadFromManyLeaves(Block[] blocks, DocVector docVector) throws IOException { + IntVector shards = docVector.shards(); + IntVector segments = docVector.segments(); + IntVector docs = docVector.docs(); + Block.Builder[] builders = new Block.Builder[blocks.length]; int[] forwards = docVector.shardSegmentDocMapForwards(); - int doc = docVector.docs().getInt(forwards[0]); - setupReader(docVector.shards().getInt(forwards[0]), docVector.segments().getInt(forwards[0]), doc); - try (BlockLoader.Builder builder = lastReader.builder(blockFactory, forwards.length)) { - lastReader.readValuesFromSingleDoc(doc, builder); - for (int i = 1; i < forwards.length; i++) { - int shard = docVector.shards().getInt(forwards[i]); - int segment = docVector.segments().getInt(forwards[i]); - doc = docVector.docs().getInt(forwards[i]); - if (segment != lastSegment || shard != lastShard) { - setupReader(shard, segment, doc); + try { + for (int b = 0; b < fields.size(); b++) { + FieldWork field = fields.get(b); + builders[b] = builderFromFirstNonNull(field, docs.getPositionCount()); + } + int lastShard = -1; + int lastSegment = -1; + BlockLoaderStoredFieldsFromLeafLoader storedFields = null; + for (int i = 0; i < forwards.length; i++) { + int p = forwards[i]; + int shard = shards.getInt(p); + int segment = segments.getInt(p); + int doc = docs.getInt(p); + if (shard != lastShard || segment != lastSegment) { + lastShard = shard; + lastSegment = segment; + StoredFieldsSpec storedFieldsSpec = storedFieldsSpecForShard(shard); + storedFields = new BlockLoaderStoredFieldsFromLeafLoader( + StoredFieldLoader.fromSpec(storedFieldsSpec).getLoader(ctx(shard, segment), null), + storedFieldsSpec.requiresSource() + ); + if (false == storedFieldsSpec.equals(StoredFieldsSpec.NO_REQUIREMENTS)) { + trackStoredFields(storedFieldsSpec); + } + } + storedFields.advanceTo(doc); + for (int r = 0; r < blocks.length; r++) { + fields.get(r).rowStride.reader(shard, segment, doc).read(doc, storedFields, builders[r]); } - lastReader.readValuesFromSingleDoc(doc, builder); } - try (Block orig = ((Block.Builder) builder).build()) { - return orig.filter(docVector.shardSegmentDocMapBackwards()); + for (int r = 0; r < blocks.length; r++) { + try (Block orig = builders[r].build()) { + blocks[r] = orig.filter(docVector.shardSegmentDocMapBackwards()); + } } + } finally { + Releasables.closeExpectNoException(builders); } } - private void setupReader(int shard, int segment, int doc) throws IOException { - if (lastSegment == segment && lastShard == shard && BlockDocValuesReader.canReuse(lastReader, doc)) { - return; + private void trackStoredFields(StoredFieldsSpec spec) { + readersBuilt.merge( + "stored_fields[" + "requires_source:" + spec.requiresSource() + ", fields:" + spec.requiredStoredFields().size() + "]", + 1, + (prev, one) -> prev + one + ); + } + + /** + * Returns a builder from the first non - {@link BlockLoader#CONSTANT_NULLS} loader + * in the list. If they are all the null loader then returns a null builder. + */ + private Block.Builder builderFromFirstNonNull(FieldWork field, int positionCount) { + for (BlockLoader loader : field.info.blockLoaders) { + if (loader != BlockLoader.CONSTANT_NULLS) { + return (Block.Builder) loader.builder(blockFactory, positionCount); + } } + // All null, just let the first one build the null block loader. + return (Block.Builder) field.info.blockLoaders.get(0).builder(blockFactory, positionCount); + } - lastReader = factories.get(shard).build(segment); - lastShard = shard; - lastSegment = segment; - readersBuilt.compute(lastReader.toString(), (k, v) -> v == null ? 1 : v + 1); + private StoredFieldsSpec storedFieldsSpecForShard(int shard) { + StoredFieldsSpec storedFieldsSpec = StoredFieldsSpec.NO_REQUIREMENTS; + for (int b = 0; b < fields.size(); b++) { + FieldWork field = fields.get(b); + storedFieldsSpec = storedFieldsSpec.merge(field.info.blockLoaders.get(shard).rowStrideStoredFieldSpec()); + } + return storedFieldsSpec; + } + + private class FieldWork { + final FieldInfo info; + final GuardedReader columnAtATime = new GuardedReader<>() { + @Override + BlockLoader.ColumnAtATimeReader build(BlockLoader loader, LeafReaderContext ctx) throws IOException { + return loader.columnAtATimeReader(ctx); + } + + @Override + String type() { + return "column_at_a_time"; + } + }; + + final GuardedReader rowStride = new GuardedReader<>() { + @Override + BlockLoader.RowStrideReader build(BlockLoader loader, LeafReaderContext ctx) throws IOException { + return loader.rowStrideReader(ctx); + } + + @Override + String type() { + return "row_stride"; + } + }; + + FieldWork(FieldInfo info) { + this.info = info; + } + + private abstract class GuardedReader { + private int lastShard = -1; + private int lastSegment = -1; + V lastReader; + + V reader(int shard, int segment, int startingDocId) throws IOException { + if (lastShard == shard && lastSegment == segment) { + if (lastReader == null) { + return null; + } + if (lastReader.canReuse(startingDocId)) { + return lastReader; + } + } + lastShard = shard; + lastSegment = segment; + lastReader = build(info.blockLoaders.get(shard), ctx(shard, segment)); + readersBuilt.merge(info.name + ":" + type() + ":" + lastReader, 1, (prev, one) -> prev + one); + return lastReader; + } + + abstract V build(BlockLoader loader, LeafReaderContext ctx) throws IOException; + + abstract String type(); + } + } + + private record RowStrideReaderWork(BlockLoader.RowStrideReader reader, Block.Builder builder, int offset) implements Releasable { + @Override + public void close() { + builder.close(); + } + } + + private LeafReaderContext ctx(int shard, int segment) { + return readers.get(shard).leaves().get(segment); } @Override public String toString() { - return "ValuesSourceReaderOperator[field = " + field + "]"; + return "ValuesSourceReaderOperator[field = " + fields.stream().map(f -> f.info.name).collect(Collectors.joining(", ")) + "]"; } @Override @@ -233,7 +406,7 @@ public String toString() { } } - private static class ComputeBlockLoaderFactory implements BlockLoader.BuilderFactory { + private static class ComputeBlockLoaderFactory implements BlockLoader.BlockFactory { private final BlockFactory factory; private ComputeBlockLoaderFactory(BlockFactory factory) { @@ -295,9 +468,21 @@ public BlockLoader.Builder nulls(int expectedCount) { return ElementType.NULL.newBlockBuilder(expectedCount, factory); } + @Override + public Block constantNulls(int size) { + return factory.newConstantNullBlock(size); + } + + @Override + public BytesRefBlock constantBytes(BytesRef value, int size) { + return factory.newConstantBytesRefBlockWith(value, size); + } + @Override public BlockLoader.SingletonOrdinalsBuilder singletonOrdinalsBuilder(SortedDocValues ordinals, int count) { return new SingletonOrdinalsBuilder(factory, ordinals, count); } } + + // TODO tests that mix source loaded fields and doc values in the same block } diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/AggregationOperator.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/AggregationOperator.java index 3e653b1a19750..07d1809262c9b 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/AggregationOperator.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/AggregationOperator.java @@ -124,10 +124,11 @@ public boolean isFinished() { @Override public void close() { - if (output != null) { - Releasables.closeExpectNoException(() -> output.releaseBlocks()); - } - Releasables.close(aggregators); + Releasables.closeExpectNoException(() -> { + if (output != null) { + Releasables.closeExpectNoException(() -> output.releaseBlocks()); + } + }, Releasables.wrap(aggregators)); } private static void checkState(boolean condition, String msg) { diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/AsyncOperator.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/AsyncOperator.java index 8eea50226253b..f2011d1cdb987 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/AsyncOperator.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/AsyncOperator.java @@ -166,6 +166,7 @@ public void finish() { @Override public boolean isFinished() { + checkFailure(); return finished && checkpoint.getPersistedCheckpoint() == checkpoint.getMaxSeqNo(); } diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/EvalOperator.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/EvalOperator.java index 65efdc4266b28..1612a2b2ece18 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/EvalOperator.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/EvalOperator.java @@ -9,7 +9,6 @@ import org.elasticsearch.compute.data.Block; import org.elasticsearch.compute.data.BlockFactory; -import org.elasticsearch.compute.data.BlockUtils; import org.elasticsearch.compute.data.Page; import org.elasticsearch.core.Releasable; import org.elasticsearch.core.Releasables; @@ -44,7 +43,11 @@ public EvalOperator(BlockFactory blockFactory, ExpressionEvaluator evaluator) { @Override protected Page process(Page page) { Block.Ref ref = evaluator.eval(page); - Block block = ref.floating() ? ref.block() : BlockUtils.deepCopyOf(ref.block(), blockFactory); + Block block = ref.block(); + if (ref.floating() == false) { + // We take ownership of this block, so we need to shallow copy (incRef) to avoid double releases. + block.incRef(); + } return page.appendBlock(block); } diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/FilterOperator.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/FilterOperator.java index be4996e129d7b..d3e86352d1f29 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/FilterOperator.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/FilterOperator.java @@ -78,7 +78,7 @@ protected Page process(Page page) { } success = true; } finally { - Releasables.closeExpectNoException(page::releaseBlocks); + page.releaseBlocks(); if (success == false) { Releasables.closeExpectNoException(filteredBlocks); } diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/HashAggregationOperator.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/HashAggregationOperator.java index 4b26b74b42a1d..39068787f3c9e 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/HashAggregationOperator.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/HashAggregationOperator.java @@ -161,7 +161,7 @@ public void finish() { } finally { // selected should always be closed if (selected != null) { - Releasables.closeExpectNoException(selected.asBlock()); // we always close blocks, not vectors + selected.close(); } if (success == false && blocks != null) { Releasables.closeExpectNoException(blocks); diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/LimitOperator.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/LimitOperator.java index a41057386d365..bcd2ffa1f3855 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/LimitOperator.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/LimitOperator.java @@ -100,11 +100,12 @@ public Page getOutput() { } success = true; } finally { - Releasables.closeExpectNoException(lastInput::releaseBlocks); - lastInput = null; if (success == false) { - Releasables.closeExpectNoException(blocks); + Releasables.closeExpectNoException(lastInput::releaseBlocks, Releasables.wrap(blocks)); + } else { + lastInput.releaseBlocks(); } + lastInput = null; } result = new Page(blocks); limitRemaining = 0; diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/OrdinalsGroupingOperator.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/OrdinalsGroupingOperator.java index 07494f97cfd6d..2e1cbf9a1135d 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/OrdinalsGroupingOperator.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/OrdinalsGroupingOperator.java @@ -7,6 +7,7 @@ package org.elasticsearch.compute.operator; +import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.SortedSetDocValues; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.BytesRefBuilder; @@ -32,7 +33,7 @@ import org.elasticsearch.compute.operator.HashAggregationOperator.GroupSpec; import org.elasticsearch.core.Releasable; import org.elasticsearch.core.Releasables; -import org.elasticsearch.index.mapper.BlockDocValuesReader; +import org.elasticsearch.index.mapper.BlockLoader; import java.io.IOException; import java.io.UncheckedIOException; @@ -52,7 +53,8 @@ */ public class OrdinalsGroupingOperator implements Operator { public record OrdinalsGroupingOperatorFactory( - List readerFactories, + List blockLoaders, + List readers, ElementType groupingElementType, int docChannel, String groupingField, @@ -64,7 +66,8 @@ public record OrdinalsGroupingOperatorFactory( @Override public Operator get(DriverContext driverContext) { return new OrdinalsGroupingOperator( - readerFactories, + blockLoaders, + readers, groupingElementType, docChannel, groupingField, @@ -81,7 +84,8 @@ public String describe() { } } - private final List readerFactories; + private final List blockLoaders; + private final List readers; private final int docChannel; private final String groupingField; @@ -99,7 +103,8 @@ public String describe() { private ValuesAggregator valuesAggregator; public OrdinalsGroupingOperator( - List readerFactories, + List blockLoaders, + List readers, ElementType groupingElementType, int docChannel, String groupingField, @@ -109,7 +114,8 @@ public OrdinalsGroupingOperator( DriverContext driverContext ) { Objects.requireNonNull(aggregatorFactories); - this.readerFactories = readerFactories; + this.blockLoaders = blockLoaders; + this.readers = readers; this.groupingElementType = groupingElementType; this.docChannel = docChannel; this.groupingField = groupingField; @@ -131,10 +137,10 @@ public void addInput(Page page) { requireNonNull(page, "page is null"); DocVector docVector = page.getBlock(docChannel).asVector(); final int shardIndex = docVector.shards().getInt(0); - final var readerFactory = readerFactories.get(shardIndex); + final var blockLoader = blockLoaders.get(shardIndex); boolean pagePassed = false; try { - if (docVector.singleSegmentNonDecreasing() && readerFactory.supportsOrdinals()) { + if (docVector.singleSegmentNonDecreasing() && blockLoader.supportsOrdinals()) { final IntVector segmentIndexVector = docVector.segments(); assert segmentIndexVector.isConstant(); final OrdinalSegmentAggregator ordinalAggregator = this.ordinalAggregators.computeIfAbsent( @@ -144,7 +150,7 @@ public void addInput(Page page) { return new OrdinalSegmentAggregator( driverContext.blockFactory(), this::createGroupingAggregators, - () -> readerFactory.ordinals(k.segmentIndex), + () -> blockLoader.ordinals(readers.get(k.shardIndex).leaves().get(k.segmentIndex)), bigArrays ); } catch (IOException e) { @@ -158,7 +164,8 @@ public void addInput(Page page) { if (valuesAggregator == null) { int channelIndex = page.getBlockCount(); // extractor will append a new block at the end valuesAggregator = new ValuesAggregator( - readerFactories, + blockLoaders, + readers, groupingElementType, docChannel, groupingField, @@ -458,7 +465,8 @@ private static class ValuesAggregator implements Releasable { private final HashAggregationOperator aggregator; ValuesAggregator( - List factories, + List blockLoaders, + List readers, ElementType groupingElementType, int docChannel, String groupingField, @@ -467,7 +475,12 @@ private static class ValuesAggregator implements Releasable { int maxPageSize, DriverContext driverContext ) { - this.extractor = new ValuesSourceReaderOperator(BlockFactory.getNonBreakingInstance(), factories, docChannel, groupingField); + this.extractor = new ValuesSourceReaderOperator( + BlockFactory.getNonBreakingInstance(), + List.of(new ValuesSourceReaderOperator.FieldInfo(groupingField, blockLoaders)), + readers, + docChannel + ); this.aggregator = new HashAggregationOperator( aggregatorFactories, () -> BlockHash.build(List.of(new GroupSpec(channelIndex, groupingElementType)), driverContext, maxPageSize, false), diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/ProjectOperator.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/ProjectOperator.java index 6e52a5351de58..4f2790d1d1e53 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/ProjectOperator.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/ProjectOperator.java @@ -67,8 +67,13 @@ protected Page process(Page page) { } var block = page.getBlock(source); blocks[b++] = block; + block.incRef(); } - return page.newPageAndRelease(blocks); + int positionCount = page.getPositionCount(); + page.releaseBlocks(); + // Use positionCount explicitly to avoid re-computing - also, if the projection is empty, there may be + // no more blocks left to determine the positionCount from. + return new Page(positionCount, blocks); } @Override diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/OperatorTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/OperatorTests.java index b45f597553e1b..d9730d3f602c7 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/OperatorTests.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/OperatorTests.java @@ -45,7 +45,6 @@ import org.elasticsearch.compute.data.IntVector; import org.elasticsearch.compute.data.LongBlock; import org.elasticsearch.compute.data.Page; -import org.elasticsearch.compute.lucene.BlockReaderFactories; import org.elasticsearch.compute.lucene.DataPartitioning; import org.elasticsearch.compute.lucene.LuceneOperator; import org.elasticsearch.compute.lucene.LuceneSourceOperator; @@ -230,9 +229,8 @@ public String toString() { } }, new OrdinalsGroupingOperator( - List.of( - BlockReaderFactories.loaderToFactory(reader, new KeywordFieldMapper.KeywordFieldType("g").blockLoader(null)) - ), + List.of(new KeywordFieldMapper.KeywordFieldType("g").blockLoader(null)), + List.of(reader), ElementType.BYTES_REF, 0, gField, diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/BasicBlockTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/BasicBlockTests.java index 0a36617f35b18..2a49feeab9a30 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/BasicBlockTests.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/BasicBlockTests.java @@ -39,7 +39,6 @@ import static org.mockito.Mockito.when; public class BasicBlockTests extends ESTestCase { - final CircuitBreaker breaker = new MockBigArrays.LimitedBreaker("esql-test-breaker", ByteSizeValue.ofGb(1)); final BigArrays bigArrays = new MockBigArrays(PageCacheRecycler.NON_RECYCLING_INSTANCE, mockBreakerService(breaker)); final BlockFactory blockFactory = BlockFactory.getInstance(breaker, bigArrays); @@ -1004,6 +1003,12 @@ static void assertCannotDoubleRelease(Block block) { assertThat(ex.getMessage(), containsString("can't release already released block")); } + static void assertCannotReleaseIfVectorAlreadyReleased(Block block) { + var ex = expectThrows(IllegalStateException.class, () -> block.close()); + assertThat(ex.getMessage(), containsString("can't release block")); + assertThat(ex.getMessage(), containsString("containing already released vector")); + } + static void assertCannotReadFromPage(Page page) { var e = expectThrows(IllegalStateException.class, () -> page.getBlock(0)); assertThat(e.getMessage(), containsString("can't read released block")); @@ -1028,4 +1033,156 @@ static CircuitBreakerService mockBreakerService(CircuitBreaker breaker) { when(breakerService.getBreaker(CircuitBreaker.REQUEST)).thenReturn(breaker); return breakerService; } + + public void testRefCountingArrayBlock() { + Block block = randomArrayBlock(); + assertThat(breaker.getUsed(), greaterThan(0L)); + assertRefCountingBehavior(block); + assertThat(breaker.getUsed(), is(0L)); + } + + public void testRefCountingConstantNullBlock() { + Block block = blockFactory.newConstantNullBlock(10); + assertThat(breaker.getUsed(), greaterThan(0L)); + assertRefCountingBehavior(block); + assertThat(breaker.getUsed(), is(0L)); + } + + public void testRefCountingDocBlock() { + int positionCount = randomIntBetween(0, 100); + DocBlock block = new DocVector(intVector(positionCount), intVector(positionCount), intVector(positionCount), true).asBlock(); + assertThat(breaker.getUsed(), greaterThan(0L)); + assertRefCountingBehavior(block); + assertThat(breaker.getUsed(), is(0L)); + } + + public void testRefCountingVectorBlock() { + Block block = randomNonDocVector().asBlock(); + assertThat(breaker.getUsed(), greaterThan(0L)); + assertRefCountingBehavior(block); + assertThat(breaker.getUsed(), is(0L)); + } + + // Take a block with exactly 1 reference and assert that ref counting works fine. + static void assertRefCountingBehavior(Block b) { + assertTrue(b.hasReferences()); + int numShallowCopies = randomIntBetween(0, 15); + for (int i = 0; i < numShallowCopies; i++) { + if (randomBoolean()) { + b.incRef(); + } else { + assertTrue(b.tryIncRef()); + } + } + + for (int i = 0; i < numShallowCopies; i++) { + if (randomBoolean()) { + b.close(); + } else { + // closing and decRef'ing must be equivalent + assertFalse(b.decRef()); + } + assertTrue(b.hasReferences()); + } + + if (randomBoolean()) { + b.close(); + } else { + assertTrue(b.decRef()); + } + + assertFalse(b.hasReferences()); + assertFalse(b.tryIncRef()); + + expectThrows(IllegalStateException.class, b::close); + expectThrows(IllegalStateException.class, b::incRef); + } + + public void testReleasedVectorInvalidatesBlockState() { + Vector vector = randomNonDocVector(); + Block block = vector.asBlock(); + + int numRefs = randomIntBetween(1, 10); + for (int i = 0; i < numRefs - 1; i++) { + block.incRef(); + } + + vector.close(); + assertEquals(false, block.tryIncRef()); + expectThrows(IllegalStateException.class, block::close); + expectThrows(IllegalStateException.class, block::incRef); + } + + public void testReleasedDocVectorInvalidatesBlockState() { + int positionCount = randomIntBetween(0, 100); + DocVector vector = new DocVector(intVector(positionCount), intVector(positionCount), intVector(positionCount), true); + DocBlock block = vector.asBlock(); + + int numRefs = randomIntBetween(1, 10); + for (int i = 0; i < numRefs - 1; i++) { + block.incRef(); + } + + vector.close(); + assertEquals(false, block.tryIncRef()); + expectThrows(IllegalStateException.class, block::close); + expectThrows(IllegalStateException.class, block::incRef); + } + + private IntVector intVector(int positionCount) { + return blockFactory.newIntArrayVector(IntStream.range(0, positionCount).toArray(), positionCount); + } + + private Vector randomNonDocVector() { + int positionCount = randomIntBetween(0, 100); + int vectorType = randomIntBetween(0, 4); + + return switch (vectorType) { + case 0 -> blockFactory.newConstantBooleanVector(true, positionCount); + case 1 -> blockFactory.newConstantBytesRefVector(new BytesRef(), positionCount); + case 2 -> blockFactory.newConstantDoubleVector(1.0, positionCount); + case 3 -> blockFactory.newConstantIntVector(1, positionCount); + default -> blockFactory.newConstantLongVector(1L, positionCount); + }; + } + + private Block randomArrayBlock() { + int positionCount = randomIntBetween(0, 100); + int arrayType = randomIntBetween(0, 4); + + return switch (arrayType) { + case 0 -> { + boolean[] values = new boolean[positionCount]; + Arrays.fill(values, true); + + yield blockFactory.newBooleanArrayBlock(values, positionCount, new int[] {}, new BitSet(), randomOrdering()); + } + case 1 -> { + BytesRefArray values = new BytesRefArray(positionCount, BigArrays.NON_RECYCLING_INSTANCE); + for (int i = 0; i < positionCount; i++) { + values.append(new BytesRef(randomByteArrayOfLength(between(1, 20)))); + } + + yield blockFactory.newBytesRefArrayBlock(values, positionCount, new int[] {}, new BitSet(), randomOrdering()); + } + case 2 -> { + double[] values = new double[positionCount]; + Arrays.fill(values, 1.0); + + yield blockFactory.newDoubleArrayBlock(values, positionCount, new int[] {}, new BitSet(), randomOrdering()); + } + case 3 -> { + int[] values = new int[positionCount]; + Arrays.fill(values, 1); + + yield blockFactory.newIntArrayBlock(values, positionCount, new int[] {}, new BitSet(), randomOrdering()); + } + default -> { + long[] values = new long[positionCount]; + Arrays.fill(values, 1L); + + yield blockFactory.newLongArrayBlock(values, positionCount, new int[] {}, new BitSet(), randomOrdering()); + } + }; + } } diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/BasicPageTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/BasicPageTests.java index 23a257e7afbbe..25cd9ed5b9fe5 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/BasicPageTests.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/BasicPageTests.java @@ -205,16 +205,6 @@ public void testPageMultiRelease() { page.releaseBlocks(); } - public void testNewPageAndRelease() { - int positions = randomInt(1024); - var blockA = new IntArrayVector(IntStream.range(0, positions).toArray(), positions).asBlock(); - var blockB = new IntArrayVector(IntStream.range(0, positions).toArray(), positions).asBlock(); - Page page = new Page(blockA, blockB); - Page newPage = page.newPageAndRelease(blockA); - assertThat(blockA.isReleased(), is(false)); - assertThat(blockB.isReleased(), is(true)); - } - BytesRefArray bytesRefArrayOf(String... values) { var array = new BytesRefArray(values.length, bigArrays); Arrays.stream(values).map(BytesRef::new).forEach(array::append); diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/ValuesSourceReaderOperatorTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/ValuesSourceReaderOperatorTests.java index 269a478560bac..76810dbf2e3bc 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/ValuesSourceReaderOperatorTests.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/ValuesSourceReaderOperatorTests.java @@ -9,9 +9,12 @@ import org.apache.lucene.document.Document; import org.apache.lucene.document.DoubleDocValuesField; +import org.apache.lucene.document.FieldType; import org.apache.lucene.document.NumericDocValuesField; import org.apache.lucene.document.SortedDocValuesField; import org.apache.lucene.document.SortedNumericDocValuesField; +import org.apache.lucene.document.StoredField; +import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexableField; import org.apache.lucene.index.NoMergePolicy; @@ -23,6 +26,8 @@ import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.NumericUtils; import org.elasticsearch.common.Randomness; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.lucene.Lucene; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.util.BigArrays; import org.elasticsearch.compute.data.Block; @@ -46,20 +51,41 @@ import org.elasticsearch.compute.operator.PageConsumerOperator; import org.elasticsearch.compute.operator.SourceOperator; import org.elasticsearch.core.IOUtils; +import org.elasticsearch.index.IndexMode; +import org.elasticsearch.index.IndexVersion; +import org.elasticsearch.index.mapper.BlockLoader; import org.elasticsearch.index.mapper.BooleanFieldMapper; +import org.elasticsearch.index.mapper.IdFieldMapper; import org.elasticsearch.index.mapper.KeywordFieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.NumberFieldMapper; +import org.elasticsearch.index.mapper.ProvidedIdFieldMapper; +import org.elasticsearch.index.mapper.SourceFieldMapper; +import org.elasticsearch.index.mapper.TextFieldMapper; +import org.elasticsearch.index.mapper.TextSearchInfo; +import org.elasticsearch.index.mapper.TsidExtractingIdFieldMapper; +import org.elasticsearch.search.lookup.SearchLookup; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.json.JsonXContent; +import org.hamcrest.Matcher; import org.junit.After; import java.io.IOException; import java.util.ArrayList; +import java.util.Collections; import java.util.List; +import java.util.Map; +import java.util.Set; import java.util.stream.IntStream; import static org.elasticsearch.compute.lucene.LuceneSourceOperatorTests.mockSearchContext; +import static org.elasticsearch.test.MapMatcher.assertMap; +import static org.elasticsearch.test.MapMatcher.matchesMap; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasSize; +import static org.hamcrest.Matchers.lessThanOrEqualTo; +import static org.hamcrest.Matchers.not; +import static org.hamcrest.Matchers.nullValue; /** * Tests for {@link ValuesSourceReaderOperator}. Turns off {@link HandleLimitFS} @@ -86,19 +112,48 @@ public void closeIndex() throws IOException { @Override protected Operator.OperatorFactory simple(BigArrays bigArrays) { - return factory(reader, new NumberFieldMapper.NumberFieldType("long", NumberFieldMapper.NumberType.LONG)); + if (reader == null) { + // Init a reader if one hasn't been built, so things don't blow up + try { + initIndex(100); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + return factory(reader, docValuesNumberField("long", NumberFieldMapper.NumberType.LONG)); } static Operator.OperatorFactory factory(IndexReader reader, MappedFieldType ft) { - return new ValuesSourceReaderOperator.ValuesSourceReaderOperatorFactory( - List.of(BlockReaderFactories.loaderToFactory(reader, ft.blockLoader(null))), - 0, - ft.name() + return factory(reader, ft.name(), ft.blockLoader(null)); + } + + static Operator.OperatorFactory factory(IndexReader reader, String name, BlockLoader loader) { + return new ValuesSourceReaderOperator.Factory( + List.of(new ValuesSourceReaderOperator.FieldInfo(name, List.of(loader))), + List.of(reader), + 0 ); } @Override protected SourceOperator simpleInput(BlockFactory blockFactory, int size) { + try { + initIndex(size); + } catch (IOException e) { + throw new RuntimeException(e); + } + var luceneFactory = new LuceneSourceOperator.Factory( + List.of(mockSearchContext(reader)), + ctx -> new MatchAllDocsQuery(), + randomFrom(DataPartitioning.values()), + randomIntBetween(1, 10), + randomPageSize(), + LuceneOperator.NO_LIMIT + ); + return luceneFactory.get(driverContext()); + } + + private void initIndex(int size) throws IOException { // The test wants more than one segment. We shoot for about 10. int commitEvery = Math.max(1, size / 10); try ( @@ -110,40 +165,68 @@ protected SourceOperator simpleInput(BlockFactory blockFactory, int size) { ) { for (int d = 0; d < size; d++) { List doc = new ArrayList<>(); + doc.add(IdFieldMapper.standardIdField("id")); doc.add(new SortedNumericDocValuesField("key", d)); + doc.add(new SortedNumericDocValuesField("int", d)); + doc.add(new SortedNumericDocValuesField("short", (short) d)); + doc.add(new SortedNumericDocValuesField("byte", (byte) d)); doc.add(new SortedNumericDocValuesField("long", d)); doc.add( new KeywordFieldMapper.KeywordField("kwd", new BytesRef(Integer.toString(d)), KeywordFieldMapper.Defaults.FIELD_TYPE) ); + doc.add(new StoredField("stored_kwd", new BytesRef(Integer.toString(d)))); + doc.add(new StoredField("stored_text", Integer.toString(d))); doc.add(new SortedNumericDocValuesField("bool", d % 2 == 0 ? 1 : 0)); doc.add(new SortedNumericDocValuesField("double", NumericUtils.doubleToSortableLong(d / 123_456d))); for (int v = 0; v <= d % 3; v++) { - doc.add( - new KeywordFieldMapper.KeywordField("mv_kwd", new BytesRef(PREFIX[v] + d), KeywordFieldMapper.Defaults.FIELD_TYPE) - ); doc.add(new SortedNumericDocValuesField("mv_bool", v % 2 == 0 ? 1 : 0)); - doc.add(new SortedNumericDocValuesField("mv_key", 1_000 * d + v)); + doc.add(new SortedNumericDocValuesField("mv_int", 1_000 * d + v)); + doc.add(new SortedNumericDocValuesField("mv_short", (short) (2_000 * d + v))); + doc.add(new SortedNumericDocValuesField("mv_byte", (byte) (3_000 * d + v))); doc.add(new SortedNumericDocValuesField("mv_long", -1_000 * d + v)); doc.add(new SortedNumericDocValuesField("mv_double", NumericUtils.doubleToSortableLong(d / 123_456d + v))); + doc.add( + new KeywordFieldMapper.KeywordField("mv_kwd", new BytesRef(PREFIX[v] + d), KeywordFieldMapper.Defaults.FIELD_TYPE) + ); + doc.add(new StoredField("mv_stored_kwd", new BytesRef(PREFIX[v] + d))); + doc.add(new StoredField("mv_stored_text", PREFIX[v] + d)); + } + XContentBuilder source = JsonXContent.contentBuilder(); + source.startObject(); + source.field("source_kwd", Integer.toString(d)); + source.startArray("mv_source_kwd"); + for (int v = 0; v <= d % 3; v++) { + source.value(PREFIX[v] + d); + } + source.endArray(); + source.field("source_text", Integer.toString(d)); + source.startArray("mv_source_text"); + for (int v = 0; v <= d % 3; v++) { + source.value(PREFIX[v] + d); } + source.endArray(); + source.field("source_long", (long) d); + source.startArray("mv_source_long"); + for (int v = 0; v <= d % 3; v++) { + source.value((long) (-1_000 * d + v)); + } + source.endArray(); + source.field("source_int", d); + source.startArray("mv_source_int"); + for (int v = 0; v <= d % 3; v++) { + source.value(1_000 * d + v); + } + source.endArray(); + + source.endObject(); + doc.add(new StoredField(SourceFieldMapper.NAME, BytesReference.bytes(source).toBytesRef())); writer.addDocument(doc); if (d % commitEvery == 0) { writer.commit(); } } reader = writer.getReader(); - } catch (IOException e) { - throw new RuntimeException(e); } - var luceneFactory = new LuceneSourceOperator.Factory( - List.of(mockSearchContext(reader)), - ctx -> new MatchAllDocsQuery(), - randomFrom(DataPartitioning.values()), - randomIntBetween(1, 10), - randomPageSize(), - LuceneOperator.NO_LIMIT - ); - return luceneFactory.get(driverContext()); } @Override @@ -184,7 +267,8 @@ public void testLoadAll() { DriverContext driverContext = driverContext(); loadSimpleAndAssert( driverContext, - CannedSourceOperator.collectPages(simpleInput(driverContext.blockFactory(), between(100, 5000))) + CannedSourceOperator.collectPages(simpleInput(driverContext.blockFactory(), between(100, 5000))), + Block.MvOrdering.DEDUPLICATED_AND_SORTED_ASCENDING ); } @@ -196,13 +280,18 @@ public void testLoadAllInOnePage() { CannedSourceOperator.mergePages( CannedSourceOperator.collectPages(simpleInput(driverContext.blockFactory(), between(100, 5000))) ) - ) + ), + Block.MvOrdering.UNORDERED ); } public void testEmpty() { DriverContext driverContext = driverContext(); - loadSimpleAndAssert(driverContext, CannedSourceOperator.collectPages(simpleInput(driverContext.blockFactory(), 0))); + loadSimpleAndAssert( + driverContext, + CannedSourceOperator.collectPages(simpleInput(driverContext.blockFactory(), 0)), + Block.MvOrdering.UNORDERED + ); } public void testLoadAllInOnePageShuffled() { @@ -219,99 +308,681 @@ public void testLoadAllInOnePageShuffled() { shuffledBlocks[b] = source.getBlock(b).filter(shuffleArray); } source = new Page(shuffledBlocks); - loadSimpleAndAssert(driverContext, List.of(source)); - } - - private void loadSimpleAndAssert(DriverContext driverContext, List input) { - List operators = List.of( - factory(reader, new NumberFieldMapper.NumberFieldType("key", NumberFieldMapper.NumberType.INTEGER)).get(driverContext), - factory(reader, new NumberFieldMapper.NumberFieldType("long", NumberFieldMapper.NumberType.LONG)).get(driverContext), - factory(reader, new KeywordFieldMapper.KeywordFieldType("kwd")).get(driverContext), - factory(reader, new KeywordFieldMapper.KeywordFieldType("mv_kwd")).get(driverContext), - factory(reader, new BooleanFieldMapper.BooleanFieldType("bool")).get(driverContext), - factory(reader, new BooleanFieldMapper.BooleanFieldType("mv_bool")).get(driverContext), - factory(reader, new NumberFieldMapper.NumberFieldType("mv_key", NumberFieldMapper.NumberType.INTEGER)).get(driverContext), - factory(reader, new NumberFieldMapper.NumberFieldType("mv_long", NumberFieldMapper.NumberType.LONG)).get(driverContext), - factory(reader, new NumberFieldMapper.NumberFieldType("double", NumberFieldMapper.NumberType.DOUBLE)).get(driverContext), - factory(reader, new NumberFieldMapper.NumberFieldType("mv_double", NumberFieldMapper.NumberType.DOUBLE)).get(driverContext) + loadSimpleAndAssert(driverContext, List.of(source), Block.MvOrdering.UNORDERED); + } + + private static ValuesSourceReaderOperator.FieldInfo fieldInfo(MappedFieldType ft) { + return new ValuesSourceReaderOperator.FieldInfo(ft.name(), List.of(ft.blockLoader(new MappedFieldType.BlockLoaderContext() { + @Override + public String indexName() { + return "test_index"; + } + + @Override + public SearchLookup lookup() { + throw new UnsupportedOperationException(); + } + + @Override + public Set sourcePaths(String name) { + return Set.of(name); + } + }))); + } + + private void loadSimpleAndAssert(DriverContext driverContext, List input, Block.MvOrdering docValuesMvOrdering) { + List cases = infoAndChecksForEachType(docValuesMvOrdering); + + List operators = new ArrayList<>(); + operators.add( + new ValuesSourceReaderOperator.Factory( + List.of(fieldInfo(docValuesNumberField("key", NumberFieldMapper.NumberType.INTEGER))), + List.of(reader), + 0 + ).get(driverContext) ); + List tests = new ArrayList<>(); + while (cases.isEmpty() == false) { + List b = randomNonEmptySubsetOf(cases); + cases.removeAll(b); + tests.addAll(b); + operators.add( + new ValuesSourceReaderOperator.Factory(b.stream().map(i -> i.info).toList(), List.of(reader), 0).get(driverContext) + ); + } List results = drive(operators, input.iterator(), driverContext); assertThat(results, hasSize(input.size())); - for (Page p : results) { - assertThat(p.getBlockCount(), equalTo(11)); - IntVector keys = p.getBlock(1).asVector(); - LongVector longs = p.getBlock(2).asVector(); - BytesRefVector keywords = p.getBlock(3).asVector(); - BytesRefBlock mvKeywords = p.getBlock(4); - BooleanVector bools = p.getBlock(5).asVector(); - BooleanBlock mvBools = p.getBlock(6); - IntBlock mvInts = p.getBlock(7); - LongBlock mvLongs = p.getBlock(8); - DoubleVector doubles = p.getBlock(9).asVector(); - DoubleBlock mvDoubles = p.getBlock(10); - - for (int i = 0; i < p.getPositionCount(); i++) { - int key = keys.getInt(i); - assertThat(longs.getLong(i), equalTo((long) key)); - assertThat(keywords.getBytesRef(i, new BytesRef()).utf8ToString(), equalTo(Integer.toString(key))); - - assertThat(mvKeywords.getValueCount(i), equalTo(key % 3 + 1)); - int offset = mvKeywords.getFirstValueIndex(i); - for (int v = 0; v <= key % 3; v++) { - assertThat(mvKeywords.getBytesRef(offset + v, new BytesRef()).utf8ToString(), equalTo(PREFIX[v] + key)); - } - if (key % 3 > 0) { - assertThat(mvKeywords.mvOrdering(), equalTo(Block.MvOrdering.DEDUPLICATED_AND_SORTED_ASCENDING)); + for (Page page : results) { + assertThat(page.getBlockCount(), equalTo(tests.size() + 2 /* one for doc and one for keys */)); + IntVector keys = page.getBlock(1).asVector(); + for (int p = 0; p < page.getPositionCount(); p++) { + int key = keys.getInt(p); + for (int i = 0; i < tests.size(); i++) { + try { + tests.get(i).checkResults.check(page.getBlock(2 + i), p, key); + } catch (AssertionError e) { + throw new AssertionError("error checking " + tests.get(i).info.name() + "[" + p + "]: " + e.getMessage(), e); + } } + } + } + for (Operator op : operators) { + assertThat(((ValuesSourceReaderOperator) op).status().pagesProcessed(), equalTo(input.size())); + } + assertDriverContext(driverContext); + } - assertThat(bools.getBoolean(i), equalTo(key % 2 == 0)); - assertThat(mvBools.getValueCount(i), equalTo(key % 3 + 1)); - offset = mvBools.getFirstValueIndex(i); - for (int v = 0; v <= key % 3; v++) { - assertThat(mvBools.getBoolean(offset + v), equalTo(BOOLEANS[key % 3][v])); - } - if (key % 3 > 0) { - assertThat(mvBools.mvOrdering(), equalTo(Block.MvOrdering.DEDUPLICATED_AND_SORTED_ASCENDING)); - } + interface CheckResults { + void check(Block block, int position, int key); + } - assertThat(mvInts.getValueCount(i), equalTo(key % 3 + 1)); - offset = mvInts.getFirstValueIndex(i); - for (int v = 0; v <= key % 3; v++) { - assertThat(mvInts.getInt(offset + v), equalTo(1_000 * key + v)); - } - if (key % 3 > 0) { - assertThat(mvInts.mvOrdering(), equalTo(Block.MvOrdering.DEDUPLICATED_AND_SORTED_ASCENDING)); - } + interface CheckReaders { + void check(boolean forcedRowByRow, int pageCount, int segmentCount, Map readersBuilt); + } - assertThat(mvLongs.getValueCount(i), equalTo(key % 3 + 1)); - offset = mvLongs.getFirstValueIndex(i); - for (int v = 0; v <= key % 3; v++) { - assertThat(mvLongs.getLong(offset + v), equalTo(-1_000L * key + v)); - } - if (key % 3 > 0) { - assertThat(mvLongs.mvOrdering(), equalTo(Block.MvOrdering.DEDUPLICATED_AND_SORTED_ASCENDING)); - } + record FieldCase(ValuesSourceReaderOperator.FieldInfo info, CheckResults checkResults, CheckReaders checkReaders) { + FieldCase(MappedFieldType ft, CheckResults checkResults, CheckReaders checkReaders) { + this(fieldInfo(ft), checkResults, checkReaders); + } + } + + /** + * Asserts that {@link ValuesSourceReaderOperator#status} claims that only + * the expected readers are built after loading singleton pages. + */ + // @Repeat(iterations = 100) + public void testLoadAllStatus() { + DriverContext driverContext = driverContext(); + testLoadAllStatus(false); + } + + /** + * Asserts that {@link ValuesSourceReaderOperator#status} claims that only + * the expected readers are built after loading non-singleton pages. + */ + // @Repeat(iterations = 100) + public void testLoadAllStatusAllInOnePage() { + testLoadAllStatus(true); + } + + private void testLoadAllStatus(boolean allInOnePage) { + DriverContext driverContext = driverContext(); + List input = CannedSourceOperator.collectPages(simpleInput(driverContext.blockFactory(), between(100, 5000))); + List cases = infoAndChecksForEachType(Block.MvOrdering.DEDUPLICATED_AND_SORTED_ASCENDING); + // Build one operator for each field, so we get a unique map to assert on + List operators = cases.stream() + .map(i -> new ValuesSourceReaderOperator.Factory(List.of(i.info), List.of(reader), 0).get(driverContext)) + .toList(); + if (allInOnePage) { + input = List.of(CannedSourceOperator.mergePages(input)); + } + drive(operators, input.iterator(), driverContext); + for (int i = 0; i < cases.size(); i++) { + ValuesSourceReaderOperator.Status status = (ValuesSourceReaderOperator.Status) operators.get(i).status(); + assertThat(status.pagesProcessed(), equalTo(input.size())); + FieldCase fc = cases.get(i); + fc.checkReaders.check(allInOnePage, input.size(), reader.leaves().size(), status.readersBuilt()); + } + } + + private List infoAndChecksForEachType(Block.MvOrdering docValuesMvOrdering) { + Checks checks = new Checks(docValuesMvOrdering); + List r = new ArrayList<>(); + r.add( + new FieldCase(docValuesNumberField("long", NumberFieldMapper.NumberType.LONG), checks::longs, StatusChecks::longsFromDocValues) + ); + r.add( + new FieldCase( + docValuesNumberField("mv_long", NumberFieldMapper.NumberType.LONG), + checks::mvLongsFromDocValues, + StatusChecks::mvLongsFromDocValues + ) + ); + r.add( + new FieldCase(sourceNumberField("source_long", NumberFieldMapper.NumberType.LONG), checks::longs, StatusChecks::longsFromSource) + ); + r.add( + new FieldCase( + sourceNumberField("mv_source_long", NumberFieldMapper.NumberType.LONG), + checks::mvLongsUnordered, + StatusChecks::mvLongsFromSource + ) + ); + r.add( + new FieldCase(docValuesNumberField("int", NumberFieldMapper.NumberType.INTEGER), checks::ints, StatusChecks::intsFromDocValues) + ); + r.add( + new FieldCase( + docValuesNumberField("mv_int", NumberFieldMapper.NumberType.INTEGER), + checks::mvIntsFromDocValues, + StatusChecks::mvIntsFromDocValues + ) + ); + r.add( + new FieldCase(sourceNumberField("source_int", NumberFieldMapper.NumberType.INTEGER), checks::ints, StatusChecks::intsFromSource) + ); + r.add( + new FieldCase( + sourceNumberField("mv_source_int", NumberFieldMapper.NumberType.INTEGER), + checks::mvIntsUnordered, + StatusChecks::mvIntsFromSource + ) + ); + r.add( + new FieldCase( + docValuesNumberField("short", NumberFieldMapper.NumberType.SHORT), + checks::shorts, + StatusChecks::shortsFromDocValues + ) + ); + r.add( + new FieldCase( + docValuesNumberField("mv_short", NumberFieldMapper.NumberType.SHORT), + checks::mvShorts, + StatusChecks::mvShortsFromDocValues + ) + ); + r.add( + new FieldCase(docValuesNumberField("byte", NumberFieldMapper.NumberType.BYTE), checks::bytes, StatusChecks::bytesFromDocValues) + ); + r.add( + new FieldCase( + docValuesNumberField("mv_byte", NumberFieldMapper.NumberType.BYTE), + checks::mvBytes, + StatusChecks::mvBytesFromDocValues + ) + ); + r.add( + new FieldCase( + docValuesNumberField("double", NumberFieldMapper.NumberType.DOUBLE), + checks::doubles, + StatusChecks::doublesFromDocValues + ) + ); + r.add( + new FieldCase( + docValuesNumberField("mv_double", NumberFieldMapper.NumberType.DOUBLE), + checks::mvDoubles, + StatusChecks::mvDoublesFromDocValues + ) + ); + r.add(new FieldCase(new BooleanFieldMapper.BooleanFieldType("bool"), checks::bools, StatusChecks::boolFromDocValues)); + r.add(new FieldCase(new BooleanFieldMapper.BooleanFieldType("mv_bool"), checks::mvBools, StatusChecks::mvBoolFromDocValues)); + r.add(new FieldCase(new KeywordFieldMapper.KeywordFieldType("kwd"), checks::strings, StatusChecks::keywordsFromDocValues)); + r.add( + new FieldCase( + new KeywordFieldMapper.KeywordFieldType("mv_kwd"), + checks::mvStringsFromDocValues, + StatusChecks::mvKeywordsFromDocValues + ) + ); + r.add(new FieldCase(storedKeywordField("stored_kwd"), checks::strings, StatusChecks::keywordsFromStored)); + r.add(new FieldCase(storedKeywordField("mv_stored_kwd"), checks::mvStringsUnordered, StatusChecks::mvKeywordsFromStored)); + r.add(new FieldCase(sourceKeywordField("source_kwd"), checks::strings, StatusChecks::keywordsFromSource)); + r.add(new FieldCase(sourceKeywordField("mv_source_kwd"), checks::mvStringsUnordered, StatusChecks::mvKeywordsFromSource)); + r.add(new FieldCase(new TextFieldMapper.TextFieldType("source_text", false), checks::strings, StatusChecks::textFromSource)); + r.add( + new FieldCase( + new TextFieldMapper.TextFieldType("mv_source_text", false), + checks::mvStringsUnordered, + StatusChecks::mvTextFromSource + ) + ); + r.add(new FieldCase(storedTextField("stored_text"), checks::strings, StatusChecks::textFromStored)); + r.add(new FieldCase(storedTextField("mv_stored_text"), checks::mvStringsUnordered, StatusChecks::mvTextFromStored)); + r.add( + new FieldCase( + textFieldWithDelegate("text_with_delegate", new KeywordFieldMapper.KeywordFieldType("kwd")), + checks::strings, + StatusChecks::textWithDelegate + ) + ); + r.add( + new FieldCase( + textFieldWithDelegate("mv_text_with_delegate", new KeywordFieldMapper.KeywordFieldType("mv_kwd")), + checks::mvStringsFromDocValues, + StatusChecks::mvTextWithDelegate + ) + ); + r.add(new FieldCase(new ProvidedIdFieldMapper(() -> false).fieldType(), checks::ids, StatusChecks::id)); + r.add(new FieldCase(TsidExtractingIdFieldMapper.INSTANCE.fieldType(), checks::ids, StatusChecks::id)); + r.add( + new FieldCase( + new ValuesSourceReaderOperator.FieldInfo("constant_bytes", List.of(BlockLoader.constantBytes(new BytesRef("foo")))), + checks::constantBytes, + StatusChecks::constantBytes + ) + ); + r.add( + new FieldCase( + new ValuesSourceReaderOperator.FieldInfo("null", List.of(BlockLoader.CONSTANT_NULLS)), + checks::constantNulls, + StatusChecks::constantNulls + ) + ); + Collections.shuffle(r, random()); + return r; + } + + record Checks(Block.MvOrdering docValuesMvOrdering) { + void longs(Block block, int position, int key) { + LongVector longs = ((LongBlock) block).asVector(); + assertThat(longs.getLong(position), equalTo((long) key)); + } + + void ints(Block block, int position, int key) { + IntVector ints = ((IntBlock) block).asVector(); + assertThat(ints.getInt(position), equalTo(key)); + } + + void shorts(Block block, int position, int key) { + IntVector ints = ((IntBlock) block).asVector(); + assertThat(ints.getInt(position), equalTo((int) (short) key)); + } + + void bytes(Block block, int position, int key) { + IntVector ints = ((IntBlock) block).asVector(); + assertThat(ints.getInt(position), equalTo((int) (byte) key)); + } + + void doubles(Block block, int position, int key) { + DoubleVector doubles = ((DoubleBlock) block).asVector(); + assertThat(doubles.getDouble(position), equalTo(key / 123_456d)); + } + + void strings(Block block, int position, int key) { + BytesRefVector keywords = ((BytesRefBlock) block).asVector(); + assertThat(keywords.getBytesRef(position, new BytesRef()).utf8ToString(), equalTo(Integer.toString(key))); + } + + void bools(Block block, int position, int key) { + BooleanVector bools = ((BooleanBlock) block).asVector(); + assertThat(bools.getBoolean(position), equalTo(key % 2 == 0)); + } + + void ids(Block block, int position, int key) { + BytesRefVector ids = ((BytesRefBlock) block).asVector(); + assertThat(ids.getBytesRef(position, new BytesRef()).utf8ToString(), equalTo("id")); + } + + void constantBytes(Block block, int position, int key) { + BytesRefVector keywords = ((BytesRefBlock) block).asVector(); + assertThat(keywords.getBytesRef(position, new BytesRef()).utf8ToString(), equalTo("foo")); + } + + void constantNulls(Block block, int position, int key) { + assertTrue(block.areAllValuesNull()); + assertTrue(block.isNull(position)); + } + + void mvLongsFromDocValues(Block block, int position, int key) { + mvLongs(block, position, key, docValuesMvOrdering); + } + + void mvLongsUnordered(Block block, int position, int key) { + mvLongs(block, position, key, Block.MvOrdering.UNORDERED); + } + + private void mvLongs(Block block, int position, int key, Block.MvOrdering expectedMv) { + LongBlock longs = (LongBlock) block; + assertThat(longs.getValueCount(position), equalTo(key % 3 + 1)); + int offset = longs.getFirstValueIndex(position); + for (int v = 0; v <= key % 3; v++) { + assertThat(longs.getLong(offset + v), equalTo(-1_000L * key + v)); + } + if (key % 3 > 0) { + assertThat(longs.mvOrdering(), equalTo(expectedMv)); + } + } + + void mvIntsFromDocValues(Block block, int position, int key) { + mvInts(block, position, key, docValuesMvOrdering); + } + + void mvIntsUnordered(Block block, int position, int key) { + mvInts(block, position, key, Block.MvOrdering.UNORDERED); + } + + private void mvInts(Block block, int position, int key, Block.MvOrdering expectedMv) { + IntBlock ints = (IntBlock) block; + assertThat(ints.getValueCount(position), equalTo(key % 3 + 1)); + int offset = ints.getFirstValueIndex(position); + for (int v = 0; v <= key % 3; v++) { + assertThat(ints.getInt(offset + v), equalTo(1_000 * key + v)); + } + if (key % 3 > 0) { + assertThat(ints.mvOrdering(), equalTo(expectedMv)); + } + } + + void mvShorts(Block block, int position, int key) { + IntBlock ints = (IntBlock) block; + assertThat(ints.getValueCount(position), equalTo(key % 3 + 1)); + int offset = ints.getFirstValueIndex(position); + for (int v = 0; v <= key % 3; v++) { + assertThat(ints.getInt(offset + v), equalTo((int) (short) (2_000 * key + v))); + } + if (key % 3 > 0) { + assertThat(ints.mvOrdering(), equalTo(docValuesMvOrdering)); + } + } + + void mvBytes(Block block, int position, int key) { + IntBlock ints = (IntBlock) block; + assertThat(ints.getValueCount(position), equalTo(key % 3 + 1)); + int offset = ints.getFirstValueIndex(position); + for (int v = 0; v <= key % 3; v++) { + assertThat(ints.getInt(offset + v), equalTo((int) (byte) (3_000 * key + v))); + } + if (key % 3 > 0) { + assertThat(ints.mvOrdering(), equalTo(docValuesMvOrdering)); + } + } + + void mvDoubles(Block block, int position, int key) { + DoubleBlock doubles = (DoubleBlock) block; + int offset = doubles.getFirstValueIndex(position); + for (int v = 0; v <= key % 3; v++) { + assertThat(doubles.getDouble(offset + v), equalTo(key / 123_456d + v)); + } + if (key % 3 > 0) { + assertThat(doubles.mvOrdering(), equalTo(docValuesMvOrdering)); + } + } + + void mvStringsFromDocValues(Block block, int position, int key) { + mvStrings(block, position, key, docValuesMvOrdering); + } + + void mvStringsUnordered(Block block, int position, int key) { + mvStrings(block, position, key, Block.MvOrdering.UNORDERED); + } + + void mvStrings(Block block, int position, int key, Block.MvOrdering expectedMv) { + BytesRefBlock text = (BytesRefBlock) block; + assertThat(text.getValueCount(position), equalTo(key % 3 + 1)); + int offset = text.getFirstValueIndex(position); + for (int v = 0; v <= key % 3; v++) { + assertThat(text.getBytesRef(offset + v, new BytesRef()).utf8ToString(), equalTo(PREFIX[v] + key)); + } + if (key % 3 > 0) { + assertThat(text.mvOrdering(), equalTo(expectedMv)); + } + } + + void mvBools(Block block, int position, int key) { + BooleanBlock bools = (BooleanBlock) block; + assertThat(bools.getValueCount(position), equalTo(key % 3 + 1)); + int offset = bools.getFirstValueIndex(position); + for (int v = 0; v <= key % 3; v++) { + assertThat(bools.getBoolean(offset + v), equalTo(BOOLEANS[key % 3][v])); + } + if (key % 3 > 0) { + assertThat(bools.mvOrdering(), equalTo(docValuesMvOrdering)); + } + } + } + + class StatusChecks { + static void longsFromDocValues(boolean forcedRowByRow, int pageCount, int segmentCount, Map readers) { + docValues("long", "Longs", forcedRowByRow, pageCount, segmentCount, readers); + } + + static void longsFromSource(boolean forcedRowByRow, int pageCount, int segmentCount, Map readers) { + source("source_long", "Longs", forcedRowByRow, pageCount, segmentCount, readers); + } + + static void intsFromDocValues(boolean forcedRowByRow, int pageCount, int segmentCount, Map readers) { + docValues("int", "Ints", forcedRowByRow, pageCount, segmentCount, readers); + } + + static void intsFromSource(boolean forcedRowByRow, int pageCount, int segmentCount, Map readers) { + source("source_int", "Ints", forcedRowByRow, pageCount, segmentCount, readers); + } + + static void shortsFromDocValues(boolean forcedRowByRow, int pageCount, int segmentCount, Map readers) { + docValues("short", "Ints", forcedRowByRow, pageCount, segmentCount, readers); + } + + static void bytesFromDocValues(boolean forcedRowByRow, int pageCount, int segmentCount, Map readers) { + docValues("byte", "Ints", forcedRowByRow, pageCount, segmentCount, readers); + } + + static void doublesFromDocValues(boolean forcedRowByRow, int pageCount, int segmentCount, Map readers) { + docValues("double", "Doubles", forcedRowByRow, pageCount, segmentCount, readers); + } + + static void boolFromDocValues(boolean forcedRowByRow, int pageCount, int segmentCount, Map readers) { + docValues("bool", "Booleans", forcedRowByRow, pageCount, segmentCount, readers); + } + + static void keywordsFromDocValues(boolean forcedRowByRow, int pageCount, int segmentCount, Map readers) { + docValues("kwd", "Ordinals", forcedRowByRow, pageCount, segmentCount, readers); + } + + static void keywordsFromStored(boolean forcedRowByRow, int pageCount, int segmentCount, Map readers) { + stored("stored_kwd", "Bytes", forcedRowByRow, pageCount, segmentCount, readers); + } + + static void keywordsFromSource(boolean forcedRowByRow, int pageCount, int segmentCount, Map readers) { + source("source_kwd", "Bytes", forcedRowByRow, pageCount, segmentCount, readers); + } + + static void textFromSource(boolean forcedRowByRow, int pageCount, int segmentCount, Map readers) { + source("source_text", "Bytes", forcedRowByRow, pageCount, segmentCount, readers); + } + + static void textFromStored(boolean forcedRowByRow, int pageCount, int segmentCount, Map readers) { + stored("stored_text", "Bytes", forcedRowByRow, pageCount, segmentCount, readers); + } + + static void mvLongsFromDocValues(boolean forcedRowByRow, int pageCount, int segmentCount, Map readers) { + mvDocValues("mv_long", "Longs", forcedRowByRow, pageCount, segmentCount, readers); + } + + static void mvLongsFromSource(boolean forcedRowByRow, int pageCount, int segmentCount, Map readers) { + source("mv_source_long", "Longs", forcedRowByRow, pageCount, segmentCount, readers); + } + + static void mvIntsFromDocValues(boolean forcedRowByRow, int pageCount, int segmentCount, Map readers) { + mvDocValues("mv_int", "Ints", forcedRowByRow, pageCount, segmentCount, readers); + } + + static void mvIntsFromSource(boolean forcedRowByRow, int pageCount, int segmentCount, Map readers) { + source("mv_source_int", "Ints", forcedRowByRow, pageCount, segmentCount, readers); + } + + static void mvShortsFromDocValues(boolean forcedRowByRow, int pageCount, int segmentCount, Map readers) { + mvDocValues("mv_short", "Ints", forcedRowByRow, pageCount, segmentCount, readers); + } + + static void mvBytesFromDocValues(boolean forcedRowByRow, int pageCount, int segmentCount, Map readers) { + mvDocValues("mv_byte", "Ints", forcedRowByRow, pageCount, segmentCount, readers); + } + + static void mvDoublesFromDocValues(boolean forcedRowByRow, int pageCount, int segmentCount, Map readers) { + mvDocValues("mv_double", "Doubles", forcedRowByRow, pageCount, segmentCount, readers); + } + + static void mvBoolFromDocValues(boolean forcedRowByRow, int pageCount, int segmentCount, Map readers) { + mvDocValues("mv_bool", "Booleans", forcedRowByRow, pageCount, segmentCount, readers); + } + + static void mvKeywordsFromDocValues(boolean forcedRowByRow, int pageCount, int segmentCount, Map readers) { + mvDocValues("mv_kwd", "Ordinals", forcedRowByRow, pageCount, segmentCount, readers); + } + + static void mvKeywordsFromStored(boolean forcedRowByRow, int pageCount, int segmentCount, Map readers) { + stored("mv_stored_kwd", "Bytes", forcedRowByRow, pageCount, segmentCount, readers); + } + + static void mvKeywordsFromSource(boolean forcedRowByRow, int pageCount, int segmentCount, Map readers) { + source("mv_source_kwd", "Bytes", forcedRowByRow, pageCount, segmentCount, readers); + } + + static void mvTextFromStored(boolean forcedRowByRow, int pageCount, int segmentCount, Map readers) { + stored("mv_stored_text", "Bytes", forcedRowByRow, pageCount, segmentCount, readers); + } + + static void mvTextFromSource(boolean forcedRowByRow, int pageCount, int segmentCount, Map readers) { + source("mv_source_text", "Bytes", forcedRowByRow, pageCount, segmentCount, readers); + } + + static void textWithDelegate(boolean forcedRowByRow, int pageCount, int segmentCount, Map readers) { + if (forcedRowByRow) { + assertMap( + readers, + matchesMap().entry( + "text_with_delegate:row_stride:Delegating[to=kwd, impl=BlockDocValuesReader.SingletonOrdinals]", + segmentCount + ) + ); + } else { + assertMap( + readers, + matchesMap().entry( + "text_with_delegate:column_at_a_time:Delegating[to=kwd, impl=BlockDocValuesReader.SingletonOrdinals]", + lessThanOrEqualTo(pageCount) + ) + ); + } + } + + static void mvTextWithDelegate(boolean forcedRowByRow, int pageCount, int segmentCount, Map readers) { + if (forcedRowByRow) { + assertMap( + readers, + matchesMap().entry( + "mv_text_with_delegate:row_stride:Delegating[to=mv_kwd, impl=BlockDocValuesReader.SingletonOrdinals]", + lessThanOrEqualTo(segmentCount) + ) + .entry( + "mv_text_with_delegate:row_stride:Delegating[to=mv_kwd, impl=BlockDocValuesReader.Ordinals]", + lessThanOrEqualTo(segmentCount) + ) + ); + } else { + assertMap( + readers, + matchesMap().entry( + "mv_text_with_delegate:column_at_a_time:Delegating[to=mv_kwd, impl=BlockDocValuesReader.SingletonOrdinals]", + lessThanOrEqualTo(pageCount) + ) + .entry( + "mv_text_with_delegate:column_at_a_time:Delegating[to=mv_kwd, impl=BlockDocValuesReader.Ordinals]", + lessThanOrEqualTo(pageCount) + ) + ); + } + } + + private static void docValues( + String name, + String type, + boolean forcedRowByRow, + int pageCount, + int segmentCount, + Map readers + ) { + if (forcedRowByRow) { + assertMap( + readers, + matchesMap().entry(name + ":row_stride:BlockDocValuesReader.Singleton" + type, lessThanOrEqualTo(segmentCount)) + ); + } else { + assertMap( + readers, + matchesMap().entry(name + ":column_at_a_time:BlockDocValuesReader.Singleton" + type, lessThanOrEqualTo(pageCount)) + ); + } + } - assertThat(doubles.getDouble(i), equalTo(key / 123_456d)); - offset = mvDoubles.getFirstValueIndex(i); - for (int v = 0; v <= key % 3; v++) { - assertThat(mvDoubles.getDouble(offset + v), equalTo(key / 123_456d + v)); + private static void mvDocValues( + String name, + String type, + boolean forcedRowByRow, + int pageCount, + int segmentCount, + Map readers + ) { + if (forcedRowByRow) { + Integer singletons = (Integer) readers.remove(name + ":row_stride:BlockDocValuesReader.Singleton" + type); + if (singletons != null) { + segmentCount -= singletons; } - if (key % 3 > 0) { - assertThat(mvDoubles.mvOrdering(), equalTo(Block.MvOrdering.DEDUPLICATED_AND_SORTED_ASCENDING)); + assertMap(readers, matchesMap().entry(name + ":row_stride:BlockDocValuesReader." + type, segmentCount)); + } else { + Integer singletons = (Integer) readers.remove(name + ":column_at_a_time:BlockDocValuesReader.Singleton" + type); + if (singletons != null) { + pageCount -= singletons; } + assertMap( + readers, + matchesMap().entry(name + ":column_at_a_time:BlockDocValuesReader." + type, lessThanOrEqualTo(pageCount)) + ); } } - for (Operator op : operators) { - assertThat(((ValuesSourceReaderOperator) op).status().pagesProcessed(), equalTo(input.size())); + + static void id(boolean forcedRowByRow, int pageCount, int segmentCount, Map readers) { + stored("_id", "Id", forcedRowByRow, pageCount, segmentCount, readers); + } + + private static void source(String name, String type, boolean forcedRowByRow, int pageCount, int segmentCount, Map readers) { + Matcher count; + if (forcedRowByRow) { + count = equalTo(segmentCount); + } else { + count = lessThanOrEqualTo(pageCount); + Integer columnAttempts = (Integer) readers.remove(name + ":column_at_a_time:null"); + assertThat(columnAttempts, not(nullValue())); + } + assertMap( + readers, + matchesMap().entry(name + ":row_stride:BlockSourceReader." + type, count) + .entry("stored_fields[requires_source:true, fields:0]", count) + ); + } + + private static void stored(String name, String type, boolean forcedRowByRow, int pageCount, int segmentCount, Map readers) { + Matcher count; + if (forcedRowByRow) { + count = equalTo(segmentCount); + } else { + count = lessThanOrEqualTo(pageCount); + Integer columnAttempts = (Integer) readers.remove(name + ":column_at_a_time:null"); + assertThat(columnAttempts, not(nullValue())); + } + assertMap( + readers, + matchesMap().entry(name + ":row_stride:BlockStoredFieldsReader." + type, count) + .entry("stored_fields[requires_source:false, fields:1]", count) + ); + } + + static void constantBytes(boolean forcedRowByRow, int pageCount, int segmentCount, Map readers) { + if (forcedRowByRow) { + assertMap(readers, matchesMap().entry("constant_bytes:row_stride:constant[[66 6f 6f]]", segmentCount)); + } else { + assertMap( + readers, + matchesMap().entry("constant_bytes:column_at_a_time:constant[[66 6f 6f]]", lessThanOrEqualTo(pageCount)) + ); + } + } + + static void constantNulls(boolean forcedRowByRow, int pageCount, int segmentCount, Map readers) { + if (forcedRowByRow) { + assertMap(readers, matchesMap().entry("null:row_stride:constant_nulls", segmentCount)); + } else { + assertMap(readers, matchesMap().entry("null:column_at_a_time:constant_nulls", lessThanOrEqualTo(pageCount))); + } } - assertDriverContext(driverContext); } - public void testValuesSourceReaderOperatorWithNulls() throws IOException { - MappedFieldType intFt = new NumberFieldMapper.NumberFieldType("i", NumberFieldMapper.NumberType.INTEGER); - MappedFieldType longFt = new NumberFieldMapper.NumberFieldType("j", NumberFieldMapper.NumberType.LONG); - MappedFieldType doubleFt = new NumberFieldMapper.NumberFieldType("d", NumberFieldMapper.NumberType.DOUBLE); + public void testWithNulls() throws IOException { + MappedFieldType intFt = docValuesNumberField("i", NumberFieldMapper.NumberType.INTEGER); + MappedFieldType longFt = docValuesNumberField("j", NumberFieldMapper.NumberType.LONG); + MappedFieldType doubleFt = docValuesNumberField("d", NumberFieldMapper.NumberType.DOUBLE); MappedFieldType kwFt = new KeywordFieldMapper.KeywordFieldType("kw"); NumericDocValuesField intField = new NumericDocValuesField(intFt.name(), 0); @@ -384,4 +1055,85 @@ public void testValuesSourceReaderOperatorWithNulls() throws IOException { } assertDriverContext(driverContext); } + + private NumberFieldMapper.NumberFieldType docValuesNumberField(String name, NumberFieldMapper.NumberType type) { + return new NumberFieldMapper.NumberFieldType(name, type); + } + + private NumberFieldMapper.NumberFieldType sourceNumberField(String name, NumberFieldMapper.NumberType type) { + return new NumberFieldMapper.NumberFieldType( + name, + type, + randomBoolean(), + false, + false, + randomBoolean(), + null, + Map.of(), + null, + false, + null, + randomFrom(IndexMode.values()) + ); + } + + private KeywordFieldMapper.KeywordFieldType storedKeywordField(String name) { + FieldType ft = new FieldType(KeywordFieldMapper.Defaults.FIELD_TYPE); + ft.setDocValuesType(DocValuesType.NONE); + ft.setStored(true); + ft.freeze(); + return new KeywordFieldMapper.KeywordFieldType( + name, + ft, + Lucene.KEYWORD_ANALYZER, + Lucene.KEYWORD_ANALYZER, + Lucene.KEYWORD_ANALYZER, + new KeywordFieldMapper.Builder(name, IndexVersion.current()).docValues(false), + true // TODO randomize - load from stored keyword fields if stored even in synthetic source + ); + } + + private KeywordFieldMapper.KeywordFieldType sourceKeywordField(String name) { + FieldType ft = new FieldType(KeywordFieldMapper.Defaults.FIELD_TYPE); + ft.setDocValuesType(DocValuesType.NONE); + ft.setStored(false); + ft.freeze(); + return new KeywordFieldMapper.KeywordFieldType( + name, + ft, + Lucene.KEYWORD_ANALYZER, + Lucene.KEYWORD_ANALYZER, + Lucene.KEYWORD_ANALYZER, + new KeywordFieldMapper.Builder(name, IndexVersion.current()).docValues(false), + false + ); + } + + private TextFieldMapper.TextFieldType storedTextField(String name) { + return new TextFieldMapper.TextFieldType( + name, + false, + true, + new TextSearchInfo(TextFieldMapper.Defaults.FIELD_TYPE, null, Lucene.STANDARD_ANALYZER, Lucene.STANDARD_ANALYZER), + true, // TODO randomize - if the field is stored we should load from the stored field even if there is source + null, + Map.of(), + false, + false + ); + } + + private TextFieldMapper.TextFieldType textFieldWithDelegate(String name, KeywordFieldMapper.KeywordFieldType delegate) { + return new TextFieldMapper.TextFieldType( + name, + false, + false, + new TextSearchInfo(TextFieldMapper.Defaults.FIELD_TYPE, null, Lucene.STANDARD_ANALYZER, Lucene.STANDARD_ANALYZER), + randomBoolean(), + delegate, + Map.of(), + false, + false + ); + } } diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/AsyncOperatorTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/AsyncOperatorTests.java index b35dc8b7b2e80..00b046abdca24 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/AsyncOperatorTests.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/AsyncOperatorTests.java @@ -185,7 +185,6 @@ protected void doClose() { operator.close(); } - @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/102264") public void testFailure() throws Exception { DriverContext driverContext = driverContext(); final SequenceLongBlockSourceOperator sourceOperator = new SequenceLongBlockSourceOperator( @@ -226,15 +225,17 @@ protected void doClose() { PlainActionFuture future = new PlainActionFuture<>(); Driver driver = new Driver(driverContext, sourceOperator, List.of(asyncOperator), outputOperator, () -> {}); Driver.start(threadPool.getThreadContext(), threadPool.executor(ESQL_TEST_EXECUTOR), driver, between(1, 1000), future); - assertBusy(() -> { - assertTrue(asyncOperator.isFinished()); - assertTrue(future.isDone()); - }); + assertBusy(() -> assertTrue(future.isDone())); if (failed.get()) { ElasticsearchException error = expectThrows(ElasticsearchException.class, future::actionGet); assertThat(error.getMessage(), containsString("simulated")); + error = expectThrows(ElasticsearchException.class, asyncOperator::isFinished); + assertThat(error.getMessage(), containsString("simulated")); + error = expectThrows(ElasticsearchException.class, asyncOperator::getOutput); + assertThat(error.getMessage(), containsString("simulated")); } else { - future.actionGet(); + assertTrue(asyncOperator.isFinished()); + assertNull(asyncOperator.getOutput()); } } diff --git a/x-pack/plugin/esql/qa/server/single-node/build.gradle b/x-pack/plugin/esql/qa/server/single-node/build.gradle index 3131b4176ee25..2d430965efb21 100644 --- a/x-pack/plugin/esql/qa/server/single-node/build.gradle +++ b/x-pack/plugin/esql/qa/server/single-node/build.gradle @@ -9,10 +9,9 @@ restResources { restApi { include '_common', 'bulk', 'get', 'indices', 'esql', 'xpack', 'enrich', 'cluster' } -} - -artifacts { - restXpackTests(new File(projectDir, "src/yamlRestTest/resources/rest-api-spec/test")) + restTests { + includeXpack 'esql' + } } testClusters.configureEach { diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/drop.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/drop.csv-spec index cd3afa25fc0a6..601b4f329f9d7 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/drop.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/drop.csv-spec @@ -54,3 +54,36 @@ c:l|mi:i|s:l 0 |null|null ; + +// see https://github.com/elastic/elasticsearch/issues/102121 +dropGrouping#[skip:-8.11.99, reason:planning bug fixed in v8.12] +row a = 1 | rename a AS foo | stats bar = count(*) by foo | drop foo; + +bar:long +1 +; + +dropGroupingMulti#[skip:-8.11.99] +row a = 1, b = 2 | rename a AS foo, b as bar | stats baz = count(*) by foo, bar | drop foo; + +baz:long | bar:integer +1 | 2 +; + +dropGroupingMulti2#[skip:-8.11.99] +row a = 1, b = 2 | rename a AS foo, b as bar | stats baz = count(*) by foo, bar | drop foo, bar; + +baz:long +1 +; + + +dropGroupingMultirow#[skip:-8.11.99] +from employees | rename gender AS foo | stats bar = count(*) by foo | drop foo | sort bar; + +bar:long +10 +33 +57 +; + diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EnrichIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EnrichIT.java index b855fbd15be12..46aaa6fab16a5 100644 --- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EnrichIT.java +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EnrichIT.java @@ -7,7 +7,6 @@ package org.elasticsearch.xpack.esql.action; -import org.apache.lucene.tests.util.LuceneTestCase; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.TransportAction; import org.elasticsearch.client.internal.Client; @@ -60,7 +59,6 @@ import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasSize; -@LuceneTestCase.AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/pull/102184") public class EnrichIT extends AbstractEsqlIntegTestCase { @Override diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlActionIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlActionIT.java index 06fd9bd469b84..a141db7037263 100644 --- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlActionIT.java +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlActionIT.java @@ -16,6 +16,7 @@ import org.elasticsearch.action.support.WriteRequest; import org.elasticsearch.client.internal.ClusterAdminClient; import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.Index; @@ -1242,8 +1243,15 @@ public void testFilterNestedFields() { } public void testStatsNestFields() { - String node1 = internalCluster().startDataOnlyNode(); - String node2 = internalCluster().startDataOnlyNode(); + final String node1, node2; + if (randomBoolean()) { + internalCluster().ensureAtLeastNumDataNodes(2); + node1 = randomDataNode().getName(); + node2 = randomValueOtherThan(node1, () -> randomDataNode().getName()); + } else { + node1 = randomDataNode().getName(); + node2 = randomDataNode().getName(); + } assertAcked( client().admin() .indices() @@ -1276,8 +1284,15 @@ public void testStatsNestFields() { } public void testStatsMissingFields() { - String node1 = internalCluster().startDataOnlyNode(); - String node2 = internalCluster().startDataOnlyNode(); + final String node1, node2; + if (randomBoolean()) { + internalCluster().ensureAtLeastNumDataNodes(2); + node1 = randomDataNode().getName(); + node2 = randomValueOtherThan(node1, () -> randomDataNode().getName()); + } else { + node1 = randomDataNode().getName(); + node2 = randomDataNode().getName(); + } assertAcked( client().admin() .indices() @@ -1292,7 +1307,6 @@ public void testStatsMissingFields() { .setSettings(Settings.builder().put("index.routing.allocation.require._name", node2)) .setMapping("bar_int", "type=integer", "bar_long", "type=long", "bar_float", "type=float", "bar_double", "type=double") ); - var fields = List.of("foo_int", "foo_long", "foo_float", "foo_double"); var functions = List.of("sum", "count", "avg", "count_distinct"); for (String field : fields) { @@ -1510,4 +1524,8 @@ private void clearPersistentSettings(Setting... settings) { var clearSettingsRequest = new ClusterUpdateSettingsRequest().persistentSettings(clearedSettings.build()); admin().cluster().updateSettings(clearSettingsRequest).actionGet(); } + + private DiscoveryNode randomDataNode() { + return randomFrom(clusterService().state().nodes().getDataNodes().values()); + } } diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlActionTaskIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlActionTaskIT.java index edaf9d91e9771..402ae2722d7ca 100644 --- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlActionTaskIT.java +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlActionTaskIT.java @@ -177,7 +177,10 @@ public void testTaskContents() throws Exception { } if (o.operator().equals("ValuesSourceReaderOperator[field = pause_me]")) { ValuesSourceReaderOperator.Status oStatus = (ValuesSourceReaderOperator.Status) o.status(); - assertMap(oStatus.readersBuilt(), matchesMap().entry("ScriptLongs", greaterThanOrEqualTo(1))); + assertMap( + oStatus.readersBuilt(), + matchesMap().entry("pause_me:column_at_a_time:ScriptLongs", greaterThanOrEqualTo(1)) + ); assertThat(oStatus.pagesProcessed(), greaterThanOrEqualTo(1)); valuesSourceReaders++; continue; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/EnrichLookupService.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/EnrichLookupService.java index 7dd9f01a9d6c9..bad7dd00d6c18 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/EnrichLookupService.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/EnrichLookupService.java @@ -257,21 +257,28 @@ private void doLookup( }; List intermediateOperators = new ArrayList<>(extractFields.size() + 2); final ElementType[] mergingTypes = new ElementType[extractFields.size()]; - // extract-field operators + + // load the fields + List fields = new ArrayList<>(extractFields.size()); for (int i = 0; i < extractFields.size(); i++) { NamedExpression extractField = extractFields.get(i); final ElementType elementType = LocalExecutionPlanner.toElementType(extractField.dataType()); mergingTypes[i] = elementType; - var sources = BlockReaderFactories.factories( + var loaders = BlockReaderFactories.loaders( List.of(searchContext), extractField instanceof Alias a ? ((NamedExpression) a.child()).name() : extractField.name(), EsqlDataTypes.isUnsupported(extractField.dataType()) ); - intermediateOperators.add(new ValuesSourceReaderOperator(blockFactory, sources, 0, extractField.name())); + fields.add(new ValuesSourceReaderOperator.FieldInfo(extractField.name(), loaders)); } + intermediateOperators.add( + new ValuesSourceReaderOperator(blockFactory, fields, List.of(searchContext.searcher().getIndexReader()), 0) + ); + // drop docs block intermediateOperators.add(droppingBlockOperator(extractFields.size() + 2, 0)); boolean singleLeaf = searchContext.searcher().getLeafContexts().size() == 1; + // merging field-values by position final int[] mergingChannels = IntStream.range(0, extractFields.size()).map(i -> i + 1).toArray(); intermediateOperators.add( @@ -335,7 +342,8 @@ private static Operator droppingBlockOperator(int totalBlocks, int droppingPosit private class TransportHandler implements TransportRequestHandler { @Override public void messageReceived(LookupRequest request, TransportChannel channel, Task task) { - ActionListener listener = new ChannelActionListener<>(channel); + request.incRef(); + ActionListener listener = ActionListener.runBefore(new ChannelActionListener<>(channel), request::decRef); doLookup( request.sessionId, (CancellableTask) task, diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizer.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizer.java index bfa9ea449ad46..29b61949b6778 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizer.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizer.java @@ -281,7 +281,10 @@ protected LogicalPlan rule(UnaryPlan plan) { // eliminate lower project but first replace the aliases in the upper one return p.withProjections(combineProjections(project.projections(), p.projections())); } else if (child instanceof Aggregate a) { - return new Aggregate(a.source(), a.child(), a.groupings(), combineProjections(project.projections(), a.aggregates())); + var aggs = a.aggregates(); + var newAggs = combineProjections(project.projections(), aggs); + var newGroups = replacePrunedAliasesUsedInGroupBy(a.groupings(), aggs, newAggs); + return new Aggregate(a.source(), a.child(), newGroups, newAggs); } } @@ -320,6 +323,39 @@ private List combineProjections(List return replaced; } + /** + * Replace grouping alias previously contained in the aggregations that might have been projected away. + */ + private List replacePrunedAliasesUsedInGroupBy( + List groupings, + List oldAggs, + List newAggs + ) { + AttributeMap removedAliases = new AttributeMap<>(); + AttributeSet currentAliases = new AttributeSet(Expressions.asAttributes(newAggs)); + + // record only removed aliases + for (NamedExpression ne : oldAggs) { + if (ne instanceof Alias alias) { + var attr = ne.toAttribute(); + if (currentAliases.contains(attr) == false) { + removedAliases.put(attr, alias.child()); + } + } + } + + if (removedAliases.isEmpty()) { + return groupings; + } + + var newGroupings = new ArrayList(groupings.size()); + for (Expression group : groupings) { + newGroupings.add(group.transformUp(Attribute.class, a -> removedAliases.resolve(a, a))); + } + + return newGroupings; + } + public static Expression trimNonTopLevelAliases(Expression e) { if (e instanceof Alias a) { return new Alias(a.source(), a.name(), a.qualifier(), trimAliases(a.child()), a.id()); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsPhysicalOperationProviders.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsPhysicalOperationProviders.java index f73ab716cb534..1dddee5ed54ea 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsPhysicalOperationProviders.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsPhysicalOperationProviders.java @@ -7,6 +7,7 @@ package org.elasticsearch.xpack.esql.planner; +import org.apache.lucene.index.IndexReader; import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.Query; @@ -19,7 +20,7 @@ import org.elasticsearch.compute.lucene.ValuesSourceReaderOperator; import org.elasticsearch.compute.operator.Operator; import org.elasticsearch.compute.operator.OrdinalsGroupingOperator; -import org.elasticsearch.index.mapper.BlockDocValuesReader; +import org.elasticsearch.index.mapper.BlockLoader; import org.elasticsearch.index.mapper.NestedLookup; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryBuilders; @@ -75,16 +76,17 @@ public final PhysicalOperation fieldExtractPhysicalOperation(FieldExtractExec fi DataType dataType = attr.dataType(); String fieldName = attr.name(); - List factories = BlockReaderFactories.factories( - searchContexts, - fieldName, - EsqlDataTypes.isUnsupported(dataType) - ); + List loaders = BlockReaderFactories.loaders(searchContexts, fieldName, EsqlDataTypes.isUnsupported(dataType)); + List readers = searchContexts.stream().map(s -> s.searcher().getIndexReader()).toList(); int docChannel = previousLayout.get(sourceAttr.id()).channel(); op = op.with( - new ValuesSourceReaderOperator.ValuesSourceReaderOperatorFactory(factories, docChannel, fieldName), + new ValuesSourceReaderOperator.Factory( + List.of(new ValuesSourceReaderOperator.FieldInfo(fieldName, loaders)), + readers, + docChannel + ), layout.build() ); } @@ -173,7 +175,8 @@ public final Operator.OperatorFactory ordinalGroupingOperatorFactory( // The grouping-by values are ready, let's group on them directly. // Costin: why are they ready and not already exposed in the layout? return new OrdinalsGroupingOperator.OrdinalsGroupingOperatorFactory( - BlockReaderFactories.factories(searchContexts, attrSource.name(), EsqlDataTypes.isUnsupported(attrSource.dataType())), + BlockReaderFactories.loaders(searchContexts, attrSource.name(), EsqlDataTypes.isUnsupported(attrSource.dataType())), + searchContexts.stream().map(s -> s.searcher().getIndexReader()).toList(), groupElementType, docChannel, attrSource.name(), diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/LocalExecutionPlanner.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/LocalExecutionPlanner.java index 9a76bc0865865..c749f505b86a7 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/LocalExecutionPlanner.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/LocalExecutionPlanner.java @@ -335,8 +335,10 @@ private static Function alignPageToAttributes(List attrs, var blocks = new Block[mappedPosition.length]; for (int i = 0; i < blocks.length; i++) { blocks[i] = p.getBlock(mappedPosition[i]); + blocks[i].incRef(); } - return p.newPageAndRelease(blocks); + p.releaseBlocks(); + return new Page(blocks); } : Function.identity(); return transformer; @@ -360,11 +362,10 @@ private PhysicalOperation planExchangeSink(ExchangeSinkExec exchangeSink, LocalE // the outputs are going to be similar except for the bool "seen" flags which are added in below List blocks = new ArrayList<>(asList(localExec.supplier().get())); if (blocks.size() > 0) { - Block boolBlock = BooleanBlock.newConstantBlockWith(true, 1); for (int i = 0, s = output.size(); i < s; i++) { var out = output.get(i); if (out.dataType() == DataTypes.BOOLEAN) { - blocks.add(i, boolBlock); + blocks.add(i, BooleanBlock.newConstantBlockWith(true, 1)); } } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java index e825f1f96a8b3..b82bb46ec103e 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java @@ -2454,6 +2454,63 @@ public void testMvExpandFoldable() { var row = as(expand.child(), Row.class); } + /** + * Expected + * Limit[500[INTEGER]] + * \_Aggregate[[a{r}#2],[COUNT([2a][KEYWORD]) AS bar]] + * \_Row[[1[INTEGER] AS a]] + */ + public void testRenameStatsDropGroup() { + LogicalPlan plan = optimizedPlan(""" + row a = 1 + | rename a AS foo + | stats bar = count(*) by foo + | drop foo"""); + + var limit = as(plan, Limit.class); + var agg = as(limit.child(), Aggregate.class); + assertThat(Expressions.names(agg.groupings()), contains("a")); + var row = as(agg.child(), Row.class); + } + + /** + * Expected + * Limit[500[INTEGER]] + * \_Aggregate[[a{r}#2, bar{r}#8],[COUNT([2a][KEYWORD]) AS baz, b{r}#4 AS bar]] + * \_Row[[1[INTEGER] AS a, 2[INTEGER] AS b]] + */ + public void testMultipleRenameStatsDropGroup() { + LogicalPlan plan = optimizedPlan(""" + row a = 1, b = 2 + | rename a AS foo, b as bar + | stats baz = count(*) by foo, bar + | drop foo"""); + + var limit = as(plan, Limit.class); + var agg = as(limit.child(), Aggregate.class); + assertThat(Expressions.names(agg.groupings()), contains("a", "bar")); + var row = as(agg.child(), Row.class); + } + + /** + * Expected + * Limit[500[INTEGER]] + * \_Aggregate[[emp_no{f}#11, bar{r}#4],[MAX(salary{f}#16) AS baz, gender{f}#13 AS bar]] + * \_EsRelation[test][_meta_field{f}#17, emp_no{f}#11, first_name{f}#12, ..] + */ + public void testMultipleRenameStatsDropGroupMultirow() { + LogicalPlan plan = optimizedPlan(""" + from test + | rename emp_no AS foo, gender as bar + | stats baz = max(salary) by foo, bar + | drop foo"""); + + var limit = as(plan, Limit.class); + var agg = as(limit.child(), Aggregate.class); + assertThat(Expressions.names(agg.groupings()), contains("emp_no", "bar")); + var row = as(agg.child(), EsRelation.class); + } + private T aliased(Expression exp, Class clazz) { var alias = as(exp, Alias.class); return as(alias.child(), clazz); diff --git a/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapper.java b/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapper.java index 6c8462c9e4948..ebe25ea1da1d9 100644 --- a/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapper.java +++ b/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapper.java @@ -30,7 +30,6 @@ import org.elasticsearch.index.fielddata.FieldDataContext; import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.fielddata.plain.ConstantIndexFieldData; -import org.elasticsearch.index.mapper.BlockDocValuesReader; import org.elasticsearch.index.mapper.BlockLoader; import org.elasticsearch.index.mapper.ConstantFieldType; import org.elasticsearch.index.mapper.DocumentParserContext; @@ -137,45 +136,10 @@ public String familyTypeName() { @Override public BlockLoader blockLoader(BlockLoaderContext blContext) { - // TODO build a constant block directly if (value == null) { - return BlockDocValuesReader.nulls(); + return BlockLoader.CONSTANT_NULLS; } - BytesRef bytes = new BytesRef(value); - return context -> new BlockDocValuesReader() { - private int docId; - - @Override - public int docID() { - return docId; - } - - @Override - public BlockLoader.BytesRefBuilder builder(BlockLoader.BuilderFactory factory, int expectedCount) { - return factory.bytesRefs(expectedCount); - } - - @Override - public BlockLoader.Block readValues(BlockLoader.BuilderFactory factory, BlockLoader.Docs docs) { - try (BlockLoader.BytesRefBuilder builder = builder(factory, docs.count())) { - for (int i = 0; i < docs.count(); i++) { - builder.appendBytesRef(bytes); - } - return builder.build(); - } - } - - @Override - public void readValuesFromSingleDoc(int docId, BlockLoader.Builder builder) { - this.docId = docId; - ((BlockLoader.BytesRefBuilder) builder).appendBytesRef(bytes); - } - - @Override - public String toString() { - return "ConstantKeyword"; - } - }; + return BlockLoader.constantBytes(new BytesRef(value)); } @Override diff --git a/x-pack/plugin/mapper-constant-keyword/src/test/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapperTests.java b/x-pack/plugin/mapper-constant-keyword/src/test/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapperTests.java index aaa28e28b72c9..87db404a40142 100644 --- a/x-pack/plugin/mapper-constant-keyword/src/test/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapperTests.java +++ b/x-pack/plugin/mapper-constant-keyword/src/test/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapperTests.java @@ -16,8 +16,6 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.core.CheckedFunction; -import org.elasticsearch.index.mapper.BlockDocValuesReader; import org.elasticsearch.index.mapper.BlockLoader; import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.index.mapper.DocumentParsingException; @@ -229,24 +227,7 @@ protected boolean allowsNullValues() { * for newly created indices that haven't received any documents that * contain the field. */ - public void testNullValueBlockLoaderReadValues() throws IOException { - testNullBlockLoader(blockReader -> (TestBlock) blockReader.readValues(TestBlock.FACTORY, TestBlock.docs(0))); - } - - /** - * Test loading blocks when there is no defined value. This is allowed - * for newly created indices that haven't received any documents that - * contain the field. - */ - public void testNullValueBlockLoaderReadValuesFromSingleDoc() throws IOException { - testNullBlockLoader(blockReader -> { - TestBlock block = (TestBlock) blockReader.builder(TestBlock.FACTORY, 1); - blockReader.readValuesFromSingleDoc(0, block); - return block; - }); - } - - private void testNullBlockLoader(CheckedFunction body) throws IOException { + public void testNullValueBlockLoader() throws IOException { MapperService mapper = createMapperService(syntheticSourceMapping(b -> { b.startObject("field"); b.field("type", "constant_keyword"); @@ -274,7 +255,18 @@ public Set sourcePaths(String name) { iw.addDocument(doc); iw.close(); try (DirectoryReader reader = DirectoryReader.open(directory)) { - TestBlock block = body.apply(loader.reader(reader.leaves().get(0))); + TestBlock block = (TestBlock) loader.columnAtATimeReader(reader.leaves().get(0)) + .read(TestBlock.FACTORY, new BlockLoader.Docs() { + @Override + public int count() { + return 1; + } + + @Override + public int get(int i) { + return 0; + } + }); assertThat(block.get(0), nullValue()); } } diff --git a/x-pack/plugin/mapper-counted-keyword/src/main/java/org/elasticsearch/xpack/countedkeyword/CountedTermsAggregationBuilder.java b/x-pack/plugin/mapper-counted-keyword/src/main/java/org/elasticsearch/xpack/countedkeyword/CountedTermsAggregationBuilder.java index 23adacd8f65fa..72e3eb4efacf9 100644 --- a/x-pack/plugin/mapper-counted-keyword/src/main/java/org/elasticsearch/xpack/countedkeyword/CountedTermsAggregationBuilder.java +++ b/x-pack/plugin/mapper-counted-keyword/src/main/java/org/elasticsearch/xpack/countedkeyword/CountedTermsAggregationBuilder.java @@ -30,10 +30,12 @@ import java.util.Map; import java.util.Objects; -class CountedTermsAggregationBuilder extends ValuesSourceAggregationBuilder { +public class CountedTermsAggregationBuilder extends ValuesSourceAggregationBuilder { public static final String NAME = "counted_terms"; - public static final ValuesSourceRegistry.RegistryKey REGISTRY_KEY = - new ValuesSourceRegistry.RegistryKey<>(NAME, CountedTermsAggregatorSupplier.class); + static final ValuesSourceRegistry.RegistryKey REGISTRY_KEY = new ValuesSourceRegistry.RegistryKey<>( + NAME, + CountedTermsAggregatorSupplier.class + ); public static final ParseField REQUIRED_SIZE_FIELD_NAME = new ParseField("size"); @@ -50,7 +52,7 @@ class CountedTermsAggregationBuilder extends ValuesSourceAggregationBuilder oneHotEncoding = new HashMap<>(); diff --git a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/http/PublishableHttpResource.java b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/http/PublishableHttpResource.java index d37f4669484a0..e2d4d173af013 100644 --- a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/http/PublishableHttpResource.java +++ b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/http/PublishableHttpResource.java @@ -27,9 +27,11 @@ import java.util.Collections; import java.util.Map; import java.util.Set; -import java.util.stream.Collectors; +import static java.util.stream.Collectors.joining; +import static org.elasticsearch.client.RestClient.IGNORE_RESPONSE_CODES_PARAM; import static org.elasticsearch.core.Strings.format; +import static org.elasticsearch.rest.RestStatus.NOT_FOUND; /** * {@code PublishableHttpResource} represents an {@link HttpResource} that is a single file or object that can be checked and @@ -254,7 +256,7 @@ protected void checkForResource( // avoid exists and DNE parameters from being an exception by default final Set expectedResponseCodes = Sets.union(exists, doesNotExist); - request.addParameter("ignore", expectedResponseCodes.stream().map(i -> i.toString()).collect(Collectors.joining(","))); + request.addParameter(IGNORE_RESPONSE_CODES_PARAM, expectedResponseCodes.stream().map(Object::toString).collect(joining(","))); client.performRequestAsync(request, new ResponseListener() { @@ -436,9 +438,9 @@ protected void deleteResource( final Request request = new Request("DELETE", resourceBasePath + "/" + resourceName); addDefaultParameters(request); - if (false == defaultParameters.containsKey("ignore")) { + if (false == defaultParameters.containsKey(IGNORE_RESPONSE_CODES_PARAM)) { // avoid 404 being an exception by default - request.addParameter("ignore", Integer.toString(RestStatus.NOT_FOUND.getStatus())); + request.addParameter(IGNORE_RESPONSE_CODES_PARAM, Integer.toString(NOT_FOUND.getStatus())); } client.performRequestAsync(request, new ResponseListener() { diff --git a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/exporter/http/AbstractPublishableHttpResourceTestCase.java b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/exporter/http/AbstractPublishableHttpResourceTestCase.java index 4878289cae8d6..b72891708e780 100644 --- a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/exporter/http/AbstractPublishableHttpResourceTestCase.java +++ b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/exporter/http/AbstractPublishableHttpResourceTestCase.java @@ -30,8 +30,9 @@ import java.util.Map; import java.util.Set; import java.util.function.Predicate; -import java.util.stream.Collectors; +import static java.util.stream.Collectors.joining; +import static org.elasticsearch.client.RestClient.IGNORE_RESPONSE_CODES_PARAM; import static org.elasticsearch.xpack.monitoring.exporter.http.AsyncHttpResourceHelper.mockBooleanActionListener; import static org.elasticsearch.xpack.monitoring.exporter.http.AsyncHttpResourceHelper.mockPublishResultActionListener; import static org.elasticsearch.xpack.monitoring.exporter.http.AsyncHttpResourceHelper.whenPerformRequestAsyncWith; @@ -443,7 +444,7 @@ protected Map getParameters( final Set statusCodes = Sets.union(exists, doesNotExist); final Map parametersWithIgnore = new HashMap<>(parameters); - parametersWithIgnore.putIfAbsent("ignore", statusCodes.stream().map(i -> i.toString()).collect(Collectors.joining(","))); + parametersWithIgnore.putIfAbsent(IGNORE_RESPONSE_CODES_PARAM, statusCodes.stream().map(Object::toString).collect(joining(","))); return parametersWithIgnore; } @@ -451,7 +452,7 @@ protected Map getParameters( protected Map deleteParameters(final Map parameters) { final Map parametersWithIgnore = new HashMap<>(parameters); - parametersWithIgnore.putIfAbsent("ignore", "404"); + parametersWithIgnore.putIfAbsent(IGNORE_RESPONSE_CODES_PARAM, Integer.toString(RestStatus.NOT_FOUND.getStatus())); return parametersWithIgnore; } diff --git a/x-pack/plugin/profiling/build.gradle b/x-pack/plugin/profiling/build.gradle index 30bcb5a8756dc..8275bfe633c91 100644 --- a/x-pack/plugin/profiling/build.gradle +++ b/x-pack/plugin/profiling/build.gradle @@ -17,6 +17,7 @@ esplugin { dependencies { compileOnly project(path: xpackModule('core')) + compileOnly project(path: xpackModule('mapper-counted-keyword')) testImplementation(testArtifact(project(xpackModule('core')))) testImplementation project(path: xpackModule('mapper-unsigned-long')) testImplementation project(path: xpackModule('mapper-version')) diff --git a/x-pack/plugin/profiling/src/internalClusterTest/java/org/elasticsearch/xpack/profiling/GetStackTracesActionIT.java b/x-pack/plugin/profiling/src/internalClusterTest/java/org/elasticsearch/xpack/profiling/GetStackTracesActionIT.java index 1d199c95dc633..82226a4bad0f5 100644 --- a/x-pack/plugin/profiling/src/internalClusterTest/java/org/elasticsearch/xpack/profiling/GetStackTracesActionIT.java +++ b/x-pack/plugin/profiling/src/internalClusterTest/java/org/elasticsearch/xpack/profiling/GetStackTracesActionIT.java @@ -44,10 +44,11 @@ public void testGetStackTracesFromAPMWithMatch() throws Exception { GetStackTracesRequest request = new GetStackTracesRequest(null, query, "apm-test-*", "transaction.profiler_stack_trace_ids"); GetStackTracesResponse response = client().execute(GetStackTracesAction.INSTANCE, request).get(); - assertEquals(39, response.getTotalFrames()); + assertEquals(43, response.getTotalFrames()); assertNotNull(response.getStackTraceEvents()); - assertEquals(1L, (long) response.getStackTraceEvents().get("Ce77w10WeIDow3kd1jowlA")); + assertEquals(3L, (long) response.getStackTraceEvents().get("Ce77w10WeIDow3kd1jowlA")); + assertEquals(2L, (long) response.getStackTraceEvents().get("JvISdnJ47BQ01489cwF9DA")); assertNotNull(response.getStackTraces()); // just do a high-level spot check. Decoding is tested in unit-tests diff --git a/x-pack/plugin/profiling/src/internalClusterTest/java/org/elasticsearch/xpack/profiling/ProfilingTestCase.java b/x-pack/plugin/profiling/src/internalClusterTest/java/org/elasticsearch/xpack/profiling/ProfilingTestCase.java index 19bebd05234e6..9ac616af21e7c 100644 --- a/x-pack/plugin/profiling/src/internalClusterTest/java/org/elasticsearch/xpack/profiling/ProfilingTestCase.java +++ b/x-pack/plugin/profiling/src/internalClusterTest/java/org/elasticsearch/xpack/profiling/ProfilingTestCase.java @@ -24,6 +24,7 @@ import org.elasticsearch.xcontent.XContentType; import org.elasticsearch.xpack.core.XPackSettings; import org.elasticsearch.xpack.core.ilm.LifecycleSettings; +import org.elasticsearch.xpack.countedkeyword.CountedKeywordMapperPlugin; import org.elasticsearch.xpack.ilm.IndexLifecycle; import org.elasticsearch.xpack.unsignedlong.UnsignedLongMapperPlugin; import org.elasticsearch.xpack.versionfield.VersionFieldPlugin; @@ -45,6 +46,7 @@ protected Collection> nodePlugins() { LocalStateProfilingXPackPlugin.class, IndexLifecycle.class, UnsignedLongMapperPlugin.class, + CountedKeywordMapperPlugin.class, VersionFieldPlugin.class, getTestTransportPlugin() ); diff --git a/x-pack/plugin/profiling/src/internalClusterTest/resources/data/apm-test.ndjson b/x-pack/plugin/profiling/src/internalClusterTest/resources/data/apm-test.ndjson index d68c6b5e4f2b1..d147256d6b90f 100644 --- a/x-pack/plugin/profiling/src/internalClusterTest/resources/data/apm-test.ndjson +++ b/x-pack/plugin/profiling/src/internalClusterTest/resources/data/apm-test.ndjson @@ -1,2 +1,2 @@ {"create": {"_index": "apm-test-001"}} -{"@timestamp": "1698624000", "transaction.name": "encodeSha1", "transaction.profiler_stack_trace_ids": "Ce77w10WeIDow3kd1jowlA"} +{"@timestamp": "1698624000", "transaction.name": "encodeSha1", "transaction.profiler_stack_trace_ids": ["Ce77w10WeIDow3kd1jowlA", "JvISdnJ47BQ01489cwF9DA", "JvISdnJ47BQ01489cwF9DA", "Ce77w10WeIDow3kd1jowlA", "Ce77w10WeIDow3kd1jowlA"]} diff --git a/x-pack/plugin/profiling/src/internalClusterTest/resources/indices/apm-test.json b/x-pack/plugin/profiling/src/internalClusterTest/resources/indices/apm-test.json index eba8ed14059a7..e0aeb707ffc76 100644 --- a/x-pack/plugin/profiling/src/internalClusterTest/resources/indices/apm-test.json +++ b/x-pack/plugin/profiling/src/internalClusterTest/resources/indices/apm-test.json @@ -13,7 +13,7 @@ "type": "date" }, "transaction.profiler_stack_trace_ids": { - "type": "keyword" + "type": "counted_keyword" }, "transaction.name": { "type": "keyword" diff --git a/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/TransportGetStackTracesAction.java b/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/TransportGetStackTracesAction.java index 28adb58593eef..a781f91f30bbe 100644 --- a/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/TransportGetStackTracesAction.java +++ b/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/TransportGetStackTracesAction.java @@ -40,6 +40,7 @@ import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xcontent.ObjectPath; +import org.elasticsearch.xpack.countedkeyword.CountedTermsAggregationBuilder; import java.time.Instant; import java.util.ArrayList; @@ -184,8 +185,7 @@ private void searchGenericEvents(GetStackTracesRequest request, ActionListener collectErrorsFromStoreAsMap() { Map indicesAndErrors = new HashMap<>(); for (DataStreamLifecycleService lifecycleService : lifecycleServices) { DataStreamLifecycleErrorStore errorStore = lifecycleService.getErrorStore(); - List allIndices = errorStore.getAllIndices(); + Set allIndices = errorStore.getAllIndices(); for (var index : allIndices) { ErrorEntry error = errorStore.getError(index); if (error != null) { diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/DataStreamLifecycleServiceRuntimeSecurityIT.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/DataStreamLifecycleServiceRuntimeSecurityIT.java index fbb2832461a7c..f5349cac99ed7 100644 --- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/DataStreamLifecycleServiceRuntimeSecurityIT.java +++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/DataStreamLifecycleServiceRuntimeSecurityIT.java @@ -48,6 +48,7 @@ import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.Set; import java.util.concurrent.ExecutionException; import static org.elasticsearch.cluster.metadata.DataStreamTestHelper.backingIndexEqualTo; @@ -168,7 +169,7 @@ private Map collectErrorsFromStoreAsMap() { Map indicesAndErrors = new HashMap<>(); for (DataStreamLifecycleService lifecycleService : lifecycleServices) { DataStreamLifecycleErrorStore errorStore = lifecycleService.getErrorStore(); - List allIndices = errorStore.getAllIndices(); + Set allIndices = errorStore.getAllIndices(); for (var index : allIndices) { ErrorEntry error = errorStore.getError(index); if (error != null) { diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/test/TestSecurityClient.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/test/TestSecurityClient.java index 13c8612487d89..4888c0f4c9721 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/test/TestSecurityClient.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/test/TestSecurityClient.java @@ -25,6 +25,7 @@ import org.elasticsearch.common.xcontent.XContentParserUtils; import org.elasticsearch.core.Nullable; import org.elasticsearch.core.Tuple; +import org.elasticsearch.rest.RestStatus; import org.elasticsearch.xcontent.ObjectPath; import org.elasticsearch.xcontent.ToXContent; import org.elasticsearch.xcontent.XContentBuilder; @@ -51,6 +52,7 @@ import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken; import static org.elasticsearch.test.rest.ESRestTestCase.entityAsMap; +import static org.elasticsearch.test.rest.ESRestTestCase.setIgnoredErrorResponseCodes; public class TestSecurityClient { @@ -395,7 +397,7 @@ public TokenInvalidation invalidateTokens(String requestBody) throws IOException final Request request = new Request(HttpDelete.METHOD_NAME, endpoint); // This API returns 404 (with the same body as a 200 response) if there's nothing to delete. // RestClient will throw an exception on 404, but we don't want that, we want to parse the body and return it - request.addParameter("ignore", "404"); + setIgnoredErrorResponseCodes(request, RestStatus.NOT_FOUND); request.setJsonEntity(requestBody); final Map responseBody = entityAsMap(execute(request)); final List> errors = (List>) responseBody.get("error_details"); diff --git a/x-pack/plugin/snapshot-repo-test-kit/qa/s3/build.gradle b/x-pack/plugin/snapshot-repo-test-kit/qa/s3/build.gradle index 36b13ef8b12a7..513589cfbfa06 100644 --- a/x-pack/plugin/snapshot-repo-test-kit/qa/s3/build.gradle +++ b/x-pack/plugin/snapshot-repo-test-kit/qa/s3/build.gradle @@ -69,6 +69,12 @@ testClusters.matching { it.name == "javaRestTest" }.configureEach { println "Using an external service to test " + project.name } setting 'xpack.security.enabled', 'false' + + // Additional tracing related to investigation into https://github.com/elastic/elasticsearch/issues/102294 + setting 'logger.org.elasticsearch.repositories.s3', 'TRACE' + setting 'logger.org.elasticsearch.repositories.blobstore.testkit', 'TRACE' + setting 'logger.com.amazonaws.request', 'DEBUG' + setting 'logger.org.apache.http.wire', 'DEBUG' } tasks.register("s3ThirdPartyTest") { diff --git a/x-pack/plugin/snapshot-repo-test-kit/qa/s3/src/javaRestTest/java/org/elasticsearch/repositories/blobstore/testkit/S3SnapshotRepoTestKitIT.java b/x-pack/plugin/snapshot-repo-test-kit/qa/s3/src/javaRestTest/java/org/elasticsearch/repositories/blobstore/testkit/S3SnapshotRepoTestKitIT.java index 4b1abfa9dcd62..9e40f7b7aada2 100644 --- a/x-pack/plugin/snapshot-repo-test-kit/qa/s3/src/javaRestTest/java/org/elasticsearch/repositories/blobstore/testkit/S3SnapshotRepoTestKitIT.java +++ b/x-pack/plugin/snapshot-repo-test-kit/qa/s3/src/javaRestTest/java/org/elasticsearch/repositories/blobstore/testkit/S3SnapshotRepoTestKitIT.java @@ -7,7 +7,6 @@ package org.elasticsearch.repositories.blobstore.testkit; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.test.junit.annotations.TestIssueLogging; import static org.hamcrest.Matchers.blankOrNullString; import static org.hamcrest.Matchers.not; @@ -31,10 +30,6 @@ protected Settings repositorySettings() { } @Override - @TestIssueLogging( - issueUrl = "https://github.com/elastic/elasticsearch/issues/102294", - value = "org.elasticsearch.repositories.s3:TRACE,org.elasticsearch.repositories.blobstore.testkit:TRACE" - ) public void testRepositoryAnalysis() throws Exception { super.testRepositoryAnalysis(); } diff --git a/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/100_bug_fix.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/100_bug_fix.yml similarity index 94% rename from x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/100_bug_fix.yml rename to x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/100_bug_fix.yml index d5f5bee46f50a..1876d1a6d3881 100644 --- a/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/100_bug_fix.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/100_bug_fix.yml @@ -1,6 +1,8 @@ --- -"Bug fix https://github.com/elastic/elasticsearch/issues/99472": +"Coalesce and to_ip functions": - skip: + version: " - 8.11.99" + reason: "fixes in 8.12 or later" features: warnings - do: bulk: @@ -54,7 +56,10 @@ - match: { values.1: [ 20, null, "255.255.255.255", "255.255.255.255"] } --- -"Bug fix https://github.com/elastic/elasticsearch/issues/101489": +"unsupported and invalid mapped fields": + - skip: + version: " - 8.11.99" + reason: "fixes in 8.12 or later" - do: indices.create: index: index1 diff --git a/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/10_basic.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/10_basic.yml similarity index 99% rename from x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/10_basic.yml rename to x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/10_basic.yml index a3b2de27bcb5b..e15372bc3088e 100644 --- a/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/10_basic.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/10_basic.yml @@ -1,6 +1,8 @@ --- setup: - skip: + version: " - 8.10.99" + reason: "ESQL is available in 8.11+" features: allowed_warnings_regex - do: indices.create: diff --git a/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/20_aggs.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/20_aggs.yml similarity index 81% rename from x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/20_aggs.yml rename to x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/20_aggs.yml index 1087bd5ce06eb..4019b3a303345 100644 --- a/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/20_aggs.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/20_aggs.yml @@ -1,6 +1,8 @@ --- setup: - skip: + version: " - 8.10.99" + reason: "ESQL is available in 8.11+" features: warnings - do: indices.create: @@ -22,91 +24,93 @@ setup: type: long color: type: keyword + text: + type: text - do: bulk: index: "test" refresh: true body: - { "index": { } } - - { "data": 1, "count": 40, "data_d": 1, "count_d": 40, "time": 1674835275187, "color": "red" } + - { "data": 1, "count": 40, "data_d": 1, "count_d": 40, "time": 1674835275187, "color": "red", "text": "rr red" } - { "index": { } } - - { "data": 2, "count": 42, "data_d": 2, "count_d": 42, "time": 1674835275188, "color": "blue" } + - { "data": 2, "count": 42, "data_d": 2, "count_d": 42, "time": 1674835275188, "color": "blue", "text": "bb blue" } - { "index": { } } - - { "data": 1, "count": 44, "data_d": 1, "count_d": 44, "time": 1674835275189, "color": "green" } + - { "data": 1, "count": 44, "data_d": 1, "count_d": 44, "time": 1674835275189, "color": "green", "text": "gg green" } - { "index": { } } - - { "data": 2, "count": 46, "data_d": 2, "count_d": 46, "time": 1674835275190, "color": "red" } + - { "data": 2, "count": 46, "data_d": 2, "count_d": 46, "time": 1674835275190, "color": "red", "text": "rr red" } - { "index": { } } - - { "data": 1, "count": 40, "data_d": 1, "count_d": 40, "time": 1674835275191, "color": "red" } + - { "data": 1, "count": 40, "data_d": 1, "count_d": 40, "time": 1674835275191, "color": "red", "text": "rr red" } - { "index": { } } - - { "data": 2, "count": 42, "data_d": 2, "count_d": 42, "time": 1674835275192, "color": "blue" } + - { "data": 2, "count": 42, "data_d": 2, "count_d": 42, "time": 1674835275192, "color": "blue", "text": "bb blue" } - { "index": { } } - - { "data": 1, "count": 44, "data_d": 1, "count_d": 44, "time": 1674835275193, "color": "green" } + - { "data": 1, "count": 44, "data_d": 1, "count_d": 44, "time": 1674835275193, "color": "green", "text": "gg green" } - { "index": { } } - - { "data": 2, "count": 46, "data_d": 2, "count_d": 46, "time": 1674835275194, "color": "red" } + - { "data": 2, "count": 46, "data_d": 2, "count_d": 46, "time": 1674835275194, "color": "red", "text": "rr red" } - { "index": { } } - - { "data": 1, "count": 40, "data_d": 1, "count_d": 40, "time": 1674835275195, "color": "red" } + - { "data": 1, "count": 40, "data_d": 1, "count_d": 40, "time": 1674835275195, "color": "red", "text": "rr red" } - { "index": { } } - - { "data": 2, "count": 42, "data_d": 2, "count_d": 42, "time": 1674835275196, "color": "blue" } + - { "data": 2, "count": 42, "data_d": 2, "count_d": 42, "time": 1674835275196, "color": "blue", "text": "bb blue" } - { "index": { } } - - { "data": 1, "count": 44, "data_d": 1, "count_d": 44, "time": 1674835275197, "color": "green" } + - { "data": 1, "count": 44, "data_d": 1, "count_d": 44, "time": 1674835275197, "color": "green", "text": "gg green" } - { "index": { } } - - { "data": 2, "count": 46, "data_d": 2, "count_d": 46, "time": 1674835275198, "color": "red" } + - { "data": 2, "count": 46, "data_d": 2, "count_d": 46, "time": 1674835275198, "color": "red", "text": "rr red" } - { "index": { } } - - { "data": 1, "count": 40, "data_d": 1, "count_d": 40, "time": 1674835275199, "color": "red" } + - { "data": 1, "count": 40, "data_d": 1, "count_d": 40, "time": 1674835275199, "color": "red", "text": "rr red" } - { "index": { } } - - { "data": 2, "count": 42, "data_d": 2, "count_d": 42, "time": 1674835275200, "color": "blue" } + - { "data": 2, "count": 42, "data_d": 2, "count_d": 42, "time": 1674835275200, "color": "blue", "text": "bb blue" } - { "index": { } } - - { "data": 1, "count": 44, "data_d": 1, "count_d": 44, "time": 1674835275201, "color": "green" } + - { "data": 1, "count": 44, "data_d": 1, "count_d": 44, "time": 1674835275201, "color": "green", "text": "gg green" } - { "index": { } } - - { "data": 2, "count": 46, "data_d": 2, "count_d": 46, "time": 1674835275202, "color": "red" } + - { "data": 2, "count": 46, "data_d": 2, "count_d": 46, "time": 1674835275202, "color": "red", "text": "rr red" } - { "index": { } } - - { "data": 1, "count": 40, "data_d": 1, "count_d": 40, "time": 1674835275203, "color": "red" } + - { "data": 1, "count": 40, "data_d": 1, "count_d": 40, "time": 1674835275203, "color": "red", "text": "rr red" } - { "index": { } } - - { "data": 2, "count": 42, "data_d": 2, "count_d": 42, "time": 1674835275204, "color": "blue" } + - { "data": 2, "count": 42, "data_d": 2, "count_d": 42, "time": 1674835275204, "color": "blue", "text": "bb blue" } - { "index": { } } - - { "data": 1, "count": 44, "data_d": 1, "count_d": 44, "time": 1674835275205, "color": "green" } + - { "data": 1, "count": 44, "data_d": 1, "count_d": 44, "time": 1674835275205, "color": "green", "text": "gg green" } - { "index": { } } - - { "data": 2, "count": 46, "data_d": 2, "count_d": 46, "time": 1674835275206, "color": "red" } + - { "data": 2, "count": 46, "data_d": 2, "count_d": 46, "time": 1674835275206, "color": "red", "text": "rr red" } - { "index": { } } - - { "data": 1, "count": 40, "data_d": 1, "count_d": 40, "time": 1674835275207, "color": "red" } + - { "data": 1, "count": 40, "data_d": 1, "count_d": 40, "time": 1674835275207, "color": "red", "text": "rr red" } - { "index": { } } - - { "data": 2, "count": 42, "data_d": 2, "count_d": 42, "time": 1674835275208, "color": "blue" } + - { "data": 2, "count": 42, "data_d": 2, "count_d": 42, "time": 1674835275208, "color": "blue", "text": "bb blue" } - { "index": { } } - - { "data": 1, "count": 44, "data_d": 1, "count_d": 44, "time": 1674835275209, "color": "green" } + - { "data": 1, "count": 44, "data_d": 1, "count_d": 44, "time": 1674835275209, "color": "green", "text": "gg green" } - { "index": { } } - - { "data": 2, "count": 46, "data_d": 2, "count_d": 46, "time": 1674835275210, "color": "red" } + - { "data": 2, "count": 46, "data_d": 2, "count_d": 46, "time": 1674835275210, "color": "red", "text": "rr red" } - { "index": { } } - - { "data": 1, "count": 40, "data_d": 1, "count_d": 40, "time": 1674835275211, "color": "red" } + - { "data": 1, "count": 40, "data_d": 1, "count_d": 40, "time": 1674835275211, "color": "red", "text": "rr red" } - { "index": { } } - - { "data": 2, "count": 42, "data_d": 2, "count_d": 42, "time": 1674835275212, "color": "blue" } + - { "data": 2, "count": 42, "data_d": 2, "count_d": 42, "time": 1674835275212, "color": "blue", "text": "bb blue" } - { "index": { } } - - { "data": 1, "count": 44, "data_d": 1, "count_d": 44, "time": 1674835275213, "color": "green" } + - { "data": 1, "count": 44, "data_d": 1, "count_d": 44, "time": 1674835275213, "color": "green", "text": "gg green" } - { "index": { } } - - { "data": 2, "count": 46, "data_d": 2, "count_d": 46, "time": 1674835275214, "color": "red" } + - { "data": 2, "count": 46, "data_d": 2, "count_d": 46, "time": 1674835275214, "color": "red", "text": "rr red" } - { "index": { } } - - { "data": 1, "count": 40, "data_d": 1, "count_d": 40, "time": 1674835275215, "color": "red" } + - { "data": 1, "count": 40, "data_d": 1, "count_d": 40, "time": 1674835275215, "color": "red", "text": "rr red" } - { "index": { } } - - { "data": 2, "count": 42, "data_d": 2, "count_d": 42, "time": 1674835275216, "color": "blue" } + - { "data": 2, "count": 42, "data_d": 2, "count_d": 42, "time": 1674835275216, "color": "blue", "text": "bb blue" } - { "index": { } } - - { "data": 1, "count": 44, "data_d": 1, "count_d": 44, "time": 1674835275217, "color": "green" } + - { "data": 1, "count": 44, "data_d": 1, "count_d": 44, "time": 1674835275217, "color": "green", "text": "gg green" } - { "index": { } } - - { "data": 2, "count": 46, "data_d": 2, "count_d": 46, "time": 1674835275218, "color": "red" } + - { "data": 2, "count": 46, "data_d": 2, "count_d": 46, "time": 1674835275218, "color": "red", "text": "rr red" } - { "index": { } } - - { "data": 1, "count": 40, "data_d": 1, "count_d": 40, "time": 1674835275219, "color": "red" } + - { "data": 1, "count": 40, "data_d": 1, "count_d": 40, "time": 1674835275219, "color": "red", "text": "rr red" } - { "index": { } } - - { "data": 2, "count": 42, "data_d": 2, "count_d": 42, "time": 1674835275220, "color": "blue" } + - { "data": 2, "count": 42, "data_d": 2, "count_d": 42, "time": 1674835275220, "color": "blue", "text": "bb blue" } - { "index": { } } - - { "data": 1, "count": 44, "data_d": 1, "count_d": 44, "time": 1674835275221, "color": "green" } + - { "data": 1, "count": 44, "data_d": 1, "count_d": 44, "time": 1674835275221, "color": "green", "text": "gg green" } - { "index": { } } - - { "data": 2, "count": 46, "data_d": 2, "count_d": 46, "time": 1674835275222, "color": "red" } + - { "data": 2, "count": 46, "data_d": 2, "count_d": 46, "time": 1674835275222, "color": "red", "text": "rr red" } - { "index": { } } - - { "data": 1, "count": 40, "data_d": 1, "count_d": 40, "time": 1674835275223, "color": "red" } + - { "data": 1, "count": 40, "data_d": 1, "count_d": 40, "time": 1674835275223, "color": "red", "text": "rr red" } - { "index": { } } - - { "data": 2, "count": 42, "data_d": 2, "count_d": 42, "time": 1674835275224, "color": "blue" } + - { "data": 2, "count": 42, "data_d": 2, "count_d": 42, "time": 1674835275224, "color": "blue", "text": "bb blue" } - { "index": { } } - - { "data": 1, "count": 44, "data_d": 1, "count_d": 44, "time": 1674835275225, "color": "green" } + - { "data": 1, "count": 44, "data_d": 1, "count_d": 44, "time": 1674835275225, "color": "green", "text": "gg green" } - { "index": { } } - - { "data": 2, "count": 46, "data_d": 2, "count_d": 46, "time": 1674835275226, "color": "red" } + - { "data": 2, "count": 46, "data_d": 2, "count_d": 46, "time": 1674835275226, "color": "red", "text": "rr red" } --- "Test From": @@ -127,8 +131,10 @@ setup: - match: {columns.3.type: "long"} - match: {columns.4.name: "data_d"} - match: {columns.4.type: "double"} - - match: {columns.5.name: "time"} - - match: {columns.5.type: "long"} + - match: {columns.5.name: "text"} + - match: {columns.5.type: "text"} + - match: {columns.6.name: "time"} + - match: {columns.6.type: "long"} - length: {values: 40} --- @@ -429,11 +435,11 @@ setup: body: query: 'from test | eval nullsum = count_d + null | sort nullsum | limit 1' - - length: {columns: 7} + - length: {columns: 8} - length: {values: 1} - - match: {columns.6.name: "nullsum"} - - match: {columns.6.type: "double"} - - match: {values.0.6: null} + - match: {columns.7.name: "nullsum"} + - match: {columns.7.type: "double"} + - match: {values.0.7: null} --- "Test Eval Row With Null": @@ -501,3 +507,19 @@ setup: - match: {values.0.2: null} - match: {values.0.3: null} +--- +grouping on text: + - do: + warnings: + - "No limit defined, adding default limit of [500]" + esql.query: + body: + query: 'FROM test | STATS med=median(count) BY text | SORT med' + columnar: true + + - match: {columns.0.name: "med"} + - match: {columns.0.type: "double"} + - match: {columns.1.name: "text"} + - match: {columns.1.type: "text"} + - match: {values.0: [42.0, 43.0, 44.0]} + - match: {values.1: ["bb blue", "rr red", "gg green"]} diff --git a/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/30_types.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/30_types.yml similarity index 99% rename from x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/30_types.yml rename to x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/30_types.yml index bf159455d00ca..406ae169872a2 100644 --- a/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/30_types.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/30_types.yml @@ -1,6 +1,8 @@ --- setup: - skip: + version: " - 8.10.99" + reason: "ESQL is available in 8.11+" features: warnings --- diff --git a/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/40_tsdb.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/40_tsdb.yml similarity index 99% rename from x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/40_tsdb.yml rename to x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/40_tsdb.yml index 6a90fc5a7b8f8..1f9dc67dbfbbd 100644 --- a/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/40_tsdb.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/40_tsdb.yml @@ -1,5 +1,7 @@ setup: - skip: + version: " - 8.10.99" + reason: "ESQL is available in 8.11+" features: allowed_warnings_regex - do: indices.create: diff --git a/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/40_unsupported_types.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/40_unsupported_types.yml similarity index 99% rename from x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/40_unsupported_types.yml rename to x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/40_unsupported_types.yml index c06456f7f127d..be5b43433983e 100644 --- a/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/40_unsupported_types.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/40_unsupported_types.yml @@ -1,5 +1,7 @@ setup: - skip: + version: " - 8.10.99" + reason: "ESQL is available in 8.11+" features: allowed_warnings_regex - do: indices.create: diff --git a/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/45_non_tsdb_counter.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/45_non_tsdb_counter.yml similarity index 98% rename from x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/45_non_tsdb_counter.yml rename to x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/45_non_tsdb_counter.yml index beb7200f01230..13a88d0c2f79f 100644 --- a/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/45_non_tsdb_counter.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/45_non_tsdb_counter.yml @@ -1,5 +1,7 @@ setup: - skip: + version: " - 8.10.99" + reason: "ESQL is available in 8.11+" features: allowed_warnings_regex - do: indices.create: diff --git a/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/50_index_patterns.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/50_index_patterns.yml similarity index 95% rename from x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/50_index_patterns.yml rename to x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/50_index_patterns.yml index 5fceeee2f6e57..38023b7791709 100644 --- a/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/50_index_patterns.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/50_index_patterns.yml @@ -1,5 +1,7 @@ setup: - skip: + version: " - 8.10.99" + reason: "ESQL is available in 8.11+" features: allowed_warnings_regex --- @@ -28,9 +30,9 @@ disjoint_mappings: index: test1 refresh: true body: - - { "index": {} } - - { "message1": "foo1"} - - { "index": {} } + - { "index": { } } + - { "message1": "foo1" } + - { "index": { } } - { "message1": "foo2" } - do: @@ -38,9 +40,9 @@ disjoint_mappings: index: test2 refresh: true body: - - { "index": {} } + - { "index": { } } - { "message2": 1 } - - { "index": {} } + - { "index": { } } - { "message2": 2 } - do: @@ -315,9 +317,9 @@ same_name_different_type: index: test1 refresh: true body: - - { "index": {} } - - { "message": "foo1"} - - { "index": {} } + - { "index": { } } + - { "message": "foo1" } + - { "index": { } } - { "message": "foo2" } - do: @@ -325,9 +327,9 @@ same_name_different_type: index: test2 refresh: true body: - - { "index": {} } + - { "index": { } } - { "message": 1 } - - { "index": {} } + - { "index": { } } - { "message": 2 } - do: @@ -367,9 +369,9 @@ same_name_different_type_same_family: index: test1 refresh: true body: - - { "index": {} } - - { "message": "foo1"} - - { "index": {} } + - { "index": { } } + - { "message": "foo1" } + - { "index": { } } - { "message": "foo2" } - do: @@ -377,9 +379,9 @@ same_name_different_type_same_family: index: test2 refresh: true body: - - { "index": {} } + - { "index": { } } - { "message": "foo3" } - - { "index": {} } + - { "index": { } } - { "message": "foo4" } - do: diff --git a/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/60_enrich.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/60_enrich.yml similarity index 95% rename from x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/60_enrich.yml rename to x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/60_enrich.yml index 84d8682508733..1673453824584 100644 --- a/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/60_enrich.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/60_enrich.yml @@ -1,6 +1,8 @@ --- setup: - skip: + version: " - 8.10.99" + reason: "ESQL is available in 8.11+" features: allowed_warnings_regex - do: indices.create: @@ -127,3 +129,8 @@ setup: - match: { values.1: [ "Bob", "nyc", "USA" ] } - match: { values.2: [ "Denise", "sgn", null ] } - match: { values.3: [ "Mario", "rom", "Italy" ] } + + - do: + enrich.delete_policy: + name: cities_policy + - is_true: acknowledged diff --git a/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/60_usage.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/60_usage.yml similarity index 96% rename from x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/60_usage.yml rename to x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/60_usage.yml index d7998651540d8..ad46a3c2d9c3e 100644 --- a/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/60_usage.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/60_usage.yml @@ -1,5 +1,9 @@ --- setup: + - skip: + version: " - 8.10.99" + reason: "ESQL is available in 8.11+" + - do: indices.create: index: test diff --git a/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/61_enrich_ip.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/61_enrich_ip.yml similarity index 94% rename from x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/61_enrich_ip.yml rename to x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/61_enrich_ip.yml index bd89af2fd3f79..0d49f169fc4b2 100644 --- a/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/61_enrich_ip.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/61_enrich_ip.yml @@ -1,6 +1,8 @@ --- setup: - skip: + version: " - 8.10.99" + reason: "ESQL is available in 8.11+" features: allowed_warnings_regex - do: indices.create: @@ -95,3 +97,8 @@ setup: - match: { values.1: [ [ "10.100.0.21", "10.101.0.107" ], [ "Production", "QA" ], [ "OPS","Engineering" ], "sending messages" ] } - match: { values.2: [ "10.101.0.107" , "QA", "Engineering", "network disconnected" ] } - match: { values.3: [ "13.101.0.114" , null, null, "authentication failed" ] } + + - do: + enrich.delete_policy: + name: networks-policy + - is_true: acknowledged diff --git a/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/70_locale.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/70_locale.yml similarity index 96% rename from x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/70_locale.yml rename to x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/70_locale.yml index a77e0569668de..bcae5e7cf24a2 100644 --- a/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/70_locale.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/70_locale.yml @@ -1,6 +1,8 @@ --- setup: - skip: + version: " - 8.10.99" + reason: "ESQL is available in 8.11+" features: allowed_warnings_regex - do: indices.create: diff --git a/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/80_text.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/80_text.yml similarity index 99% rename from x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/80_text.yml rename to x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/80_text.yml index d6d20fa0a0aee..cef7f88506de8 100644 --- a/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/80_text.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/80_text.yml @@ -1,6 +1,8 @@ --- setup: - skip: + version: " - 8.10.99" + reason: "ESQL is available in 8.11+" features: allowed_warnings_regex - do: indices.create: diff --git a/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/90_non_indexed.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/90_non_indexed.yml similarity index 97% rename from x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/90_non_indexed.yml rename to x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/90_non_indexed.yml index 9138a9454c571..c6124e7f75e96 100644 --- a/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/90_non_indexed.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/90_non_indexed.yml @@ -1,5 +1,7 @@ setup: - skip: + version: " - 8.11.99" + reason: "extracting non-indexed fields available in 8.12+" features: allowed_warnings - do: indices.create: diff --git a/x-pack/plugin/stack/src/javaRestTest/java/org/elasticsearch/xpack/stack/EcsDynamicTemplatesIT.java b/x-pack/plugin/stack/src/javaRestTest/java/org/elasticsearch/xpack/stack/EcsDynamicTemplatesIT.java index ea3286e96160c..25cea3b3f6e0a 100644 --- a/x-pack/plugin/stack/src/javaRestTest/java/org/elasticsearch/xpack/stack/EcsDynamicTemplatesIT.java +++ b/x-pack/plugin/stack/src/javaRestTest/java/org/elasticsearch/xpack/stack/EcsDynamicTemplatesIT.java @@ -32,7 +32,6 @@ import java.net.URL; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collections; import java.util.HashMap; import java.util.Iterator; import java.util.List; @@ -77,7 +76,7 @@ private static void prepareEcsDynamicTemplates() throws IOException { "/" + ECS_DYNAMIC_TEMPLATES_FILE, Integer.toString(1), StackTemplateRegistry.TEMPLATE_VERSION_VARIABLE, - Collections.emptyMap() + StackTemplateRegistry.ADDITIONAL_TEMPLATE_VARIABLES ); Map ecsDynamicTemplatesRaw; try ( diff --git a/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackTemplateRegistry.java b/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackTemplateRegistry.java index 36da14680c66a..8dc8238b8230b 100644 --- a/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackTemplateRegistry.java +++ b/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackTemplateRegistry.java @@ -57,7 +57,7 @@ public class StackTemplateRegistry extends IndexTemplateRegistry { private final FeatureService featureService; private volatile boolean stackTemplateEnabled; - private static final Map ADDITIONAL_TEMPLATE_VARIABLES = Map.of("xpack.stack.template.deprecated", "false"); + public static final Map ADDITIONAL_TEMPLATE_VARIABLES = Map.of("xpack.stack.template.deprecated", "false"); // General mappings conventions for any data that ends up in a data stream public static final String DATA_STREAMS_MAPPINGS_COMPONENT_TEMPLATE_NAME = "data-streams@mappings"; diff --git a/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformRestTestCase.java b/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformRestTestCase.java index e6388bb6fea5d..c616c1c238171 100644 --- a/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformRestTestCase.java +++ b/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformRestTestCase.java @@ -19,6 +19,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.common.xcontent.support.XContentMapValues; +import org.elasticsearch.rest.RestStatus; import org.elasticsearch.test.rest.ESRestTestCase; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xpack.core.transform.TransformField; @@ -618,7 +619,7 @@ protected static void deleteTransform(String transformId) throws IOException { protected static void deleteTransform(String transformId, boolean ignoreNotFound, boolean deleteDestIndex) throws IOException { Request request = new Request("DELETE", getTransformEndpoint() + transformId); if (ignoreNotFound) { - request.addParameter("ignore", "404"); + setIgnoredErrorResponseCodes(request, RestStatus.NOT_FOUND); } if (deleteDestIndex) { request.addParameter(TransformField.DELETE_DEST_INDEX.getPreferredName(), Boolean.TRUE.toString()); diff --git a/x-pack/plugin/transform/src/internalClusterTest/java/org/elasticsearch/xpack/transform/checkpoint/TransformCCSCanMatchIT.java b/x-pack/plugin/transform/src/internalClusterTest/java/org/elasticsearch/xpack/transform/checkpoint/TransformCCSCanMatchIT.java new file mode 100644 index 0000000000000..d05acc7a7b368 --- /dev/null +++ b/x-pack/plugin/transform/src/internalClusterTest/java/org/elasticsearch/xpack/transform/checkpoint/TransformCCSCanMatchIT.java @@ -0,0 +1,415 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.transform.checkpoint; + +import org.apache.lucene.document.LongPoint; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.PointValues; +import org.apache.lucene.util.SetOnce; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.action.support.IndicesOptions; +import org.elasticsearch.action.support.master.AcknowledgedResponse; +import org.elasticsearch.client.internal.Client; +import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.CollectionUtils; +import org.elasticsearch.common.util.concurrent.EsExecutors; +import org.elasticsearch.core.TimeValue; +import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.index.engine.EngineConfig; +import org.elasticsearch.index.engine.EngineFactory; +import org.elasticsearch.index.engine.InternalEngine; +import org.elasticsearch.index.engine.InternalEngineFactory; +import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.index.shard.IndexLongFieldRange; +import org.elasticsearch.index.shard.ShardLongFieldRange; +import org.elasticsearch.node.NodeRoleSettings; +import org.elasticsearch.plugins.EnginePlugin; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.search.SearchModule; +import org.elasticsearch.search.aggregations.BaseAggregationBuilder; +import org.elasticsearch.search.builder.SearchSourceBuilder; +import org.elasticsearch.test.AbstractMultiClustersTestCase; +import org.elasticsearch.test.hamcrest.ElasticsearchAssertions; +import org.elasticsearch.xcontent.NamedXContentRegistry; +import org.elasticsearch.xcontent.ParseField; +import org.elasticsearch.xcontent.XContentParser; +import org.elasticsearch.xcontent.json.JsonXContent; +import org.elasticsearch.xpack.core.ClientHelper; +import org.elasticsearch.xpack.core.XPackSettings; +import org.elasticsearch.xpack.core.transform.MockDeprecatedAggregationBuilder; +import org.elasticsearch.xpack.core.transform.MockDeprecatedQueryBuilder; +import org.elasticsearch.xpack.core.transform.TransformNamedXContentProvider; +import org.elasticsearch.xpack.core.transform.action.GetCheckpointAction; +import org.elasticsearch.xpack.core.transform.action.GetTransformStatsAction; +import org.elasticsearch.xpack.core.transform.action.PutTransformAction; +import org.elasticsearch.xpack.core.transform.action.StartTransformAction; +import org.elasticsearch.xpack.core.transform.transforms.DestConfig; +import org.elasticsearch.xpack.core.transform.transforms.QueryConfig; +import org.elasticsearch.xpack.core.transform.transforms.SourceConfig; +import org.elasticsearch.xpack.core.transform.transforms.TransformConfig; +import org.elasticsearch.xpack.core.transform.transforms.TransformStats; +import org.elasticsearch.xpack.core.transform.transforms.latest.LatestConfig; +import org.elasticsearch.xpack.transform.LocalStateTransform; +import org.junit.Before; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.util.Arrays; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; + +import static java.util.Collections.emptyList; +import static org.elasticsearch.xpack.core.ClientHelper.TRANSFORM_ORIGIN; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasSize; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.nullValue; + +public class TransformCCSCanMatchIT extends AbstractMultiClustersTestCase { + + private static final String REMOTE_CLUSTER = "cluster_a"; + private static final TimeValue TIMEOUT = TimeValue.timeValueMinutes(1); + + private NamedXContentRegistry namedXContentRegistry; + private long timestamp; + private int oldLocalNumShards; + private int localOldDocs; + private int oldRemoteNumShards; + private int remoteOldDocs; + private int newLocalNumShards; + private int localNewDocs; + private int newRemoteNumShards; + private int remoteNewDocs; + + @Before + public void setUpNamedXContentRegistryAndIndices() throws Exception { + SearchModule searchModule = new SearchModule(Settings.EMPTY, emptyList()); + + List namedXContents = searchModule.getNamedXContents(); + namedXContents.add( + new NamedXContentRegistry.Entry( + QueryBuilder.class, + new ParseField(MockDeprecatedQueryBuilder.NAME), + (p, c) -> MockDeprecatedQueryBuilder.fromXContent(p) + ) + ); + namedXContents.add( + new NamedXContentRegistry.Entry( + BaseAggregationBuilder.class, + new ParseField(MockDeprecatedAggregationBuilder.NAME), + (p, c) -> MockDeprecatedAggregationBuilder.fromXContent(p) + ) + ); + + namedXContents.addAll(new TransformNamedXContentProvider().getNamedXContentParsers()); + + namedXContentRegistry = new NamedXContentRegistry(namedXContents); + + timestamp = randomLongBetween(10_000_000, 50_000_000); + + oldLocalNumShards = randomIntBetween(1, 5); + localOldDocs = createIndexAndIndexDocs(LOCAL_CLUSTER, "local_old_index", oldLocalNumShards, timestamp - 10_000, true); + oldRemoteNumShards = randomIntBetween(1, 5); + remoteOldDocs = createIndexAndIndexDocs(REMOTE_CLUSTER, "remote_old_index", oldRemoteNumShards, timestamp - 10_000, true); + + newLocalNumShards = randomIntBetween(1, 5); + localNewDocs = createIndexAndIndexDocs(LOCAL_CLUSTER, "local_new_index", newLocalNumShards, timestamp, randomBoolean()); + newRemoteNumShards = randomIntBetween(1, 5); + remoteNewDocs = createIndexAndIndexDocs(REMOTE_CLUSTER, "remote_new_index", newRemoteNumShards, timestamp, randomBoolean()); + } + + private int createIndexAndIndexDocs(String cluster, String index, int numberOfShards, long timestamp, boolean exposeTimestamp) + throws Exception { + Client client = client(cluster); + ElasticsearchAssertions.assertAcked( + client.admin() + .indices() + .prepareCreate(index) + .setSettings( + Settings.builder() + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, numberOfShards) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) + ) + .setMapping("@timestamp", "type=date", "position", "type=long") + ); + int numDocs = between(100, 500); + for (int i = 0; i < numDocs; i++) { + client.prepareIndex(index).setSource("position", i, "@timestamp", timestamp + i).get(); + } + if (exposeTimestamp) { + client.admin().indices().prepareClose(index).get(); + client.admin() + .indices() + .prepareUpdateSettings(index) + .setSettings(Settings.builder().put(IndexMetadata.INDEX_BLOCKS_WRITE_SETTING.getKey(), true).build()) + .get(); + client.admin().indices().prepareOpen(index).get(); + assertBusy(() -> { + IndexLongFieldRange timestampRange = cluster(cluster).clusterService().state().metadata().index(index).getTimestampRange(); + assertTrue(Strings.toString(timestampRange), timestampRange.containsAllShardRanges()); + }); + } else { + client.admin().indices().prepareRefresh(index).get(); + } + return numDocs; + } + + public void testSearchAction_MatchAllQuery() { + testSearchAction(QueryBuilders.matchAllQuery(), true, localOldDocs + localNewDocs + remoteOldDocs + remoteNewDocs, 0); + testSearchAction(QueryBuilders.matchAllQuery(), false, localOldDocs + localNewDocs + remoteOldDocs + remoteNewDocs, 0); + } + + public void testSearchAction_RangeQuery() { + testSearchAction( + QueryBuilders.rangeQuery("@timestamp").from(timestamp), // This query only matches new documents + true, + localNewDocs + remoteNewDocs, + oldLocalNumShards + oldRemoteNumShards + ); + testSearchAction( + QueryBuilders.rangeQuery("@timestamp").from(timestamp), // This query only matches new documents + false, + localNewDocs + remoteNewDocs, + oldLocalNumShards + oldRemoteNumShards + ); + } + + public void testSearchAction_RangeQueryThatMatchesNoShards() { + testSearchAction( + QueryBuilders.rangeQuery("@timestamp").from(100_000_000), // This query matches no documents + true, + 0, + // All but 2 shards are skipped. TBH I don't know why this 2 shards are not skipped + oldLocalNumShards + newLocalNumShards + oldRemoteNumShards + newRemoteNumShards - 2 + ); + testSearchAction( + QueryBuilders.rangeQuery("@timestamp").from(100_000_000), // This query matches no documents + false, + 0, + // All but 1 shards are skipped. TBH I don't know why this 1 shard is not skipped + oldLocalNumShards + newLocalNumShards + oldRemoteNumShards + newRemoteNumShards - 1 + ); + } + + private void testSearchAction(QueryBuilder query, boolean ccsMinimizeRoundtrips, long expectedHitCount, int expectedSkippedShards) { + SearchSourceBuilder source = new SearchSourceBuilder().query(query); + SearchRequest request = new SearchRequest("local_*", "*:remote_*"); + request.source(source).setCcsMinimizeRoundtrips(ccsMinimizeRoundtrips); + SearchResponse response = client().search(request).actionGet(); + ElasticsearchAssertions.assertHitCount(response, expectedHitCount); + int expectedTotalShards = oldLocalNumShards + newLocalNumShards + oldRemoteNumShards + newRemoteNumShards; + assertThat("Response was: " + response, response.getTotalShards(), is(equalTo(expectedTotalShards))); + assertThat("Response was: " + response, response.getSuccessfulShards(), is(equalTo(expectedTotalShards))); + assertThat("Response was: " + response, response.getFailedShards(), is(equalTo(0))); + assertThat("Response was: " + response, response.getSkippedShards(), is(equalTo(expectedSkippedShards))); + } + + public void testGetCheckpointAction_MatchAllQuery() throws InterruptedException { + testGetCheckpointAction( + client(), + null, + new String[] { "local_*" }, + QueryBuilders.matchAllQuery(), + Set.of("local_old_index", "local_new_index") + ); + testGetCheckpointAction( + client().getRemoteClusterClient(REMOTE_CLUSTER, EsExecutors.DIRECT_EXECUTOR_SERVICE), + REMOTE_CLUSTER, + new String[] { "remote_*" }, + QueryBuilders.matchAllQuery(), + Set.of("remote_old_index", "remote_new_index") + ); + } + + public void testGetCheckpointAction_RangeQuery() throws InterruptedException { + testGetCheckpointAction( + client(), + null, + new String[] { "local_*" }, + QueryBuilders.rangeQuery("@timestamp").from(timestamp), + Set.of("local_new_index") + ); + testGetCheckpointAction( + client().getRemoteClusterClient(REMOTE_CLUSTER, EsExecutors.DIRECT_EXECUTOR_SERVICE), + REMOTE_CLUSTER, + new String[] { "remote_*" }, + QueryBuilders.rangeQuery("@timestamp").from(timestamp), + Set.of("remote_new_index") + ); + } + + public void testGetCheckpointAction_RangeQueryThatMatchesNoShards() throws InterruptedException { + testGetCheckpointAction( + client(), + null, + new String[] { "local_*" }, + QueryBuilders.rangeQuery("@timestamp").from(100_000_000), + Set.of() + ); + testGetCheckpointAction( + client().getRemoteClusterClient(REMOTE_CLUSTER, EsExecutors.DIRECT_EXECUTOR_SERVICE), + REMOTE_CLUSTER, + new String[] { "remote_*" }, + QueryBuilders.rangeQuery("@timestamp").from(100_000_000), + Set.of() + ); + } + + private void testGetCheckpointAction(Client client, String cluster, String[] indices, QueryBuilder query, Set expectedIndices) + throws InterruptedException { + final GetCheckpointAction.Request request = new GetCheckpointAction.Request( + indices, + IndicesOptions.LENIENT_EXPAND_OPEN, + query, + cluster, + TIMEOUT + ); + + CountDownLatch latch = new CountDownLatch(1); + SetOnce finalResponse = new SetOnce<>(); + SetOnce finalException = new SetOnce<>(); + ClientHelper.executeAsyncWithOrigin( + client, + TRANSFORM_ORIGIN, + GetCheckpointAction.INSTANCE, + request, + ActionListener.wrap(response -> { + finalResponse.set(response); + latch.countDown(); + }, e -> { + finalException.set(e); + latch.countDown(); + }) + ); + latch.await(10, TimeUnit.SECONDS); + + assertThat(finalException.get(), is(nullValue())); + assertThat("Response was: " + finalResponse.get(), finalResponse.get().getCheckpoints().keySet(), is(equalTo(expectedIndices))); + } + + public void testTransformLifecycle_MatchAllQuery() throws Exception { + testTransformLifecycle(QueryBuilders.matchAllQuery(), localOldDocs + localNewDocs + remoteOldDocs + remoteNewDocs); + } + + public void testTransformLifecycle_RangeQuery() throws Exception { + testTransformLifecycle(QueryBuilders.rangeQuery("@timestamp").from(timestamp), localNewDocs + remoteNewDocs); + } + + public void testTransformLifecycle_RangeQueryThatMatchesNoShards() throws Exception { + testTransformLifecycle(QueryBuilders.rangeQuery("@timestamp").from(100_000_000), 0); + } + + private void testTransformLifecycle(QueryBuilder query, long expectedHitCount) throws Exception { + String transformId = "test-transform-lifecycle"; + { + QueryConfig queryConfig; + try (XContentParser parser = createParser(JsonXContent.jsonXContent, query.toString())) { + queryConfig = QueryConfig.fromXContent(parser, true); + assertNotNull(queryConfig.getQuery()); + } + TransformConfig transformConfig = TransformConfig.builder() + .setId(transformId) + .setSource(new SourceConfig(new String[] { "local_*", "*:remote_*" }, queryConfig, Map.of())) + .setDest(new DestConfig(transformId + "-dest", null, null)) + .setLatestConfig(new LatestConfig(List.of("position"), "@timestamp")) + .build(); + PutTransformAction.Request request = new PutTransformAction.Request(transformConfig, false, TIMEOUT); + AcknowledgedResponse response = client().execute(PutTransformAction.INSTANCE, request).actionGet(); + assertTrue(response.isAcknowledged()); + } + { + StartTransformAction.Request request = new StartTransformAction.Request(transformId, null, TIMEOUT); + StartTransformAction.Response response = client().execute(StartTransformAction.INSTANCE, request).actionGet(); + assertTrue(response.isAcknowledged()); + } + assertBusy(() -> { + GetTransformStatsAction.Request request = new GetTransformStatsAction.Request(transformId, TIMEOUT); + GetTransformStatsAction.Response response = client().execute(GetTransformStatsAction.INSTANCE, request).actionGet(); + assertThat("Stats were: " + response.getTransformsStats(), response.getTransformsStats(), hasSize(1)); + assertThat(response.getTransformsStats().get(0).getState(), is(equalTo(TransformStats.State.STOPPED))); + assertThat(response.getTransformsStats().get(0).getIndexerStats().getNumDocuments(), is(equalTo(expectedHitCount))); + assertThat(response.getTransformsStats().get(0).getIndexerStats().getNumDeletedDocuments(), is(equalTo(0L))); + assertThat(response.getTransformsStats().get(0).getIndexerStats().getSearchFailures(), is(equalTo(0L))); + assertThat(response.getTransformsStats().get(0).getIndexerStats().getIndexFailures(), is(equalTo(0L))); + }); + } + + @Override + protected NamedXContentRegistry xContentRegistry() { + return namedXContentRegistry; + } + + @Override + protected Collection remoteClusterAlias() { + return List.of(REMOTE_CLUSTER); + } + + @Override + protected Collection> nodePlugins(String clusterAlias) { + return CollectionUtils.appendToCopy( + CollectionUtils.appendToCopy(super.nodePlugins(clusterAlias), LocalStateTransform.class), + ExposingTimestampEnginePlugin.class + ); + } + + @Override + protected Settings nodeSettings() { + return Settings.builder() + .put(NodeRoleSettings.NODE_ROLES_SETTING.getKey(), "master, data, ingest, transform, remote_cluster_client") + .put(XPackSettings.SECURITY_ENABLED.getKey(), false) + .build(); + } + + private static class EngineWithExposingTimestamp extends InternalEngine { + EngineWithExposingTimestamp(EngineConfig engineConfig) { + super(engineConfig); + assert IndexMetadata.INDEX_BLOCKS_WRITE_SETTING.get(config().getIndexSettings().getSettings()) : "require read-only index"; + } + + @Override + public ShardLongFieldRange getRawFieldRange(String field) { + try (Searcher searcher = acquireSearcher("test")) { + final DirectoryReader directoryReader = searcher.getDirectoryReader(); + + final byte[] minPackedValue = PointValues.getMinPackedValue(directoryReader, field); + final byte[] maxPackedValue = PointValues.getMaxPackedValue(directoryReader, field); + if (minPackedValue == null || maxPackedValue == null) { + assert minPackedValue == null && maxPackedValue == null + : Arrays.toString(minPackedValue) + "-" + Arrays.toString(maxPackedValue); + return ShardLongFieldRange.EMPTY; + } + + return ShardLongFieldRange.of(LongPoint.decodeDimension(minPackedValue, 0), LongPoint.decodeDimension(maxPackedValue, 0)); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + } + + public static class ExposingTimestampEnginePlugin extends Plugin implements EnginePlugin { + + @Override + public Optional getEngineFactory(IndexSettings indexSettings) { + if (IndexMetadata.INDEX_BLOCKS_WRITE_SETTING.get(indexSettings.getSettings())) { + return Optional.of(EngineWithExposingTimestamp::new); + } else { + return Optional.of(new InternalEngineFactory()); + } + } + } +} diff --git a/x-pack/plugin/transform/src/internalClusterTest/java/org/elasticsearch/xpack/transform/checkpoint/TransformGetCheckpointIT.java b/x-pack/plugin/transform/src/internalClusterTest/java/org/elasticsearch/xpack/transform/checkpoint/TransformGetCheckpointIT.java index bb159856b965d..82a3ea85bfe6a 100644 --- a/x-pack/plugin/transform/src/internalClusterTest/java/org/elasticsearch/xpack/transform/checkpoint/TransformGetCheckpointIT.java +++ b/x-pack/plugin/transform/src/internalClusterTest/java/org/elasticsearch/xpack/transform/checkpoint/TransformGetCheckpointIT.java @@ -15,6 +15,7 @@ import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.common.Strings; import org.elasticsearch.core.TimeValue; +import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.xcontent.XContentType; import org.elasticsearch.xpack.core.transform.action.GetCheckpointAction; import org.elasticsearch.xpack.transform.TransformSingleNodeTestCase; @@ -25,6 +26,7 @@ import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; +import static org.hamcrest.Matchers.anEmptyMap; import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.startsWith; @@ -46,6 +48,8 @@ public void testGetCheckpoint() throws Exception { final GetCheckpointAction.Request request = new GetCheckpointAction.Request( new String[] { indexNamePrefix + "*" }, IndicesOptions.LENIENT_EXPAND_OPEN, + null, + null, TimeValue.timeValueSeconds(5) ); @@ -99,6 +103,40 @@ public void testGetCheckpoint() throws Exception { ); } + public void testGetCheckpointWithQueryThatFiltersOutEverything() throws Exception { + final String indexNamePrefix = "test_index-"; + final int indices = randomIntBetween(1, 5); + final int shards = randomIntBetween(1, 5); + final int docsToCreatePerShard = randomIntBetween(0, 10); + + for (int i = 0; i < indices; ++i) { + indicesAdmin().prepareCreate(indexNamePrefix + i) + .setSettings(indexSettings(shards, 1)) + .setMapping("field", "type=long", "@timestamp", "type=date") + .get(); + for (int j = 0; j < shards; ++j) { + for (int d = 0; d < docsToCreatePerShard; ++d) { + client().prepareIndex(indexNamePrefix + i) + .setSource(Strings.format("{ \"field\":%d, \"@timestamp\": %d }", j, 10_000_000 + d + i + j), XContentType.JSON) + .get(); + } + } + } + indicesAdmin().refresh(new RefreshRequest(indexNamePrefix + "*")); + + final GetCheckpointAction.Request request = new GetCheckpointAction.Request( + new String[] { indexNamePrefix + "*" }, + IndicesOptions.LENIENT_EXPAND_OPEN, + // This query does not match any documents + QueryBuilders.rangeQuery("@timestamp").gte(20_000_000), + null, + TimeValue.timeValueSeconds(5) + ); + + final GetCheckpointAction.Response response = client().execute(GetCheckpointAction.INSTANCE, request).get(); + assertThat("Response was: " + response.getCheckpoints(), response.getCheckpoints(), is(anEmptyMap())); + } + public void testGetCheckpointTimeoutExceeded() throws Exception { final String indexNamePrefix = "test_index-"; final int indices = 100; @@ -111,6 +149,8 @@ public void testGetCheckpointTimeoutExceeded() throws Exception { final GetCheckpointAction.Request request = new GetCheckpointAction.Request( new String[] { indexNamePrefix + "*" }, IndicesOptions.LENIENT_EXPAND_OPEN, + null, + null, TimeValue.ZERO ); diff --git a/x-pack/plugin/transform/src/internalClusterTest/java/org/elasticsearch/xpack/transform/checkpoint/TransformGetCheckpointTests.java b/x-pack/plugin/transform/src/internalClusterTest/java/org/elasticsearch/xpack/transform/checkpoint/TransformGetCheckpointTests.java index 1411576e61d58..300a075c9f1b2 100644 --- a/x-pack/plugin/transform/src/internalClusterTest/java/org/elasticsearch/xpack/transform/checkpoint/TransformGetCheckpointTests.java +++ b/x-pack/plugin/transform/src/internalClusterTest/java/org/elasticsearch/xpack/transform/checkpoint/TransformGetCheckpointTests.java @@ -15,6 +15,7 @@ import org.elasticsearch.action.support.ActionTestUtils; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.action.support.replication.ClusterStateCreationUtils; +import org.elasticsearch.client.internal.Client; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; @@ -71,6 +72,7 @@ public class TransformGetCheckpointTests extends ESSingleNodeTestCase { private IndicesService indicesService; private ThreadPool threadPool; private IndexNameExpressionResolver indexNameExpressionResolver; + private Client client; private MockTransport mockTransport; private Task transformTask; private final String indexNamePattern = "test_index-"; @@ -133,6 +135,8 @@ protected void onSendRequest(long requestId, String action, TransportRequest req .putCompatibilityVersions("node01", TransportVersions.V_8_5_0, Map.of()) .build(); + client = mock(Client.class); + transformTask = new Task( 1L, "persistent", @@ -157,6 +161,8 @@ public void testEmptyCheckpoint() throws InterruptedException { GetCheckpointAction.Request request = new GetCheckpointAction.Request( Strings.EMPTY_ARRAY, IndicesOptions.LENIENT_EXPAND_OPEN, + null, + null, TimeValue.timeValueSeconds(5) ); assertCheckpointAction(request, response -> { @@ -170,6 +176,8 @@ public void testSingleIndexRequest() throws InterruptedException { GetCheckpointAction.Request request = new GetCheckpointAction.Request( new String[] { indexNamePattern + "0" }, IndicesOptions.LENIENT_EXPAND_OPEN, + null, + null, TimeValue.timeValueSeconds(5) ); @@ -189,6 +197,8 @@ public void testMultiIndexRequest() throws InterruptedException { GetCheckpointAction.Request request = new GetCheckpointAction.Request( testIndices, IndicesOptions.LENIENT_EXPAND_OPEN, + null, + null, TimeValue.timeValueSeconds(5) ); assertCheckpointAction(request, response -> { @@ -208,7 +218,7 @@ public void testMultiIndexRequest() throws InterruptedException { class TestTransportGetCheckpointAction extends TransportGetCheckpointAction { TestTransportGetCheckpointAction() { - super(transportService, new ActionFilters(emptySet()), indicesService, clusterService, indexNameExpressionResolver); + super(transportService, new ActionFilters(emptySet()), indicesService, clusterService, indexNameExpressionResolver, client); } @Override diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportGetCheckpointAction.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportGetCheckpointAction.java index 5acc2d4541559..675d71a5d1db9 100644 --- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportGetCheckpointAction.java +++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportGetCheckpointAction.java @@ -14,9 +14,15 @@ import org.elasticsearch.action.NoShardAvailableActionException; import org.elasticsearch.action.OriginalIndices; import org.elasticsearch.action.UnavailableShardsException; +import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.action.search.SearchShardsAction; +import org.elasticsearch.action.search.SearchShardsGroup; +import org.elasticsearch.action.search.SearchShardsRequest; +import org.elasticsearch.action.search.SearchShardsResponse; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.GroupedActionListener; import org.elasticsearch.action.support.HandledTransportAction; +import org.elasticsearch.client.internal.Client; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.node.DiscoveryNode; @@ -31,10 +37,12 @@ import org.elasticsearch.indices.IndicesService; import org.elasticsearch.tasks.CancellableTask; import org.elasticsearch.tasks.Task; +import org.elasticsearch.tasks.TaskId; import org.elasticsearch.transport.ActionNotFoundTransportException; import org.elasticsearch.transport.TransportRequestOptions; import org.elasticsearch.transport.TransportResponseHandler; import org.elasticsearch.transport.TransportService; +import org.elasticsearch.xpack.core.ClientHelper; import org.elasticsearch.xpack.core.transform.action.GetCheckpointAction; import org.elasticsearch.xpack.core.transform.action.GetCheckpointAction.Request; import org.elasticsearch.xpack.core.transform.action.GetCheckpointAction.Response; @@ -57,6 +65,7 @@ public class TransportGetCheckpointAction extends HandledTransportAction listener) { - final ClusterState state = clusterService.state(); - resolveIndicesAndGetCheckpoint(task, request, listener, state); + final ClusterState clusterState = clusterService.state(); + resolveIndicesAndGetCheckpoint(task, request, listener, clusterState); } - protected void resolveIndicesAndGetCheckpoint(Task task, Request request, ActionListener listener, final ClusterState state) { + protected void resolveIndicesAndGetCheckpoint( + Task task, + Request request, + ActionListener listener, + final ClusterState clusterState + ) { + final String nodeId = clusterState.nodes().getLocalNode().getId(); + final TaskId parentTaskId = new TaskId(nodeId, task.getId()); + // note: when security is turned on, the indices are already resolved // TODO: do a quick check and only resolve if necessary?? - String[] concreteIndices = this.indexNameExpressionResolver.concreteIndexNames(state, request); - - Map> nodesAndShards = resolveIndicesToPrimaryShards(state, concreteIndices); - - if (nodesAndShards.size() == 0) { + String[] concreteIndices = this.indexNameExpressionResolver.concreteIndexNames(clusterState, request); + Map> nodesAndShards = resolveIndicesToPrimaryShards(clusterState, concreteIndices); + if (nodesAndShards.isEmpty()) { listener.onResponse(new Response(Collections.emptyMap())); return; } - new AsyncGetCheckpointsFromNodesAction(state, task, nodesAndShards, new OriginalIndices(request), request.getTimeout(), listener) - .start(); + if (request.getQuery() == null) { // If there is no query, then there is no point in filtering + getCheckpointsFromNodes(clusterState, task, nodesAndShards, new OriginalIndices(request), request.getTimeout(), listener); + return; + } + + SearchShardsRequest searchShardsRequest = new SearchShardsRequest( + request.indices(), + SearchRequest.DEFAULT_INDICES_OPTIONS, + request.getQuery(), + null, + null, + false, + request.getCluster() + ); + searchShardsRequest.setParentTask(parentTaskId); + ClientHelper.executeAsyncWithOrigin( + client, + ClientHelper.TRANSFORM_ORIGIN, + SearchShardsAction.INSTANCE, + searchShardsRequest, + ActionListener.wrap(searchShardsResponse -> { + Map> filteredNodesAndShards = filterOutSkippedShards(nodesAndShards, searchShardsResponse); + getCheckpointsFromNodes( + clusterState, + task, + filteredNodesAndShards, + new OriginalIndices(request), + request.getTimeout(), + listener + ); + }, e -> { + // search_shards API failed so we just log the error here and continue just like there was no query + logger.atWarn().withThrowable(e).log("search_shards API failed for cluster [{}]", request.getCluster()); + logger.atTrace() + .withThrowable(e) + .log("search_shards API failed for cluster [{}], request was [{}]", request.getCluster(), searchShardsRequest); + getCheckpointsFromNodes(clusterState, task, nodesAndShards, new OriginalIndices(request), request.getTimeout(), listener); + }) + ); } - private static Map> resolveIndicesToPrimaryShards(ClusterState state, String[] concreteIndices) { + private static Map> resolveIndicesToPrimaryShards(ClusterState clusterState, String[] concreteIndices) { if (concreteIndices.length == 0) { return Collections.emptyMap(); } - final DiscoveryNodes nodes = state.nodes(); + final DiscoveryNodes nodes = clusterState.nodes(); Map> nodesAndShards = new HashMap<>(); - ShardsIterator shardsIt = state.routingTable().allShards(concreteIndices); + ShardsIterator shardsIt = clusterState.routingTable().allShards(concreteIndices); for (ShardRouting shard : shardsIt) { // only take primary shards, which should be exactly 1, this isn't strictly necessary // and we should consider taking any shard copy, but then we need another way to de-dup @@ -112,7 +166,7 @@ private static Map> resolveIndicesToPrimaryShards(ClusterSt } if (shard.assignedToNode() && nodes.get(shard.currentNodeId()) != null) { // special case: The minimum TransportVersion in the cluster is on an old version - if (state.getMinTransportVersion().before(TransportVersions.V_8_2_0)) { + if (clusterState.getMinTransportVersion().before(TransportVersions.V_8_2_0)) { throw new ActionNotFoundTransportException(GetCheckpointNodeAction.NAME); } @@ -125,111 +179,128 @@ private static Map> resolveIndicesToPrimaryShards(ClusterSt return nodesAndShards; } - protected class AsyncGetCheckpointsFromNodesAction { - private final Task task; - private final ActionListener listener; - private final Map> nodesAndShards; - private final OriginalIndices originalIndices; - private final TimeValue timeout; - private final DiscoveryNodes nodes; - private final String localNodeId; - - protected AsyncGetCheckpointsFromNodesAction( - ClusterState clusterState, - Task task, - Map> nodesAndShards, - OriginalIndices originalIndices, - TimeValue timeout, - ActionListener listener - ) { - this.task = task; - this.listener = listener; - this.nodesAndShards = nodesAndShards; - this.originalIndices = originalIndices; - this.timeout = timeout; - this.nodes = clusterState.nodes(); - this.localNodeId = clusterService.localNode().getId(); + static Map> filterOutSkippedShards( + Map> nodesAndShards, + SearchShardsResponse searchShardsResponse + ) { + Map> filteredNodesAndShards = new HashMap<>(nodesAndShards.size()); + // Create a deep copy of the given nodes and shards map. + for (Map.Entry> nodeAndShardsEntry : nodesAndShards.entrySet()) { + String node = nodeAndShardsEntry.getKey(); + Set shards = nodeAndShardsEntry.getValue(); + filteredNodesAndShards.put(node, new HashSet<>(shards)); } - - public void start() { - GroupedActionListener groupedListener = new GroupedActionListener<>( - nodesAndShards.size(), - ActionListener.wrap(responses -> listener.onResponse(mergeNodeResponses(responses)), listener::onFailure) - ); - - for (Entry> oneNodeAndItsShards : nodesAndShards.entrySet()) { - if (task instanceof CancellableTask) { - // There is no point continuing this work if the task has been cancelled. - if (((CancellableTask) task).notifyIfCancelled(listener)) { - return; + // Remove (node, shard) pairs for all the skipped shards. + for (SearchShardsGroup shardGroup : searchShardsResponse.getGroups()) { + if (shardGroup.skipped()) { + for (String allocatedNode : shardGroup.allocatedNodes()) { + Set shards = filteredNodesAndShards.get(allocatedNode); + if (shards != null) { + shards.remove(shardGroup.shardId()); + if (shards.isEmpty()) { + // Remove node if no shards were left. + filteredNodesAndShards.remove(allocatedNode); + } } } - if (localNodeId.equals(oneNodeAndItsShards.getKey())) { - TransportGetCheckpointNodeAction.getGlobalCheckpoints( - indicesService, - task, - oneNodeAndItsShards.getValue(), - timeout, - Clock.systemUTC(), - groupedListener - ); - continue; - } + } + } + return filteredNodesAndShards; + } - GetCheckpointNodeAction.Request nodeCheckpointsRequest = new GetCheckpointNodeAction.Request( - oneNodeAndItsShards.getValue(), - originalIndices, - timeout - ); - DiscoveryNode node = nodes.get(oneNodeAndItsShards.getKey()); - - // paranoia: this should not be possible using the same cluster state - if (node == null) { - listener.onFailure( - new UnavailableShardsException( - oneNodeAndItsShards.getValue().iterator().next(), - "Node not found for [{}] shards", - oneNodeAndItsShards.getValue().size() - ) - ); + private void getCheckpointsFromNodes( + ClusterState clusterState, + Task task, + Map> nodesAndShards, + OriginalIndices originalIndices, + TimeValue timeout, + ActionListener listener + ) { + if (nodesAndShards.isEmpty()) { + listener.onResponse(new Response(Map.of())); + return; + } + + final String localNodeId = clusterService.localNode().getId(); + + GroupedActionListener groupedListener = new GroupedActionListener<>( + nodesAndShards.size(), + ActionListener.wrap(responses -> listener.onResponse(mergeNodeResponses(responses)), listener::onFailure) + ); + + for (Entry> oneNodeAndItsShards : nodesAndShards.entrySet()) { + if (task instanceof CancellableTask) { + // There is no point continuing this work if the task has been cancelled. + if (((CancellableTask) task).notifyIfCancelled(listener)) { return; } - - logger.trace("get checkpoints from node {}", node); - transportService.sendChildRequest( - node, - GetCheckpointNodeAction.NAME, - nodeCheckpointsRequest, + } + if (localNodeId.equals(oneNodeAndItsShards.getKey())) { + TransportGetCheckpointNodeAction.getGlobalCheckpoints( + indicesService, task, - TransportRequestOptions.EMPTY, - new ActionListenerResponseHandler<>( - groupedListener, - GetCheckpointNodeAction.Response::new, - TransportResponseHandler.TRANSPORT_WORKER + oneNodeAndItsShards.getValue(), + timeout, + Clock.systemUTC(), + groupedListener + ); + continue; + } + + DiscoveryNodes nodes = clusterState.nodes(); + DiscoveryNode node = nodes.get(oneNodeAndItsShards.getKey()); + + // paranoia: this should not be possible using the same cluster state + if (node == null) { + listener.onFailure( + new UnavailableShardsException( + oneNodeAndItsShards.getValue().iterator().next(), + "Node not found for [{}] shards", + oneNodeAndItsShards.getValue().size() ) ); + return; } + + logger.trace("get checkpoints from node {}", node); + GetCheckpointNodeAction.Request nodeCheckpointsRequest = new GetCheckpointNodeAction.Request( + oneNodeAndItsShards.getValue(), + originalIndices, + timeout + ); + transportService.sendChildRequest( + node, + GetCheckpointNodeAction.NAME, + nodeCheckpointsRequest, + task, + TransportRequestOptions.EMPTY, + new ActionListenerResponseHandler<>( + groupedListener, + GetCheckpointNodeAction.Response::new, + TransportResponseHandler.TRANSPORT_WORKER + ) + ); } + } - private static Response mergeNodeResponses(Collection responses) { - // the final list should be ordered by key - Map checkpointsByIndexReduced = new TreeMap<>(); - - // merge the node responses - for (GetCheckpointNodeAction.Response response : responses) { - response.getCheckpoints().forEach((index, checkpoint) -> { - if (checkpointsByIndexReduced.containsKey(index)) { - long[] shardCheckpoints = checkpointsByIndexReduced.get(index); - for (int i = 0; i < checkpoint.length; ++i) { - shardCheckpoints[i] = Math.max(shardCheckpoints[i], checkpoint[i]); - } - } else { - checkpointsByIndexReduced.put(index, checkpoint); - } - }); - } + private static Response mergeNodeResponses(Collection responses) { + // the final list should be ordered by key + Map checkpointsByIndexReduced = new TreeMap<>(); - return new Response(checkpointsByIndexReduced); + // merge the node responses + for (GetCheckpointNodeAction.Response response : responses) { + response.getCheckpoints().forEach((index, checkpoint) -> { + if (checkpointsByIndexReduced.containsKey(index)) { + long[] shardCheckpoints = checkpointsByIndexReduced.get(index); + for (int i = 0; i < checkpoint.length; ++i) { + shardCheckpoints[i] = Math.max(shardCheckpoints[i], checkpoint[i]); + } + } else { + checkpointsByIndexReduced.put(index, checkpoint); + } + }); } + + return new Response(checkpointsByIndexReduced); } } diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportGetTransformStatsAction.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportGetTransformStatsAction.java index 13abc427460be..f7e60b13b50a6 100644 --- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportGetTransformStatsAction.java +++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportGetTransformStatsAction.java @@ -121,14 +121,15 @@ protected void taskOperation( TransformTask transformTask, ActionListener listener ) { - // Little extra insurance, make sure we only return transforms that aren't cancelled ClusterState clusterState = clusterService.state(); String nodeId = clusterState.nodes().getLocalNode().getId(); final TaskId parentTaskId = new TaskId(nodeId, actionTask.getId()); + // If the _stats request is cancelled there is no point in continuing this work on the task level if (actionTask.notifyIfCancelled(listener)) { return; } + // Little extra insurance, make sure we only return transforms that aren't cancelled if (transformTask.isCancelled()) { listener.onResponse(new Response(Collections.emptyList())); return; diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/checkpoint/DefaultCheckpointProvider.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/checkpoint/DefaultCheckpointProvider.java index aa1332b95fe84..b9b7d9d8477cb 100644 --- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/checkpoint/DefaultCheckpointProvider.java +++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/checkpoint/DefaultCheckpointProvider.java @@ -23,6 +23,7 @@ import org.elasticsearch.common.util.set.Sets; import org.elasticsearch.core.Strings; import org.elasticsearch.core.TimeValue; +import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.transport.ActionNotFoundTransportException; import org.elasticsearch.transport.RemoteClusterService; import org.elasticsearch.xpack.core.ClientHelper; @@ -134,14 +135,16 @@ protected void getIndexCheckpoints(TimeValue timeout, ActionListener> remoteIndex : resolvedIndexes.getRemoteIndicesPerClusterAlias().entrySet()) { + String cluster = remoteIndex.getKey(); ParentTaskAssigningClient remoteClient = new ParentTaskAssigningClient( - client.getRemoteClusterClient(remoteIndex.getKey(), EsExecutors.DIRECT_EXECUTOR_SERVICE), + client.getRemoteClusterClient(cluster, EsExecutors.DIRECT_EXECUTOR_SERVICE), client.getParentTask() ); getCheckpointsFromOneCluster( @@ -149,7 +152,8 @@ protected void getIndexCheckpoints(TimeValue timeout, ActionListener headers, String[] indices, + QueryBuilder query, String cluster, ActionListener> listener ) { if (fallbackToBWC.contains(cluster)) { getCheckpointsFromOneClusterBWC(client, timeout, headers, indices, cluster, listener); } else { - getCheckpointsFromOneClusterV2(client, timeout, headers, indices, cluster, ActionListener.wrap(response -> { + getCheckpointsFromOneClusterV2(client, timeout, headers, indices, query, cluster, ActionListener.wrap(response -> { logger.debug( "[{}] Successfully retrieved checkpoints from cluster [{}] using transform checkpoint API", transformConfig.getId(), @@ -200,12 +205,15 @@ private static void getCheckpointsFromOneClusterV2( TimeValue timeout, Map headers, String[] indices, + QueryBuilder query, String cluster, ActionListener> listener ) { GetCheckpointAction.Request getCheckpointRequest = new GetCheckpointAction.Request( indices, IndicesOptions.LENIENT_EXPAND_OPEN, + query, + cluster, timeout ); ActionListener checkpointListener; @@ -239,7 +247,6 @@ private static void getCheckpointsFromOneClusterV2( getCheckpointRequest, checkpointListener ); - } /** diff --git a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/action/TransportGetCheckpointActionTests.java b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/action/TransportGetCheckpointActionTests.java new file mode 100644 index 0000000000000..0d2d9619aca68 --- /dev/null +++ b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/action/TransportGetCheckpointActionTests.java @@ -0,0 +1,134 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.transform.action; + +import org.elasticsearch.action.search.SearchShardsGroup; +import org.elasticsearch.action.search.SearchShardsResponse; +import org.elasticsearch.index.Index; +import org.elasticsearch.index.shard.ShardId; +import org.elasticsearch.test.ESTestCase; + +import java.util.List; +import java.util.Map; +import java.util.Set; + +import static org.hamcrest.Matchers.anEmptyMap; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; + +public class TransportGetCheckpointActionTests extends ESTestCase { + + private static final String NODE_0 = "node-0"; + private static final String NODE_1 = "node-1"; + private static final String NODE_2 = "node-2"; + private static final Index INDEX_A = new Index("my-index-A", "A"); + private static final Index INDEX_B = new Index("my-index-B", "B"); + private static final Index INDEX_C = new Index("my-index-C", "C"); + private static final ShardId SHARD_A_0 = new ShardId(INDEX_A, 0); + private static final ShardId SHARD_A_1 = new ShardId(INDEX_A, 1); + private static final ShardId SHARD_B_0 = new ShardId(INDEX_B, 0); + private static final ShardId SHARD_B_1 = new ShardId(INDEX_B, 1); + + private static final Map> NODES_AND_SHARDS = Map.of( + NODE_0, + Set.of(SHARD_A_0, SHARD_A_1, SHARD_B_0, SHARD_B_1), + NODE_1, + Set.of(SHARD_A_0, SHARD_A_1, SHARD_B_0, SHARD_B_1), + NODE_2, + Set.of(SHARD_A_0, SHARD_A_1, SHARD_B_0, SHARD_B_1) + ); + + public void testFilterOutSkippedShards_EmptyNodesAndShards() { + SearchShardsResponse searchShardsResponse = new SearchShardsResponse( + Set.of( + new SearchShardsGroup(SHARD_A_0, List.of(NODE_0, NODE_1), true), + new SearchShardsGroup(SHARD_B_0, List.of(NODE_1, NODE_2), false), + new SearchShardsGroup(SHARD_B_1, List.of(NODE_0, NODE_2), true) + ), + Set.of(), + Map.of() + ); + Map> filteredNodesAndShards = TransportGetCheckpointAction.filterOutSkippedShards( + Map.of(), + searchShardsResponse + ); + assertThat(filteredNodesAndShards, is(anEmptyMap())); + } + + public void testFilterOutSkippedShards_EmptySearchShardsResponse() { + SearchShardsResponse searchShardsResponse = new SearchShardsResponse(Set.of(), Set.of(), Map.of()); + Map> filteredNodesAndShards = TransportGetCheckpointAction.filterOutSkippedShards( + NODES_AND_SHARDS, + searchShardsResponse + ); + assertThat(filteredNodesAndShards, is(equalTo(NODES_AND_SHARDS))); + } + + public void testFilterOutSkippedShards_SomeNodesEmptyAfterFiltering() { + SearchShardsResponse searchShardsResponse = new SearchShardsResponse( + Set.of( + new SearchShardsGroup(SHARD_A_0, List.of(NODE_0, NODE_2), true), + new SearchShardsGroup(SHARD_A_1, List.of(NODE_0, NODE_2), true), + new SearchShardsGroup(SHARD_B_0, List.of(NODE_0, NODE_2), true), + new SearchShardsGroup(SHARD_B_1, List.of(NODE_0, NODE_2), true) + ), + Set.of(), + Map.of() + ); + Map> filteredNodesAndShards = TransportGetCheckpointAction.filterOutSkippedShards( + NODES_AND_SHARDS, + searchShardsResponse + ); + Map> expectedFilteredNodesAndShards = Map.of(NODE_1, Set.of(SHARD_A_0, SHARD_A_1, SHARD_B_0, SHARD_B_1)); + assertThat(filteredNodesAndShards, is(equalTo(expectedFilteredNodesAndShards))); + } + + public void testFilterOutSkippedShards_AllNodesEmptyAfterFiltering() { + SearchShardsResponse searchShardsResponse = new SearchShardsResponse( + Set.of( + new SearchShardsGroup(SHARD_A_0, List.of(NODE_0, NODE_1, NODE_2), true), + new SearchShardsGroup(SHARD_A_1, List.of(NODE_0, NODE_1, NODE_2), true), + new SearchShardsGroup(SHARD_B_0, List.of(NODE_0, NODE_1, NODE_2), true), + new SearchShardsGroup(SHARD_B_1, List.of(NODE_0, NODE_1, NODE_2), true) + ), + Set.of(), + Map.of() + ); + Map> filteredNodesAndShards = TransportGetCheckpointAction.filterOutSkippedShards( + NODES_AND_SHARDS, + searchShardsResponse + ); + assertThat(filteredNodesAndShards, is(equalTo(Map.of()))); + } + + public void testFilterOutSkippedShards() { + SearchShardsResponse searchShardsResponse = new SearchShardsResponse( + Set.of( + new SearchShardsGroup(SHARD_A_0, List.of(NODE_0, NODE_1), true), + new SearchShardsGroup(SHARD_B_0, List.of(NODE_1, NODE_2), false), + new SearchShardsGroup(SHARD_B_1, List.of(NODE_0, NODE_2), true), + new SearchShardsGroup(new ShardId(INDEX_C, 0), List.of(NODE_0, NODE_1, NODE_2), true) + ), + Set.of(), + Map.of() + ); + Map> filteredNodesAndShards = TransportGetCheckpointAction.filterOutSkippedShards( + NODES_AND_SHARDS, + searchShardsResponse + ); + Map> expectedFilteredNodesAndShards = Map.of( + NODE_0, + Set.of(SHARD_A_1, SHARD_B_0), + NODE_1, + Set.of(SHARD_A_1, SHARD_B_0, SHARD_B_1), + NODE_2, + Set.of(SHARD_A_0, SHARD_A_1, SHARD_B_0) + ); + assertThat(filteredNodesAndShards, is(equalTo(expectedFilteredNodesAndShards))); + } +} diff --git a/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java b/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java index 480704b89ca60..1954e291b1a7f 100644 --- a/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java +++ b/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java @@ -855,9 +855,7 @@ public Query termsQuery(Collection values, SearchExecutionContext context) { @Override public BlockLoader blockLoader(BlockLoaderContext blContext) { if (hasDocValues()) { - // TODO it'd almost certainly be faster to drop directly to doc values like we do with keyword but this'll do for now - IndexFieldData fd = new StringBinaryIndexFieldData(name(), CoreValuesSourceType.KEYWORD, null); - return BlockDocValuesReader.bytesRefsFromDocValues(context -> fd.load(context).getBytesValues()); + return new BlockDocValuesReader.BytesRefsFromBinaryBlockLoader(name()); } return null; } diff --git a/x-pack/qa/rolling-upgrade-basic/src/test/java/org/elasticsearch/upgrades/BasicLicenseUpgradeIT.java b/x-pack/qa/rolling-upgrade-basic/src/test/java/org/elasticsearch/upgrades/BasicLicenseUpgradeIT.java index 75fcc5cf6e7ad..da8a4c806a0f5 100644 --- a/x-pack/qa/rolling-upgrade-basic/src/test/java/org/elasticsearch/upgrades/BasicLicenseUpgradeIT.java +++ b/x-pack/qa/rolling-upgrade-basic/src/test/java/org/elasticsearch/upgrades/BasicLicenseUpgradeIT.java @@ -8,6 +8,7 @@ import org.elasticsearch.client.Request; import org.elasticsearch.client.Response; +import org.elasticsearch.rest.RestStatus; import java.util.Map; @@ -28,7 +29,7 @@ private void checkBasicLicense() throws Exception { final Request request = new Request("GET", "/_license"); // This avoids throwing a ResponseException when the license is not ready yet // allowing to retry the check using assertBusy - request.addParameter("ignore", "404"); + setIgnoredErrorResponseCodes(request, RestStatus.NOT_FOUND); Response licenseResponse = client().performRequest(request); assertOK(licenseResponse); Map licenseResponseMap = entityAsMap(licenseResponse); @@ -42,7 +43,7 @@ private void checkNonExpiringBasicLicense() throws Exception { final Request request = new Request("GET", "/_license"); // This avoids throwing a ResponseException when the license is not ready yet // allowing to retry the check using assertBusy - request.addParameter("ignore", "404"); + setIgnoredErrorResponseCodes(request, RestStatus.NOT_FOUND); Response licenseResponse = client().performRequest(request); assertOK(licenseResponse); Map licenseResponseMap = entityAsMap(licenseResponse); diff --git a/x-pack/qa/rolling-upgrade-multi-cluster/src/test/java/org/elasticsearch/upgrades/CcrRollingUpgradeIT.java b/x-pack/qa/rolling-upgrade-multi-cluster/src/test/java/org/elasticsearch/upgrades/CcrRollingUpgradeIT.java index b1e1888aba75d..0b7ab1fe5980d 100644 --- a/x-pack/qa/rolling-upgrade-multi-cluster/src/test/java/org/elasticsearch/upgrades/CcrRollingUpgradeIT.java +++ b/x-pack/qa/rolling-upgrade-multi-cluster/src/test/java/org/elasticsearch/upgrades/CcrRollingUpgradeIT.java @@ -88,7 +88,6 @@ public void testUniDirectionalIndexFollowing() throws Exception { } } - @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/102000") public void testAutoFollowing() throws Exception { String leaderIndex1 = "logs-20200101"; String leaderIndex2 = "logs-20200102"; @@ -372,7 +371,8 @@ private static void assertTotalHitCount(final String index, final int expectedTo private static void verifyTotalHitCount(final String index, final int expectedTotalHits, final RestClient client) throws IOException { final Request request = new Request("GET", "/" + index + "/_search"); request.addParameter(TOTAL_HITS_AS_INT_PARAM, "true"); - Map response = toMap(client.performRequest(request)); + setIgnoredErrorResponseCodes(request, RestStatus.NOT_FOUND); // trip the assertOK (i.e. retry an assertBusy) rather than throwing + Map response = toMap(assertOK(client.performRequest(request))); final int totalHits = (int) XContentMapValues.extractValue("hits.total", response); assertThat(totalHits, equalTo(expectedTotalHits)); }