From adce7657910439683a57335754a0bf6985430e64 Mon Sep 17 00:00:00 2001 From: David Turner Date: Mon, 20 Nov 2023 06:32:13 +0000 Subject: [PATCH 01/28] Add @UpdateForV9 to Engine#SYNC_COMMIT_ID (#102343) Turns a TODO comment into a proper reminder to do something about this in v9. --- .../src/main/java/org/elasticsearch/index/engine/Engine.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/index/engine/Engine.java b/server/src/main/java/org/elasticsearch/index/engine/Engine.java index ed7fab325408e..43437529cd301 100644 --- a/server/src/main/java/org/elasticsearch/index/engine/Engine.java +++ b/server/src/main/java/org/elasticsearch/index/engine/Engine.java @@ -48,6 +48,7 @@ import org.elasticsearch.core.Releasable; import org.elasticsearch.core.Releasables; import org.elasticsearch.core.TimeValue; +import org.elasticsearch.core.UpdateForV9; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.VersionType; import org.elasticsearch.index.mapper.DocumentParser; @@ -100,7 +101,8 @@ public abstract class Engine implements Closeable { - public static final String SYNC_COMMIT_ID = "sync_id"; // TODO: Remove sync_id in 9.0 + @UpdateForV9 // TODO: Remove sync_id in 9.0 + public static final String SYNC_COMMIT_ID = "sync_id"; public static final String HISTORY_UUID_KEY = "history_uuid"; public static final String FORCE_MERGE_UUID_KEY = "force_merge_uuid"; public static final String MIN_RETAINED_SEQNO = "min_retained_seq_no"; From 020f4ce6c60d7357a9e6d912e2106652d65169be Mon Sep 17 00:00:00 2001 From: David Turner Date: Mon, 20 Nov 2023 06:40:43 +0000 Subject: [PATCH 02/28] Improve update-for-v9 reminders from get-aliases (#102336) In #101815 we made some changes to the get-aliases API that need followup actions when preparing to release v9. At the time they were marked with a transport version that will become obsolete in v9, but with #101767 we now have a better mechanism for leaving a reminder for this kind of action. This commit updates things to use the new @UpdateForV9 annotation instead. --- .../action/admin/indices/alias/get/GetAliasesRequest.java | 7 +++++-- .../action/admin/indices/alias/get/GetAliasesResponse.java | 4 +++- .../admin/indices/alias/get/TransportGetAliasesAction.java | 5 +++-- .../rest/action/admin/indices/RestGetAliasesAction.java | 2 ++ .../admin/indices/alias/get/GetAliasesResponseTests.java | 7 ++++--- 5 files changed, 17 insertions(+), 8 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/alias/get/GetAliasesRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/alias/get/GetAliasesRequest.java index ee6797ca58fb9..9d10065c9c3e9 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/alias/get/GetAliasesRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/alias/get/GetAliasesRequest.java @@ -15,6 +15,7 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.core.UpdateForV9; import org.elasticsearch.tasks.CancellableTask; import org.elasticsearch.tasks.Task; import org.elasticsearch.tasks.TaskId; @@ -22,6 +23,7 @@ import java.io.IOException; import java.util.Map; +@UpdateForV9 // make this class a regular ActionRequest rather than a MasterNodeReadRequest public class GetAliasesRequest extends MasterNodeReadRequest implements AliasesRequest { public static final IndicesOptions DEFAULT_INDICES_OPTIONS = IndicesOptions.strictExpandHidden(); @@ -40,9 +42,10 @@ public GetAliasesRequest() {} /** * NB prior to 8.12 get-aliases was a TransportMasterNodeReadAction so for BwC we must remain able to read these requests until we no - * longer need to support {@link org.elasticsearch.TransportVersions#CLUSTER_FEATURES_ADDED} and earlier. Once we remove this we can - * also make this class a regular ActionRequest instead of a MasterNodeReadRequest. + * longer need to support calling this action remotely. Once we remove this we can also make this class a regular ActionRequest instead + * of a MasterNodeReadRequest. */ + @UpdateForV9 // remove this constructor public GetAliasesRequest(StreamInput in) throws IOException { super(in); indices = in.readStringArray(); diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/alias/get/GetAliasesResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/alias/get/GetAliasesResponse.java index c0e26b16585c4..edb05b0fcef75 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/alias/get/GetAliasesResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/alias/get/GetAliasesResponse.java @@ -12,6 +12,7 @@ import org.elasticsearch.cluster.metadata.AliasMetadata; import org.elasticsearch.cluster.metadata.DataStreamAlias; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.core.UpdateForV9; import java.io.IOException; import java.util.List; @@ -38,8 +39,9 @@ public Map> getDataStreamAliases() { /** * NB prior to 8.12 get-aliases was a TransportMasterNodeReadAction so for BwC we must remain able to write these responses until we no - * longer need to support {@link org.elasticsearch.TransportVersions#CLUSTER_FEATURES_ADDED} and earlier. + * longer need to support calling this action remotely. */ + @UpdateForV9 // replace this implementation with TransportAction.localOnly() @Override public void writeTo(StreamOutput out) throws IOException { out.writeMap(aliases, StreamOutput::writeCollection); diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/alias/get/TransportGetAliasesAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/alias/get/TransportGetAliasesAction.java index e43d1a825c233..9b9fb49c1bbe0 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/alias/get/TransportGetAliasesAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/alias/get/TransportGetAliasesAction.java @@ -24,6 +24,7 @@ import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.common.regex.Regex; import org.elasticsearch.common.util.concurrent.ThreadContext; +import org.elasticsearch.core.UpdateForV9; import org.elasticsearch.indices.SystemIndices; import org.elasticsearch.indices.SystemIndices.SystemIndexAccessLevel; import org.elasticsearch.tasks.CancellableTask; @@ -41,9 +42,9 @@ /** * NB prior to 8.12 this was a TransportMasterNodeReadAction so for BwC it must be registered with the TransportService (i.e. a - * HandledTransportAction) until we no longer need to support {@link org.elasticsearch.TransportVersions#CLUSTER_FEATURES_ADDED} and - * earlier. + * HandledTransportAction) until we no longer need to support calling this action remotely. */ +@UpdateForV9 // remove the HandledTransportAction superclass, this action need not be registered with the TransportService public class TransportGetAliasesAction extends TransportLocalClusterStateAction { private static final DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(TransportGetAliasesAction.class); diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetAliasesAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetAliasesAction.java index b6e1240a3f85a..a8f6fa325b468 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetAliasesAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetAliasesAction.java @@ -21,6 +21,7 @@ import org.elasticsearch.common.regex.Regex; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.core.RestApiVersion; +import org.elasticsearch.core.UpdateForV9; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.RestResponse; @@ -50,6 +51,7 @@ @ServerlessScope(Scope.PUBLIC) public class RestGetAliasesAction extends BaseRestHandler { + @UpdateForV9 // reject the deprecated ?local parameter private static final DeprecationLogger DEPRECATION_LOGGER = DeprecationLogger.getLogger(RestGetAliasesAction.class); @Override diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/alias/get/GetAliasesResponseTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/alias/get/GetAliasesResponseTests.java index 6fde4bed97a17..433563b99ef64 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/alias/get/GetAliasesResponseTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/alias/get/GetAliasesResponseTests.java @@ -15,6 +15,7 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.core.Tuple; +import org.elasticsearch.core.UpdateForV9; import org.elasticsearch.test.AbstractWireSerializingTestCase; import java.util.ArrayList; @@ -24,6 +25,7 @@ import java.util.Map; import java.util.function.Predicate; +@UpdateForV9 // no need to round-trip these objects over the wire any more, we only need a checkEqualsAndHashCode test public class GetAliasesResponseTests extends AbstractWireSerializingTestCase { @Override @@ -33,9 +35,8 @@ protected GetAliasesResponse createTestInstance() { /** * NB prior to 8.12 get-aliases was a TransportMasterNodeReadAction so for BwC we must remain able to write these responses so that - * older nodes can read them until we no longer need to support {@link org.elasticsearch.TransportVersions#CLUSTER_FEATURES_ADDED} and - * earlier. The reader implementation below is the production implementation from earlier versions, but moved here because it is unused - * in production now. + * older nodes can read them until we no longer need to support calling this action remotely. The reader implementation below is the + * production implementation from earlier versions, but moved here because it is unused in production now. */ @Override protected Writeable.Reader instanceReader() { From ca3b6f0953ef83d4f5df2c2910d54158a0792738 Mon Sep 17 00:00:00 2001 From: Daniel Mitterdorfer Date: Mon, 20 Nov 2023 08:05:25 +0100 Subject: [PATCH 03/28] Consider duplicate stacktraces in custom index (#102292) With this commit we consider also duplicate stacktraces in custom indices for profiling events. We achieve that by using the recently introduced `counted_terms` aggregation (requires that the respective field is mapped as `counted_keyword`). Relates #101826 Relates #102020 --- docs/changelog/102292.yaml | 5 +++++ .../countedkeyword/CountedTermsAggregationBuilder.java | 10 ++++++---- x-pack/plugin/profiling/build.gradle | 1 + .../xpack/profiling/GetStackTracesActionIT.java | 5 +++-- .../xpack/profiling/ProfilingTestCase.java | 2 ++ .../internalClusterTest/resources/data/apm-test.ndjson | 2 +- .../resources/indices/apm-test.json | 2 +- .../xpack/profiling/TransportGetStackTracesAction.java | 4 ++-- 8 files changed, 21 insertions(+), 10 deletions(-) create mode 100644 docs/changelog/102292.yaml diff --git a/docs/changelog/102292.yaml b/docs/changelog/102292.yaml new file mode 100644 index 0000000000000..953c3ffdf6150 --- /dev/null +++ b/docs/changelog/102292.yaml @@ -0,0 +1,5 @@ +pr: 102292 +summary: Consider duplicate stacktraces in custom index +area: Application +type: enhancement +issues: [] diff --git a/x-pack/plugin/mapper-counted-keyword/src/main/java/org/elasticsearch/xpack/countedkeyword/CountedTermsAggregationBuilder.java b/x-pack/plugin/mapper-counted-keyword/src/main/java/org/elasticsearch/xpack/countedkeyword/CountedTermsAggregationBuilder.java index 23adacd8f65fa..72e3eb4efacf9 100644 --- a/x-pack/plugin/mapper-counted-keyword/src/main/java/org/elasticsearch/xpack/countedkeyword/CountedTermsAggregationBuilder.java +++ b/x-pack/plugin/mapper-counted-keyword/src/main/java/org/elasticsearch/xpack/countedkeyword/CountedTermsAggregationBuilder.java @@ -30,10 +30,12 @@ import java.util.Map; import java.util.Objects; -class CountedTermsAggregationBuilder extends ValuesSourceAggregationBuilder { +public class CountedTermsAggregationBuilder extends ValuesSourceAggregationBuilder { public static final String NAME = "counted_terms"; - public static final ValuesSourceRegistry.RegistryKey REGISTRY_KEY = - new ValuesSourceRegistry.RegistryKey<>(NAME, CountedTermsAggregatorSupplier.class); + static final ValuesSourceRegistry.RegistryKey REGISTRY_KEY = new ValuesSourceRegistry.RegistryKey<>( + NAME, + CountedTermsAggregatorSupplier.class + ); public static final ParseField REQUIRED_SIZE_FIELD_NAME = new ParseField("size"); @@ -50,7 +52,7 @@ class CountedTermsAggregationBuilder extends ValuesSourceAggregationBuilder> nodePlugins() { LocalStateProfilingXPackPlugin.class, IndexLifecycle.class, UnsignedLongMapperPlugin.class, + CountedKeywordMapperPlugin.class, VersionFieldPlugin.class, getTestTransportPlugin() ); diff --git a/x-pack/plugin/profiling/src/internalClusterTest/resources/data/apm-test.ndjson b/x-pack/plugin/profiling/src/internalClusterTest/resources/data/apm-test.ndjson index d68c6b5e4f2b1..d147256d6b90f 100644 --- a/x-pack/plugin/profiling/src/internalClusterTest/resources/data/apm-test.ndjson +++ b/x-pack/plugin/profiling/src/internalClusterTest/resources/data/apm-test.ndjson @@ -1,2 +1,2 @@ {"create": {"_index": "apm-test-001"}} -{"@timestamp": "1698624000", "transaction.name": "encodeSha1", "transaction.profiler_stack_trace_ids": "Ce77w10WeIDow3kd1jowlA"} +{"@timestamp": "1698624000", "transaction.name": "encodeSha1", "transaction.profiler_stack_trace_ids": ["Ce77w10WeIDow3kd1jowlA", "JvISdnJ47BQ01489cwF9DA", "JvISdnJ47BQ01489cwF9DA", "Ce77w10WeIDow3kd1jowlA", "Ce77w10WeIDow3kd1jowlA"]} diff --git a/x-pack/plugin/profiling/src/internalClusterTest/resources/indices/apm-test.json b/x-pack/plugin/profiling/src/internalClusterTest/resources/indices/apm-test.json index eba8ed14059a7..e0aeb707ffc76 100644 --- a/x-pack/plugin/profiling/src/internalClusterTest/resources/indices/apm-test.json +++ b/x-pack/plugin/profiling/src/internalClusterTest/resources/indices/apm-test.json @@ -13,7 +13,7 @@ "type": "date" }, "transaction.profiler_stack_trace_ids": { - "type": "keyword" + "type": "counted_keyword" }, "transaction.name": { "type": "keyword" diff --git a/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/TransportGetStackTracesAction.java b/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/TransportGetStackTracesAction.java index 28adb58593eef..a781f91f30bbe 100644 --- a/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/TransportGetStackTracesAction.java +++ b/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/TransportGetStackTracesAction.java @@ -40,6 +40,7 @@ import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xcontent.ObjectPath; +import org.elasticsearch.xpack.countedkeyword.CountedTermsAggregationBuilder; import java.time.Instant; import java.util.ArrayList; @@ -184,8 +185,7 @@ private void searchGenericEvents(GetStackTracesRequest request, ActionListener Date: Sun, 19 Nov 2023 23:08:32 -0800 Subject: [PATCH 04/28] Fix references of ESQL enrich lookup requests (#102361) The EnrichIT suite failed because enrich lookup requests are released while a driver is still executing. While we can extend the lifecycle of the transport requests, this PR extends the lifecycle of enrich lookup requests only to minimize the impact of the fix. --- .../java/org/elasticsearch/xpack/esql/action/EnrichIT.java | 2 -- .../elasticsearch/xpack/esql/enrich/EnrichLookupService.java | 3 ++- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EnrichIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EnrichIT.java index b855fbd15be12..46aaa6fab16a5 100644 --- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EnrichIT.java +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EnrichIT.java @@ -7,7 +7,6 @@ package org.elasticsearch.xpack.esql.action; -import org.apache.lucene.tests.util.LuceneTestCase; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.TransportAction; import org.elasticsearch.client.internal.Client; @@ -60,7 +59,6 @@ import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasSize; -@LuceneTestCase.AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/pull/102184") public class EnrichIT extends AbstractEsqlIntegTestCase { @Override diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/EnrichLookupService.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/EnrichLookupService.java index 7dd9f01a9d6c9..384563cb815a4 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/EnrichLookupService.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/EnrichLookupService.java @@ -335,7 +335,8 @@ private static Operator droppingBlockOperator(int totalBlocks, int droppingPosit private class TransportHandler implements TransportRequestHandler { @Override public void messageReceived(LookupRequest request, TransportChannel channel, Task task) { - ActionListener listener = new ChannelActionListener<>(channel); + request.incRef(); + ActionListener listener = ActionListener.runBefore(new ChannelActionListener<>(channel), request::decRef); doLookup( request.sessionId, (CancellableTask) task, From e55cd4ac64a63313cf4733e67e782b3e8932190f Mon Sep 17 00:00:00 2001 From: David Turner Date: Mon, 20 Nov 2023 07:52:43 +0000 Subject: [PATCH 05/28] Retry CCR assertTotalHitCount on 404 (#102364) Some CCR tests check for leader/follower sync by performing a search on the follower, but it's possible the follower index does not even exist which today receives a 404 that maps onto a `ResponseException`. If we expect the indices to be in sync already then this should fail the test, whereas if we are retrying in an `assertBusy()` then this should trigger a retry. Either way we should map a 404 onto an `AssertionError` rather than a `ResponseException`. That's what this commit does. Closes #102000 --- .../main/java/org/elasticsearch/test/rest/ESRestTestCase.java | 3 ++- .../java/org/elasticsearch/upgrades/CcrRollingUpgradeIT.java | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java index aff8a20aa88b6..a3ff4d87da73e 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java @@ -1485,8 +1485,9 @@ private static Set runningTasks(Response response) throws IOException { return runningTasks; } - public static void assertOK(Response response) { + public static Response assertOK(Response response) { assertThat(response.getStatusLine().getStatusCode(), anyOf(equalTo(200), equalTo(201))); + return response; } public static ObjectPath assertOKAndCreateObjectPath(Response response) throws IOException { diff --git a/x-pack/qa/rolling-upgrade-multi-cluster/src/test/java/org/elasticsearch/upgrades/CcrRollingUpgradeIT.java b/x-pack/qa/rolling-upgrade-multi-cluster/src/test/java/org/elasticsearch/upgrades/CcrRollingUpgradeIT.java index b1e1888aba75d..59928470eb247 100644 --- a/x-pack/qa/rolling-upgrade-multi-cluster/src/test/java/org/elasticsearch/upgrades/CcrRollingUpgradeIT.java +++ b/x-pack/qa/rolling-upgrade-multi-cluster/src/test/java/org/elasticsearch/upgrades/CcrRollingUpgradeIT.java @@ -88,7 +88,6 @@ public void testUniDirectionalIndexFollowing() throws Exception { } } - @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/102000") public void testAutoFollowing() throws Exception { String leaderIndex1 = "logs-20200101"; String leaderIndex2 = "logs-20200102"; @@ -372,7 +371,8 @@ private static void assertTotalHitCount(final String index, final int expectedTo private static void verifyTotalHitCount(final String index, final int expectedTotalHits, final RestClient client) throws IOException { final Request request = new Request("GET", "/" + index + "/_search"); request.addParameter(TOTAL_HITS_AS_INT_PARAM, "true"); - Map response = toMap(client.performRequest(request)); + request.addParameter("ignore", "404"); // If index not found, trip the assertOK (i.e. retry an assertBusy) rather than throwing + Map response = toMap(assertOK(client.performRequest(request))); final int totalHits = (int) XContentMapValues.extractValue("hits.total", response); assertThat(totalHits, equalTo(expectedTotalHits)); } From c7985b698cf3319443fb6a57b38cd665d9f755ff Mon Sep 17 00:00:00 2001 From: David Turner Date: Mon, 20 Nov 2023 08:12:51 +0000 Subject: [PATCH 06/28] Unwrap exception more tenaciously in testQueuedOperationsAndBrokenRepoOnMasterFailOver (#102352) There can be more than 10 layers of wrapping RTEs, see #102351. As a workaround to address the test failure, this commit just manually unwraps them all. Closes #102348 --- .../snapshots/ConcurrentSnapshotsIT.java | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/ConcurrentSnapshotsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/ConcurrentSnapshotsIT.java index ca522064e3d04..f91a848ed2362 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/ConcurrentSnapshotsIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/ConcurrentSnapshotsIT.java @@ -39,6 +39,7 @@ import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.disruption.NetworkDisruption; import org.elasticsearch.test.transport.MockTransportService; +import org.elasticsearch.transport.RemoteTransportException; import java.io.IOException; import java.nio.file.Files; @@ -768,7 +769,18 @@ public void testQueuedOperationsAndBrokenRepoOnMasterFailOver() throws Exception ensureStableCluster(3); awaitNoMoreRunningOperations(); - expectThrows(RepositoryException.class, deleteFuture::actionGet); + var innerException = expectThrows(ExecutionException.class, RuntimeException.class, deleteFuture::get); + + // There may be many layers of RTE to unwrap here, see https://github.com/elastic/elasticsearch/issues/102351. + // ExceptionsHelper#unwrapCause gives up at 10 layers of wrapping so we must unwrap more tenaciously by hand here: + while (true) { + if (innerException instanceof RemoteTransportException remoteTransportException) { + innerException = asInstanceOf(RuntimeException.class, remoteTransportException.getCause()); + } else { + assertThat(innerException, instanceOf(RepositoryException.class)); + break; + } + } } public void testQueuedSnapshotOperationsAndBrokenRepoOnMasterFailOver() throws Exception { From ce700b6f2798acacdb3c89adbc6890a8335719e3 Mon Sep 17 00:00:00 2001 From: David Turner Date: Mon, 20 Nov 2023 08:27:37 +0000 Subject: [PATCH 07/28] Yet more logging for S3SnapshotRepoTestKitIT (#102331) Moves the logging added in #102299 onto the test cluster where it belongs, and adds wire logging to this test suite too. Relates #102294 --- x-pack/plugin/snapshot-repo-test-kit/qa/s3/build.gradle | 6 ++++++ .../blobstore/testkit/S3SnapshotRepoTestKitIT.java | 5 ----- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/x-pack/plugin/snapshot-repo-test-kit/qa/s3/build.gradle b/x-pack/plugin/snapshot-repo-test-kit/qa/s3/build.gradle index 36b13ef8b12a7..513589cfbfa06 100644 --- a/x-pack/plugin/snapshot-repo-test-kit/qa/s3/build.gradle +++ b/x-pack/plugin/snapshot-repo-test-kit/qa/s3/build.gradle @@ -69,6 +69,12 @@ testClusters.matching { it.name == "javaRestTest" }.configureEach { println "Using an external service to test " + project.name } setting 'xpack.security.enabled', 'false' + + // Additional tracing related to investigation into https://github.com/elastic/elasticsearch/issues/102294 + setting 'logger.org.elasticsearch.repositories.s3', 'TRACE' + setting 'logger.org.elasticsearch.repositories.blobstore.testkit', 'TRACE' + setting 'logger.com.amazonaws.request', 'DEBUG' + setting 'logger.org.apache.http.wire', 'DEBUG' } tasks.register("s3ThirdPartyTest") { diff --git a/x-pack/plugin/snapshot-repo-test-kit/qa/s3/src/javaRestTest/java/org/elasticsearch/repositories/blobstore/testkit/S3SnapshotRepoTestKitIT.java b/x-pack/plugin/snapshot-repo-test-kit/qa/s3/src/javaRestTest/java/org/elasticsearch/repositories/blobstore/testkit/S3SnapshotRepoTestKitIT.java index 4b1abfa9dcd62..9e40f7b7aada2 100644 --- a/x-pack/plugin/snapshot-repo-test-kit/qa/s3/src/javaRestTest/java/org/elasticsearch/repositories/blobstore/testkit/S3SnapshotRepoTestKitIT.java +++ b/x-pack/plugin/snapshot-repo-test-kit/qa/s3/src/javaRestTest/java/org/elasticsearch/repositories/blobstore/testkit/S3SnapshotRepoTestKitIT.java @@ -7,7 +7,6 @@ package org.elasticsearch.repositories.blobstore.testkit; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.test.junit.annotations.TestIssueLogging; import static org.hamcrest.Matchers.blankOrNullString; import static org.hamcrest.Matchers.not; @@ -31,10 +30,6 @@ protected Settings repositorySettings() { } @Override - @TestIssueLogging( - issueUrl = "https://github.com/elastic/elasticsearch/issues/102294", - value = "org.elasticsearch.repositories.s3:TRACE,org.elasticsearch.repositories.blobstore.testkit:TRACE" - ) public void testRepositoryAnalysis() throws Exception { super.testRepositoryAnalysis(); } From e8c3a72785a76159d8e341f04d9e173e4e8c6f85 Mon Sep 17 00:00:00 2001 From: David Turner Date: Mon, 20 Nov 2023 08:37:02 +0000 Subject: [PATCH 08/28] Extract constant for ?ignore pseudo-parameter (#102365) Today `RestClient` interprets an `?ignore=` request parameter as an indication that certain HTTP response codes should be considered successful and not raise a `ResponseException`. This commit replaces the magic literal `"ignore"` with a constant and adds a utility to specify the ignored codes as `RestStatus` values. --- .../java/org/elasticsearch/client/RestClient.java | 8 ++++++-- .../client/RestClientSingleHostTests.java | 2 ++ .../elasticsearch/test/rest/ESRestTestCase.java | 14 +++++++++++--- .../yaml/restspec/ClientYamlSuiteRestSpec.java | 4 +++- .../exporter/http/PublishableHttpResource.java | 10 ++++++---- .../AbstractPublishableHttpResourceTestCase.java | 7 ++++--- .../org/elasticsearch/test/TestSecurityClient.java | 4 +++- .../integration/TransformRestTestCase.java | 3 ++- .../upgrades/BasicLicenseUpgradeIT.java | 5 +++-- .../upgrades/CcrRollingUpgradeIT.java | 2 +- 10 files changed, 41 insertions(+), 18 deletions(-) diff --git a/client/rest/src/main/java/org/elasticsearch/client/RestClient.java b/client/rest/src/main/java/org/elasticsearch/client/RestClient.java index 7154a2be5bbd8..ed087bef0ac76 100644 --- a/client/rest/src/main/java/org/elasticsearch/client/RestClient.java +++ b/client/rest/src/main/java/org/elasticsearch/client/RestClient.java @@ -87,6 +87,7 @@ import static java.nio.charset.StandardCharsets.UTF_8; import static java.util.Collections.singletonList; +import static org.elasticsearch.client.RestClient.IGNORE_RESPONSE_CODES_PARAM; /** * Client that connects to an Elasticsearch cluster through HTTP. @@ -106,6 +107,9 @@ * Requests can be traced by enabling trace logging for "tracer". The trace logger outputs requests and responses in curl format. */ public class RestClient implements Closeable { + + public static final String IGNORE_RESPONSE_CODES_PARAM = "ignore"; + private static final Log logger = LogFactory.getLog(RestClient.class); private final CloseableHttpAsyncClient client; @@ -780,8 +784,8 @@ private class InternalRequest { this.request = request; Map params = new HashMap<>(request.getParameters()); params.putAll(request.getOptions().getParameters()); - // ignore is a special parameter supported by the clients, shouldn't be sent to es - String ignoreString = params.remove("ignore"); + // IGNORE_RESPONSE_CODES_PARAM is a special parameter supported by the clients, shouldn't be sent to es + String ignoreString = params.remove(IGNORE_RESPONSE_CODES_PARAM); this.ignoreErrorCodes = getIgnoreErrorCodes(ignoreString, request.getMethod()); URI uri = buildUri(pathPrefix, request.getEndpoint(), params); this.httpRequest = createHttpRequest(request.getMethod(), uri, request.getEntity(), compressionEnabled); diff --git a/client/rest/src/test/java/org/elasticsearch/client/RestClientSingleHostTests.java b/client/rest/src/test/java/org/elasticsearch/client/RestClientSingleHostTests.java index a1c4d3fab076a..10d24242ae620 100644 --- a/client/rest/src/test/java/org/elasticsearch/client/RestClientSingleHostTests.java +++ b/client/rest/src/test/java/org/elasticsearch/client/RestClientSingleHostTests.java @@ -275,6 +275,7 @@ public void testErrorStatusCodes() throws Exception { try { Request request = new Request(method, "/" + errorStatusCode); if (false == ignoreParam.isEmpty()) { + // literal "ignore" rather than IGNORE_RESPONSE_CODES_PARAM since this is something on which callers might rely request.addParameter("ignore", ignoreParam); } Response response = restClient.performRequest(request); @@ -568,6 +569,7 @@ private HttpUriRequest performRandomRequest(String method) throws Exception { if (randomBoolean()) { ignore += "," + randomFrom(RestClientTestUtil.getAllErrorStatusCodes()); } + // literal "ignore" rather than IGNORE_RESPONSE_CODES_PARAM since this is something on which callers might rely request.addParameter("ignore", ignore); } URI uri = uriBuilder.build(); diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java index a3ff4d87da73e..3327137cef7b7 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java @@ -112,6 +112,7 @@ import static java.util.Collections.sort; import static java.util.Collections.unmodifiableList; +import static org.elasticsearch.client.RestClient.IGNORE_RESPONSE_CODES_PARAM; import static org.elasticsearch.core.Strings.format; import static org.hamcrest.Matchers.anyOf; import static org.hamcrest.Matchers.containsString; @@ -1157,7 +1158,7 @@ private void wipeRollupJobs() throws IOException { @SuppressWarnings("unchecked") String jobId = (String) ((Map) jobConfig.get("config")).get("id"); Request request = new Request("POST", "/_rollup/job/" + jobId + "/_stop"); - request.addParameter("ignore", "404"); + setIgnoredErrorResponseCodes(request, RestStatus.NOT_FOUND); request.addParameter("wait_for_completion", "true"); request.addParameter("timeout", "10s"); logger.debug("stopping rollup job [{}]", jobId); @@ -1168,7 +1169,7 @@ private void wipeRollupJobs() throws IOException { @SuppressWarnings("unchecked") String jobId = (String) ((Map) jobConfig.get("config")).get("id"); Request request = new Request("DELETE", "/_rollup/job/" + jobId); - request.addParameter("ignore", "404"); // Ignore 404s because they imply someone was racing us to delete this + setIgnoredErrorResponseCodes(request, RestStatus.NOT_FOUND); // 404s imply someone was racing us to delete this logger.debug("deleting rollup job [{}]", jobId); adminClient().performRequest(request); } @@ -1846,7 +1847,7 @@ protected static void deleteSnapshot(RestClient restClient, String repository, S throws IOException { final Request request = new Request(HttpDelete.METHOD_NAME, "_snapshot/" + repository + '/' + snapshot); if (ignoreMissing) { - request.addParameter("ignore", "404"); + setIgnoredErrorResponseCodes(request, RestStatus.NOT_FOUND); } final Response response = restClient.performRequest(request); assertThat(response.getStatusLine().getStatusCode(), ignoreMissing ? anyOf(equalTo(200), equalTo(404)) : equalTo(200)); @@ -2244,4 +2245,11 @@ protected Map getHistoricalFeatures() { return historicalFeatures; } + + public static void setIgnoredErrorResponseCodes(Request request, RestStatus... restStatuses) { + request.addParameter( + IGNORE_RESPONSE_CODES_PARAM, + Arrays.stream(restStatuses).map(restStatus -> Integer.toString(restStatus.getStatus())).collect(Collectors.joining(",")) + ); + } } diff --git a/test/yaml-rest-runner/src/main/java/org/elasticsearch/test/rest/yaml/restspec/ClientYamlSuiteRestSpec.java b/test/yaml-rest-runner/src/main/java/org/elasticsearch/test/rest/yaml/restspec/ClientYamlSuiteRestSpec.java index be34ee9be0ea1..8662d886cce89 100644 --- a/test/yaml-rest-runner/src/main/java/org/elasticsearch/test/rest/yaml/restspec/ClientYamlSuiteRestSpec.java +++ b/test/yaml-rest-runner/src/main/java/org/elasticsearch/test/rest/yaml/restspec/ClientYamlSuiteRestSpec.java @@ -24,6 +24,8 @@ import java.util.Set; import java.util.stream.Stream; +import static org.elasticsearch.client.RestClient.IGNORE_RESPONSE_CODES_PARAM; + /** * Holds the specification used to turn {@code do} actions in the YAML suite into REST api calls. */ @@ -69,7 +71,7 @@ public boolean isGlobalParameter(String param) { * that they influence the client behaviour and don't get sent to Elasticsearch */ public boolean isClientParameter(String name) { - return "ignore".equals(name); + return IGNORE_RESPONSE_CODES_PARAM.equals(name); } /** diff --git a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/http/PublishableHttpResource.java b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/http/PublishableHttpResource.java index d37f4669484a0..e2d4d173af013 100644 --- a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/http/PublishableHttpResource.java +++ b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/http/PublishableHttpResource.java @@ -27,9 +27,11 @@ import java.util.Collections; import java.util.Map; import java.util.Set; -import java.util.stream.Collectors; +import static java.util.stream.Collectors.joining; +import static org.elasticsearch.client.RestClient.IGNORE_RESPONSE_CODES_PARAM; import static org.elasticsearch.core.Strings.format; +import static org.elasticsearch.rest.RestStatus.NOT_FOUND; /** * {@code PublishableHttpResource} represents an {@link HttpResource} that is a single file or object that can be checked and @@ -254,7 +256,7 @@ protected void checkForResource( // avoid exists and DNE parameters from being an exception by default final Set expectedResponseCodes = Sets.union(exists, doesNotExist); - request.addParameter("ignore", expectedResponseCodes.stream().map(i -> i.toString()).collect(Collectors.joining(","))); + request.addParameter(IGNORE_RESPONSE_CODES_PARAM, expectedResponseCodes.stream().map(Object::toString).collect(joining(","))); client.performRequestAsync(request, new ResponseListener() { @@ -436,9 +438,9 @@ protected void deleteResource( final Request request = new Request("DELETE", resourceBasePath + "/" + resourceName); addDefaultParameters(request); - if (false == defaultParameters.containsKey("ignore")) { + if (false == defaultParameters.containsKey(IGNORE_RESPONSE_CODES_PARAM)) { // avoid 404 being an exception by default - request.addParameter("ignore", Integer.toString(RestStatus.NOT_FOUND.getStatus())); + request.addParameter(IGNORE_RESPONSE_CODES_PARAM, Integer.toString(NOT_FOUND.getStatus())); } client.performRequestAsync(request, new ResponseListener() { diff --git a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/exporter/http/AbstractPublishableHttpResourceTestCase.java b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/exporter/http/AbstractPublishableHttpResourceTestCase.java index 4878289cae8d6..b72891708e780 100644 --- a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/exporter/http/AbstractPublishableHttpResourceTestCase.java +++ b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/exporter/http/AbstractPublishableHttpResourceTestCase.java @@ -30,8 +30,9 @@ import java.util.Map; import java.util.Set; import java.util.function.Predicate; -import java.util.stream.Collectors; +import static java.util.stream.Collectors.joining; +import static org.elasticsearch.client.RestClient.IGNORE_RESPONSE_CODES_PARAM; import static org.elasticsearch.xpack.monitoring.exporter.http.AsyncHttpResourceHelper.mockBooleanActionListener; import static org.elasticsearch.xpack.monitoring.exporter.http.AsyncHttpResourceHelper.mockPublishResultActionListener; import static org.elasticsearch.xpack.monitoring.exporter.http.AsyncHttpResourceHelper.whenPerformRequestAsyncWith; @@ -443,7 +444,7 @@ protected Map getParameters( final Set statusCodes = Sets.union(exists, doesNotExist); final Map parametersWithIgnore = new HashMap<>(parameters); - parametersWithIgnore.putIfAbsent("ignore", statusCodes.stream().map(i -> i.toString()).collect(Collectors.joining(","))); + parametersWithIgnore.putIfAbsent(IGNORE_RESPONSE_CODES_PARAM, statusCodes.stream().map(Object::toString).collect(joining(","))); return parametersWithIgnore; } @@ -451,7 +452,7 @@ protected Map getParameters( protected Map deleteParameters(final Map parameters) { final Map parametersWithIgnore = new HashMap<>(parameters); - parametersWithIgnore.putIfAbsent("ignore", "404"); + parametersWithIgnore.putIfAbsent(IGNORE_RESPONSE_CODES_PARAM, Integer.toString(RestStatus.NOT_FOUND.getStatus())); return parametersWithIgnore; } diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/test/TestSecurityClient.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/test/TestSecurityClient.java index 13c8612487d89..4888c0f4c9721 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/test/TestSecurityClient.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/test/TestSecurityClient.java @@ -25,6 +25,7 @@ import org.elasticsearch.common.xcontent.XContentParserUtils; import org.elasticsearch.core.Nullable; import org.elasticsearch.core.Tuple; +import org.elasticsearch.rest.RestStatus; import org.elasticsearch.xcontent.ObjectPath; import org.elasticsearch.xcontent.ToXContent; import org.elasticsearch.xcontent.XContentBuilder; @@ -51,6 +52,7 @@ import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken; import static org.elasticsearch.test.rest.ESRestTestCase.entityAsMap; +import static org.elasticsearch.test.rest.ESRestTestCase.setIgnoredErrorResponseCodes; public class TestSecurityClient { @@ -395,7 +397,7 @@ public TokenInvalidation invalidateTokens(String requestBody) throws IOException final Request request = new Request(HttpDelete.METHOD_NAME, endpoint); // This API returns 404 (with the same body as a 200 response) if there's nothing to delete. // RestClient will throw an exception on 404, but we don't want that, we want to parse the body and return it - request.addParameter("ignore", "404"); + setIgnoredErrorResponseCodes(request, RestStatus.NOT_FOUND); request.setJsonEntity(requestBody); final Map responseBody = entityAsMap(execute(request)); final List> errors = (List>) responseBody.get("error_details"); diff --git a/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformRestTestCase.java b/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformRestTestCase.java index e6388bb6fea5d..c616c1c238171 100644 --- a/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformRestTestCase.java +++ b/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformRestTestCase.java @@ -19,6 +19,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.common.xcontent.support.XContentMapValues; +import org.elasticsearch.rest.RestStatus; import org.elasticsearch.test.rest.ESRestTestCase; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xpack.core.transform.TransformField; @@ -618,7 +619,7 @@ protected static void deleteTransform(String transformId) throws IOException { protected static void deleteTransform(String transformId, boolean ignoreNotFound, boolean deleteDestIndex) throws IOException { Request request = new Request("DELETE", getTransformEndpoint() + transformId); if (ignoreNotFound) { - request.addParameter("ignore", "404"); + setIgnoredErrorResponseCodes(request, RestStatus.NOT_FOUND); } if (deleteDestIndex) { request.addParameter(TransformField.DELETE_DEST_INDEX.getPreferredName(), Boolean.TRUE.toString()); diff --git a/x-pack/qa/rolling-upgrade-basic/src/test/java/org/elasticsearch/upgrades/BasicLicenseUpgradeIT.java b/x-pack/qa/rolling-upgrade-basic/src/test/java/org/elasticsearch/upgrades/BasicLicenseUpgradeIT.java index 75fcc5cf6e7ad..da8a4c806a0f5 100644 --- a/x-pack/qa/rolling-upgrade-basic/src/test/java/org/elasticsearch/upgrades/BasicLicenseUpgradeIT.java +++ b/x-pack/qa/rolling-upgrade-basic/src/test/java/org/elasticsearch/upgrades/BasicLicenseUpgradeIT.java @@ -8,6 +8,7 @@ import org.elasticsearch.client.Request; import org.elasticsearch.client.Response; +import org.elasticsearch.rest.RestStatus; import java.util.Map; @@ -28,7 +29,7 @@ private void checkBasicLicense() throws Exception { final Request request = new Request("GET", "/_license"); // This avoids throwing a ResponseException when the license is not ready yet // allowing to retry the check using assertBusy - request.addParameter("ignore", "404"); + setIgnoredErrorResponseCodes(request, RestStatus.NOT_FOUND); Response licenseResponse = client().performRequest(request); assertOK(licenseResponse); Map licenseResponseMap = entityAsMap(licenseResponse); @@ -42,7 +43,7 @@ private void checkNonExpiringBasicLicense() throws Exception { final Request request = new Request("GET", "/_license"); // This avoids throwing a ResponseException when the license is not ready yet // allowing to retry the check using assertBusy - request.addParameter("ignore", "404"); + setIgnoredErrorResponseCodes(request, RestStatus.NOT_FOUND); Response licenseResponse = client().performRequest(request); assertOK(licenseResponse); Map licenseResponseMap = entityAsMap(licenseResponse); diff --git a/x-pack/qa/rolling-upgrade-multi-cluster/src/test/java/org/elasticsearch/upgrades/CcrRollingUpgradeIT.java b/x-pack/qa/rolling-upgrade-multi-cluster/src/test/java/org/elasticsearch/upgrades/CcrRollingUpgradeIT.java index 59928470eb247..0b7ab1fe5980d 100644 --- a/x-pack/qa/rolling-upgrade-multi-cluster/src/test/java/org/elasticsearch/upgrades/CcrRollingUpgradeIT.java +++ b/x-pack/qa/rolling-upgrade-multi-cluster/src/test/java/org/elasticsearch/upgrades/CcrRollingUpgradeIT.java @@ -371,7 +371,7 @@ private static void assertTotalHitCount(final String index, final int expectedTo private static void verifyTotalHitCount(final String index, final int expectedTotalHits, final RestClient client) throws IOException { final Request request = new Request("GET", "/" + index + "/_search"); request.addParameter(TOTAL_HITS_AS_INT_PARAM, "true"); - request.addParameter("ignore", "404"); // If index not found, trip the assertOK (i.e. retry an assertBusy) rather than throwing + setIgnoredErrorResponseCodes(request, RestStatus.NOT_FOUND); // trip the assertOK (i.e. retry an assertBusy) rather than throwing Map response = toMap(assertOK(client.performRequest(request))); final int totalHits = (int) XContentMapValues.extractValue("hits.total", response); assertThat(totalHits, equalTo(expectedTotalHits)); From 5a3409b7c5df193899341ce8b4b6d1660078a5d2 Mon Sep 17 00:00:00 2001 From: Mary Gouseti Date: Mon, 20 Nov 2023 10:38:41 +0200 Subject: [PATCH 09/28] ES-6566: [DSL] Introduce new endpoint to expose data stream lifecycle stats (#101845) --- docs/changelog/101845.yaml | 5 + .../data-streams/data-stream-apis.asciidoc | 4 + .../apis/get-lifecycle-stats.asciidoc | 93 +++++++++++ .../DataStreamLifecycleServiceIT.java | 58 +++---- .../lifecycle/DataStreamLifecycleStatsIT.java | 86 ++++++++++ .../datastreams/DataStreamsPlugin.java | 5 + .../DataStreamLifecycleErrorStore.java | 6 +- .../lifecycle/DataStreamLifecycleService.java | 32 +++- .../GetDataStreamLifecycleStatsAction.java | 154 ++++++++++++++++++ ...portGetDataStreamLifecycleStatsAction.java | 110 +++++++++++++ .../RestDataStreamLifecycleStatsAction.java | 45 +++++ .../DataStreamLifecycleErrorStoreTests.java | 7 +- .../DataStreamLifecycleServiceTests.java | 27 +++ ...DataStreamLifecycleStatsResponseTests.java | 154 ++++++++++++++++++ ...etDataStreamLifecycleStatsActionTests.java | 153 +++++++++++++++++ .../xpack/security/operator/Constants.java | 1 + ...StreamLifecycleDownsamplingSecurityIT.java | 2 +- ...reamLifecycleServiceRuntimeSecurityIT.java | 3 +- 18 files changed, 907 insertions(+), 38 deletions(-) create mode 100644 docs/changelog/101845.yaml create mode 100644 docs/reference/data-streams/lifecycle/apis/get-lifecycle-stats.asciidoc create mode 100644 modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleStatsIT.java create mode 100644 modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/GetDataStreamLifecycleStatsAction.java create mode 100644 modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/TransportGetDataStreamLifecycleStatsAction.java create mode 100644 modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/rest/RestDataStreamLifecycleStatsAction.java create mode 100644 modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/action/DataStreamLifecycleStatsResponseTests.java create mode 100644 modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/action/TransportGetDataStreamLifecycleStatsActionTests.java diff --git a/docs/changelog/101845.yaml b/docs/changelog/101845.yaml new file mode 100644 index 0000000000000..0dd95bdabca57 --- /dev/null +++ b/docs/changelog/101845.yaml @@ -0,0 +1,5 @@ +pr: 101845 +summary: Introduce new endpoint to expose data stream lifecycle stats +area: Data streams +type: enhancement +issues: [] diff --git a/docs/reference/data-streams/data-stream-apis.asciidoc b/docs/reference/data-streams/data-stream-apis.asciidoc index d3580ca4448a7..3c2e703d264ff 100644 --- a/docs/reference/data-streams/data-stream-apis.asciidoc +++ b/docs/reference/data-streams/data-stream-apis.asciidoc @@ -25,6 +25,8 @@ preview:[] preview:[] * <> preview:[] +* <> +preview:[] The following API is available for <>: @@ -55,4 +57,6 @@ include::{es-repo-dir}/data-streams/lifecycle/apis/delete-lifecycle.asciidoc[] include::{es-repo-dir}/data-streams/lifecycle/apis/explain-lifecycle.asciidoc[] +include::{es-repo-dir}/data-streams/lifecycle/apis/get-lifecycle-stats.asciidoc[] + include::{es-repo-dir}/indices/downsample-data-stream.asciidoc[] diff --git a/docs/reference/data-streams/lifecycle/apis/get-lifecycle-stats.asciidoc b/docs/reference/data-streams/lifecycle/apis/get-lifecycle-stats.asciidoc new file mode 100644 index 0000000000000..6fa82dc2a810c --- /dev/null +++ b/docs/reference/data-streams/lifecycle/apis/get-lifecycle-stats.asciidoc @@ -0,0 +1,93 @@ +[[data-streams-get-lifecycle-stats]] +=== Get data stream lifecycle stats +++++ +Get Data Stream Lifecycle +++++ + +preview::[] + +Gets stats about the execution of data stream lifecycle. + +[[get-lifecycle-stats-api-prereqs]] +==== {api-prereq-title} + +* If the {es} {security-features} are enabled, you must have the `monitor` or +`manage` <> to use this API. + +[[data-streams-get-lifecycle-stats-request]] +==== {api-request-title} + +`GET _lifecycle/stats` + +[[data-streams-get-lifecycle-stats-desc]] +==== {api-description-title} + +Gets stats about the execution of the data stream lifecycle. The data stream level stats include only stats about data streams +managed by the data stream lifecycle. + +[[get-lifecycle-stats-api-response-body]] +==== {api-response-body-title} + +`last_run_duration_in_millis`:: +(Optional, long) +The duration of the last data stream lifecycle execution. +`time_between_starts_in_millis`:: +(Optional, long) +The time passed between the start of the last two data stream lifecycle executions. This should amount approximately to +<>. +`data_stream_count`:: +(integer) +The count of data streams currently being managed by the data stream lifecycle. +`data_streams`:: +(array of objects) +Contains information about the retrieved data stream lifecycles. ++ +.Properties of objects in `data_streams` +[%collapsible%open] +==== +`name`:: +(string) +The name of the data stream. +`backing_indices_in_total`:: +(integer) +The count of the backing indices of this data stream that are managed by the data stream lifecycle. +`backing_indices_in_error`:: +(integer) +The count of the backing indices of this data stream that are managed by the data stream lifecycle and have encountered an error. +==== + +[[data-streams-get-lifecycle-stats-example]] +==== {api-examples-title} + +Let's retrieve the data stream lifecycle stats of a cluster that has already executed the lifecycle more than once: + +[source,console] +-------------------------------------------------- +GET _lifecycle/stats?human&pretty +-------------------------------------------------- +// TEST[skip:this is for demonstration purposes only, we cannot ensure that DSL has run] + +The response will look like the following: + +[source,console-result] +-------------------------------------------------- +{ + "last_run_duration_in_millis": 2, + "last_run_duration": "2ms", + "time_between_starts_in_millis": 9998, + "time_between_starts": "9.99s", + "data_streams_count": 2, + "data_streams": [ + { + "name": "my-data-stream", + "backing_indices_in_total": 2, + "backing_indices_in_error": 0 + }, + { + "name": "my-other-stream", + "backing_indices_in_total": 2, + "backing_indices_in_error": 1 + } + ] +} +-------------------------------------------------- \ No newline at end of file diff --git a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceIT.java b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceIT.java index 5bbc007cfb272..7ac86c8aee614 100644 --- a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceIT.java +++ b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceIT.java @@ -622,35 +622,6 @@ public void testDataLifecycleServiceConfiguresTheMergePolicy() throws Exception }); } - private static List getBackingIndices(String dataStreamName) { - GetDataStreamAction.Request getDataStreamRequest = new GetDataStreamAction.Request(new String[] { dataStreamName }); - GetDataStreamAction.Response getDataStreamResponse = client().execute(GetDataStreamAction.INSTANCE, getDataStreamRequest) - .actionGet(); - assertThat(getDataStreamResponse.getDataStreams().size(), equalTo(1)); - assertThat(getDataStreamResponse.getDataStreams().get(0).getDataStream().getName(), equalTo(dataStreamName)); - return getDataStreamResponse.getDataStreams().get(0).getDataStream().getIndices().stream().map(Index::getName).toList(); - } - - static void indexDocs(String dataStream, int numDocs) { - BulkRequest bulkRequest = new BulkRequest(); - for (int i = 0; i < numDocs; i++) { - String value = DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.formatMillis(System.currentTimeMillis()); - bulkRequest.add( - new IndexRequest(dataStream).opType(DocWriteRequest.OpType.CREATE) - .source(String.format(Locale.ROOT, "{\"%s\":\"%s\"}", DEFAULT_TIMESTAMP_FIELD, value), XContentType.JSON) - ); - } - BulkResponse bulkResponse = client().bulk(bulkRequest).actionGet(); - assertThat(bulkResponse.getItems().length, equalTo(numDocs)); - String backingIndexPrefix = DataStream.BACKING_INDEX_PREFIX + dataStream; - for (BulkItemResponse itemResponse : bulkResponse) { - assertThat(itemResponse.getFailureMessage(), nullValue()); - assertThat(itemResponse.status(), equalTo(RestStatus.CREATED)); - assertThat(itemResponse.getIndex(), startsWith(backingIndexPrefix)); - } - indicesAdmin().refresh(new RefreshRequest(dataStream)).actionGet(); - } - public void testReenableDataStreamLifecycle() throws Exception { // start with a lifecycle that's not enabled DataStreamLifecycle lifecycle = new DataStreamLifecycle(null, null, false); @@ -700,6 +671,35 @@ public void testReenableDataStreamLifecycle() throws Exception { }); } + private static List getBackingIndices(String dataStreamName) { + GetDataStreamAction.Request getDataStreamRequest = new GetDataStreamAction.Request(new String[] { dataStreamName }); + GetDataStreamAction.Response getDataStreamResponse = client().execute(GetDataStreamAction.INSTANCE, getDataStreamRequest) + .actionGet(); + assertThat(getDataStreamResponse.getDataStreams().size(), equalTo(1)); + assertThat(getDataStreamResponse.getDataStreams().get(0).getDataStream().getName(), equalTo(dataStreamName)); + return getDataStreamResponse.getDataStreams().get(0).getDataStream().getIndices().stream().map(Index::getName).toList(); + } + + static void indexDocs(String dataStream, int numDocs) { + BulkRequest bulkRequest = new BulkRequest(); + for (int i = 0; i < numDocs; i++) { + String value = DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.formatMillis(System.currentTimeMillis()); + bulkRequest.add( + new IndexRequest(dataStream).opType(DocWriteRequest.OpType.CREATE) + .source(String.format(Locale.ROOT, "{\"%s\":\"%s\"}", DEFAULT_TIMESTAMP_FIELD, value), XContentType.JSON) + ); + } + BulkResponse bulkResponse = client().bulk(bulkRequest).actionGet(); + assertThat(bulkResponse.getItems().length, equalTo(numDocs)); + String backingIndexPrefix = DataStream.BACKING_INDEX_PREFIX + dataStream; + for (BulkItemResponse itemResponse : bulkResponse) { + assertThat(itemResponse.getFailureMessage(), nullValue()); + assertThat(itemResponse.status(), equalTo(RestStatus.CREATED)); + assertThat(itemResponse.getIndex(), startsWith(backingIndexPrefix)); + } + indicesAdmin().refresh(new RefreshRequest(dataStream)).actionGet(); + } + static void putComposableIndexTemplate( String id, @Nullable String mappings, diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleStatsIT.java b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleStatsIT.java new file mode 100644 index 0000000000000..cce9132d99d19 --- /dev/null +++ b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleStatsIT.java @@ -0,0 +1,86 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ +package org.elasticsearch.datastreams.lifecycle; + +import org.elasticsearch.client.Request; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.datastreams.DisabledSecurityDataStreamTestCase; +import org.junit.After; +import org.junit.Before; + +import java.io.IOException; +import java.util.List; +import java.util.Map; + +import static org.hamcrest.Matchers.greaterThanOrEqualTo; +import static org.hamcrest.Matchers.is; + +public class DataStreamLifecycleStatsIT extends DisabledSecurityDataStreamTestCase { + + @Before + public void updateClusterSettings() throws IOException { + updateClusterSettings( + Settings.builder() + .put("data_streams.lifecycle.poll_interval", "1s") + .put("cluster.lifecycle.default.rollover", "min_docs=1,max_docs=1") + .build() + ); + } + + @After + public void cleanUp() throws IOException { + adminClient().performRequest(new Request("DELETE", "_data_stream/*?expand_wildcards=hidden")); + } + + @SuppressWarnings("unchecked") + public void testStats() throws Exception { + // Check empty stats and wait until we have 2 executions + assertBusy(() -> { + Request request = new Request("GET", "/_lifecycle/stats"); + Map response = entityAsMap(client().performRequest(request)); + assertThat(response.get("data_stream_count"), is(0)); + assertThat(response.get("data_streams"), is(List.of())); + assertThat(response.containsKey("last_run_duration_in_millis"), is(true)); + assertThat(response.containsKey("time_between_starts_in_millis"), is(true)); + }); + + // Create a template + Request putComposableIndexTemplateRequest = new Request("POST", "/_index_template/1"); + putComposableIndexTemplateRequest.setJsonEntity(""" + { + "index_patterns": ["my-data-stream-*"], + "data_stream": {}, + "template": { + "lifecycle": {} + } + } + """); + assertOK(client().performRequest(putComposableIndexTemplateRequest)); + + // Create two data streams with one doc each + Request createDocRequest = new Request("POST", "/my-data-stream-1/_doc?refresh=true"); + createDocRequest.setJsonEntity("{ \"@timestamp\": \"2022-12-12\"}"); + assertOK(client().performRequest(createDocRequest)); + createDocRequest = new Request("POST", "/my-data-stream-2/_doc?refresh=true"); + createDocRequest.setJsonEntity("{ \"@timestamp\": \"2022-12-12\"}"); + assertOK(client().performRequest(createDocRequest)); + + Request request = new Request("GET", "/_lifecycle/stats"); + Map response = entityAsMap(client().performRequest(request)); + assertThat(response.get("data_stream_count"), is(2)); + List> dataStreams = (List>) response.get("data_streams"); + assertThat(dataStreams.get(0).get("name"), is("my-data-stream-1")); + assertThat((Integer) dataStreams.get(0).get("backing_indices_in_total"), greaterThanOrEqualTo(1)); + assertThat((Integer) dataStreams.get(0).get("backing_indices_in_error"), is(0)); + assertThat(dataStreams.get(1).get("name"), is("my-data-stream-2")); + assertThat((Integer) dataStreams.get(1).get("backing_indices_in_total"), greaterThanOrEqualTo(1)); + assertThat((Integer) dataStreams.get(0).get("backing_indices_in_error"), is(0)); + assertThat(response.containsKey("last_run_duration_in_millis"), is(true)); + assertThat(response.containsKey("time_between_starts_in_millis"), is(true)); + } +} diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamsPlugin.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamsPlugin.java index 2cf44dc0e3218..dd8e13cf18408 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamsPlugin.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamsPlugin.java @@ -40,11 +40,14 @@ import org.elasticsearch.datastreams.lifecycle.action.DeleteDataStreamLifecycleAction; import org.elasticsearch.datastreams.lifecycle.action.ExplainDataStreamLifecycleAction; import org.elasticsearch.datastreams.lifecycle.action.GetDataStreamLifecycleAction; +import org.elasticsearch.datastreams.lifecycle.action.GetDataStreamLifecycleStatsAction; import org.elasticsearch.datastreams.lifecycle.action.PutDataStreamLifecycleAction; import org.elasticsearch.datastreams.lifecycle.action.TransportDeleteDataStreamLifecycleAction; import org.elasticsearch.datastreams.lifecycle.action.TransportExplainDataStreamLifecycleAction; import org.elasticsearch.datastreams.lifecycle.action.TransportGetDataStreamLifecycleAction; +import org.elasticsearch.datastreams.lifecycle.action.TransportGetDataStreamLifecycleStatsAction; import org.elasticsearch.datastreams.lifecycle.action.TransportPutDataStreamLifecycleAction; +import org.elasticsearch.datastreams.lifecycle.rest.RestDataStreamLifecycleStatsAction; import org.elasticsearch.datastreams.lifecycle.rest.RestDeleteDataStreamLifecycleAction; import org.elasticsearch.datastreams.lifecycle.rest.RestExplainDataStreamLifecycleAction; import org.elasticsearch.datastreams.lifecycle.rest.RestGetDataStreamLifecycleAction; @@ -189,6 +192,7 @@ public Collection createComponents(PluginServices services) { actions.add(new ActionHandler<>(GetDataStreamLifecycleAction.INSTANCE, TransportGetDataStreamLifecycleAction.class)); actions.add(new ActionHandler<>(DeleteDataStreamLifecycleAction.INSTANCE, TransportDeleteDataStreamLifecycleAction.class)); actions.add(new ActionHandler<>(ExplainDataStreamLifecycleAction.INSTANCE, TransportExplainDataStreamLifecycleAction.class)); + actions.add(new ActionHandler<>(GetDataStreamLifecycleStatsAction.INSTANCE, TransportGetDataStreamLifecycleStatsAction.class)); return actions; } @@ -218,6 +222,7 @@ public List getRestHandlers( handlers.add(new RestGetDataStreamLifecycleAction()); handlers.add(new RestDeleteDataStreamLifecycleAction()); handlers.add(new RestExplainDataStreamLifecycleAction()); + handlers.add(new RestDataStreamLifecycleStatsAction()); return handlers; } diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleErrorStore.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleErrorStore.java index 47589fd7276f4..01ccbdbe3ffec 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleErrorStore.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleErrorStore.java @@ -13,7 +13,7 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.core.Nullable; -import java.util.List; +import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.function.LongSupplier; @@ -87,7 +87,7 @@ public ErrorEntry getError(String indexName) { /** * Return an immutable view (a snapshot) of the tracked indices at the moment this method is called. */ - public List getAllIndices() { - return List.copyOf(indexNameToError.keySet()); + public Set getAllIndices() { + return Set.copyOf(indexNameToError.keySet()); } } diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleService.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleService.java index 03d1340c14dbb..9f9a90704167d 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleService.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleService.java @@ -175,6 +175,13 @@ public class DataStreamLifecycleService implements ClusterStateListener, Closeab */ private volatile int signallingErrorRetryInterval; + /** + * The following stats are tracking how the data stream lifecycle runs are performing time wise + */ + private volatile Long lastRunStartedAt = null; + private volatile Long lastRunDuration = null; + private volatile Long timeBetweenStarts = null; + private static final SimpleBatchedExecutor FORCE_MERGE_STATE_UPDATE_TASK_EXECUTOR = new SimpleBatchedExecutor<>() { @Override @@ -299,6 +306,11 @@ public void triggered(SchedulerEngine.Event event) { */ // default visibility for testing purposes void run(ClusterState state) { + long startTime = nowSupplier.getAsLong(); + if (lastRunStartedAt != null) { + timeBetweenStarts = startTime - lastRunStartedAt; + } + lastRunStartedAt = startTime; int affectedIndices = 0; int affectedDataStreams = 0; for (DataStream dataStream : state.metadata().dataStreams().values()) { @@ -396,8 +408,10 @@ void run(ClusterState state) { affectedIndices += indicesToExcludeForRemainingRun.size(); affectedDataStreams++; } + lastRunDuration = nowSupplier.getAsLong() - lastRunStartedAt; logger.trace( - "Data stream lifecycle service performed operations on [{}] indices, part of [{}] data streams", + "Data stream lifecycle service run for {} and performed operations on [{}] indices, part of [{}] data streams", + TimeValue.timeValueMillis(lastRunDuration).toHumanReadableString(2), affectedIndices, affectedDataStreams ); @@ -1193,6 +1207,22 @@ static TimeValue getRetentionConfiguration(DataStream dataStream) { return dataStream.getLifecycle().getEffectiveDataRetention(); } + /** + * @return the duration of the last run in millis or null if the service hasn't completed a run yet. + */ + @Nullable + public Long getLastRunDuration() { + return lastRunDuration; + } + + /** + * @return the time passed between the start times of the last two consecutive runs or null if the service hasn't started twice yet. + */ + @Nullable + public Long getTimeBetweenStarts() { + return timeBetweenStarts; + } + /** * Action listener that records the encountered failure using the provided recordError callback for the * provided target index. If the listener is notified of success it will clear the recorded entry for the provided diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/GetDataStreamLifecycleStatsAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/GetDataStreamLifecycleStatsAction.java new file mode 100644 index 0000000000000..c3444a67b847c --- /dev/null +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/GetDataStreamLifecycleStatsAction.java @@ -0,0 +1,154 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ +package org.elasticsearch.datastreams.lifecycle.action; + +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.action.ActionType; +import org.elasticsearch.action.support.master.MasterNodeReadRequest; +import org.elasticsearch.common.collect.Iterators; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.ChunkedToXContentObject; +import org.elasticsearch.core.Nullable; +import org.elasticsearch.core.TimeValue; +import org.elasticsearch.xcontent.ToXContent; + +import java.io.IOException; +import java.util.Iterator; +import java.util.List; +import java.util.Objects; + +/** + * This action retrieves the data stream lifecycle stats from the master node. + */ +public class GetDataStreamLifecycleStatsAction extends ActionType { + + public static final GetDataStreamLifecycleStatsAction INSTANCE = new GetDataStreamLifecycleStatsAction(); + public static final String NAME = "cluster:monitor/data_stream/lifecycle/stats"; + + private GetDataStreamLifecycleStatsAction() { + super(NAME, Response::new); + } + + public static class Request extends MasterNodeReadRequest { + + public Request(StreamInput in) throws IOException { + super(in); + } + + public Request() {} + + @Override + public ActionRequestValidationException validate() { + return null; + } + } + + public static class Response extends ActionResponse implements ChunkedToXContentObject { + + private final Long runDuration; + private final Long timeBetweenStarts; + private final List dataStreamStats; + + public Response(@Nullable Long runDuration, @Nullable Long timeBetweenStarts, List dataStreamStats) { + this.runDuration = runDuration; + this.timeBetweenStarts = timeBetweenStarts; + this.dataStreamStats = dataStreamStats; + } + + public Response(StreamInput in) throws IOException { + super(in); + this.runDuration = in.readOptionalVLong(); + this.timeBetweenStarts = in.readOptionalVLong(); + this.dataStreamStats = in.readCollectionAsImmutableList(DataStreamStats::read); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeOptionalVLong(runDuration); + out.writeOptionalVLong(timeBetweenStarts); + out.writeCollection(dataStreamStats, (o, v) -> v.writeTo(o)); + } + + public Long getRunDuration() { + return runDuration; + } + + public Long getTimeBetweenStarts() { + return timeBetweenStarts; + } + + public List getDataStreamStats() { + return dataStreamStats; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Response other = (Response) o; + return Objects.equals(runDuration, other.runDuration) + && Objects.equals(timeBetweenStarts, other.timeBetweenStarts) + && Objects.equals(dataStreamStats, other.dataStreamStats); + } + + @Override + public int hashCode() { + return Objects.hash(runDuration, timeBetweenStarts, dataStreamStats); + } + + @Override + public Iterator toXContentChunked(ToXContent.Params outerParams) { + return Iterators.concat(Iterators.single((builder, params) -> { + builder.startObject(); + if (runDuration != null) { + builder.field("last_run_duration_in_millis", runDuration); + if (builder.humanReadable()) { + builder.field("last_run_duration", TimeValue.timeValueMillis(runDuration).toHumanReadableString(2)); + } + } + if (timeBetweenStarts != null) { + builder.field("time_between_starts_in_millis", timeBetweenStarts); + if (builder.humanReadable()) { + builder.field("time_between_starts", TimeValue.timeValueMillis(timeBetweenStarts).toHumanReadableString(2)); + } + } + builder.field("data_stream_count", dataStreamStats.size()); + builder.startArray("data_streams"); + return builder; + }), Iterators.map(dataStreamStats.iterator(), stat -> (builder, params) -> { + builder.startObject(); + builder.field("name", stat.dataStreamName); + builder.field("backing_indices_in_total", stat.backingIndicesInTotal); + builder.field("backing_indices_in_error", stat.backingIndicesInError); + builder.endObject(); + return builder; + }), Iterators.single((builder, params) -> { + builder.endArray(); + builder.endObject(); + return builder; + })); + } + + public record DataStreamStats(String dataStreamName, int backingIndicesInTotal, int backingIndicesInError) implements Writeable { + + public static DataStreamStats read(StreamInput in) throws IOException { + return new DataStreamStats(in.readString(), in.readVInt(), in.readVInt()); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(dataStreamName); + out.writeVInt(backingIndicesInTotal); + out.writeVInt(backingIndicesInError); + } + } + } +} diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/TransportGetDataStreamLifecycleStatsAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/TransportGetDataStreamLifecycleStatsAction.java new file mode 100644 index 0000000000000..03bc1d129eaba --- /dev/null +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/TransportGetDataStreamLifecycleStatsAction.java @@ -0,0 +1,110 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ +package org.elasticsearch.datastreams.lifecycle.action; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.master.TransportMasterNodeReadAction; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.block.ClusterBlockException; +import org.elasticsearch.cluster.block.ClusterBlockLevel; +import org.elasticsearch.cluster.metadata.DataStream; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.metadata.Metadata; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.util.concurrent.EsExecutors; +import org.elasticsearch.datastreams.lifecycle.DataStreamLifecycleService; +import org.elasticsearch.index.Index; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportService; + +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; +import java.util.Set; + +/** + * Exposes stats about the latest lifecycle run and the error store. + */ +public class TransportGetDataStreamLifecycleStatsAction extends TransportMasterNodeReadAction< + GetDataStreamLifecycleStatsAction.Request, + GetDataStreamLifecycleStatsAction.Response> { + + private final DataStreamLifecycleService lifecycleService; + + @Inject + public TransportGetDataStreamLifecycleStatsAction( + TransportService transportService, + ClusterService clusterService, + ThreadPool threadPool, + ActionFilters actionFilters, + IndexNameExpressionResolver indexNameExpressionResolver, + DataStreamLifecycleService lifecycleService + ) { + super( + GetDataStreamLifecycleStatsAction.NAME, + transportService, + clusterService, + threadPool, + actionFilters, + GetDataStreamLifecycleStatsAction.Request::new, + indexNameExpressionResolver, + GetDataStreamLifecycleStatsAction.Response::new, + EsExecutors.DIRECT_EXECUTOR_SERVICE + ); + this.lifecycleService = lifecycleService; + } + + @Override + protected void masterOperation( + Task task, + GetDataStreamLifecycleStatsAction.Request request, + ClusterState state, + ActionListener listener + ) throws Exception { + listener.onResponse(collectStats(state)); + } + + // Visible for testing + GetDataStreamLifecycleStatsAction.Response collectStats(ClusterState state) { + Metadata metadata = state.metadata(); + Set indicesInErrorStore = lifecycleService.getErrorStore().getAllIndices(); + List dataStreamStats = new ArrayList<>(); + for (DataStream dataStream : state.metadata().dataStreams().values()) { + if (dataStream.getLifecycle() != null && dataStream.getLifecycle().isEnabled()) { + int total = 0; + int inError = 0; + for (Index index : dataStream.getIndices()) { + if (dataStream.isIndexManagedByDataStreamLifecycle(index, metadata::index)) { + total++; + if (indicesInErrorStore.contains(index.getName())) { + inError++; + } + } + } + dataStreamStats.add(new GetDataStreamLifecycleStatsAction.Response.DataStreamStats(dataStream.getName(), total, inError)); + } + } + return new GetDataStreamLifecycleStatsAction.Response( + lifecycleService.getLastRunDuration(), + lifecycleService.getTimeBetweenStarts(), + dataStreamStats.isEmpty() + ? dataStreamStats + : dataStreamStats.stream() + .sorted(Comparator.comparing(GetDataStreamLifecycleStatsAction.Response.DataStreamStats::dataStreamName)) + .toList() + ); + } + + @Override + protected ClusterBlockException checkBlock(GetDataStreamLifecycleStatsAction.Request request, ClusterState state) { + return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_READ); + } +} diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/rest/RestDataStreamLifecycleStatsAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/rest/RestDataStreamLifecycleStatsAction.java new file mode 100644 index 0000000000000..2daff2a05940c --- /dev/null +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/rest/RestDataStreamLifecycleStatsAction.java @@ -0,0 +1,45 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.datastreams.lifecycle.rest; + +import org.elasticsearch.client.internal.node.NodeClient; +import org.elasticsearch.datastreams.lifecycle.action.GetDataStreamLifecycleStatsAction; +import org.elasticsearch.rest.BaseRestHandler; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.Scope; +import org.elasticsearch.rest.ServerlessScope; +import org.elasticsearch.rest.action.RestChunkedToXContentListener; + +import java.util.List; + +import static org.elasticsearch.rest.RestRequest.Method.GET; + +@ServerlessScope(Scope.PUBLIC) +public class RestDataStreamLifecycleStatsAction extends BaseRestHandler { + + @Override + public String getName() { + return "data_stream_lifecycle_stats_action"; + } + + @Override + public List routes() { + return List.of(new Route(GET, "/_lifecycle/stats")); + } + + @Override + protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) { + String masterNodeTimeout = restRequest.param("master_timeout"); + GetDataStreamLifecycleStatsAction.Request request = new GetDataStreamLifecycleStatsAction.Request(); + if (masterNodeTimeout != null) { + request.masterNodeTimeout(masterNodeTimeout); + } + return channel -> client.execute(GetDataStreamLifecycleStatsAction.INSTANCE, request, new RestChunkedToXContentListener<>(channel)); + } +} diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleErrorStoreTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleErrorStoreTests.java index c1255cc9e3a72..9f1928374eb5f 100644 --- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleErrorStoreTests.java +++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleErrorStoreTests.java @@ -12,12 +12,13 @@ import org.elasticsearch.test.ESTestCase; import org.junit.Before; -import java.util.List; +import java.util.Set; import java.util.stream.Stream; import static org.elasticsearch.datastreams.lifecycle.DataStreamLifecycleErrorStore.MAX_ERROR_MESSAGE_LENGTH; import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.hasItem; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.nullValue; @@ -36,7 +37,7 @@ public void testRecordAndRetrieveError() { assertThat(existingRecordedError, is(nullValue())); assertThat(errorStore.getError("test"), is(notNullValue())); assertThat(errorStore.getAllIndices().size(), is(1)); - assertThat(errorStore.getAllIndices().get(0), is("test")); + assertThat(errorStore.getAllIndices(), hasItem("test")); existingRecordedError = errorStore.recordError("test", new IllegalStateException("bad state")); assertThat(existingRecordedError, is(notNullValue())); @@ -51,7 +52,7 @@ public void testRetrieveAfterClear() { public void testGetAllIndicesIsASnapshotViewOfTheStore() { Stream.iterate(0, i -> i + 1).limit(5).forEach(i -> errorStore.recordError("test" + i, new NullPointerException("testing"))); - List initialAllIndices = errorStore.getAllIndices(); + Set initialAllIndices = errorStore.getAllIndices(); assertThat(initialAllIndices.size(), is(5)); assertThat( initialAllIndices, diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceTests.java index 6e22f835c397e..2445e6b0d72df 100644 --- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceTests.java +++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceTests.java @@ -94,6 +94,7 @@ import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; import static org.elasticsearch.cluster.metadata.IndexMetadata.APIBlock.WRITE; @@ -119,6 +120,7 @@ import static org.hamcrest.Matchers.lessThanOrEqualTo; import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.nullValue; +import static org.mockito.Mockito.mock; public class DataStreamLifecycleServiceTests extends ESTestCase { @@ -1378,6 +1380,31 @@ public void testTimeSeriesIndicesStillWithinTimeBounds() { } } + public void testTrackingTimeStats() { + AtomicLong now = new AtomicLong(0); + long delta = randomLongBetween(10, 10000); + DataStreamLifecycleService service = new DataStreamLifecycleService( + Settings.EMPTY, + getTransportRequestsRecordingClient(), + clusterService, + Clock.systemUTC(), + threadPool, + () -> now.getAndAdd(delta), + new DataStreamLifecycleErrorStore(() -> Clock.systemUTC().millis()), + mock(AllocationService.class) + ); + assertThat(service.getLastRunDuration(), is(nullValue())); + assertThat(service.getTimeBetweenStarts(), is(nullValue())); + + service.run(ClusterState.EMPTY_STATE); + assertThat(service.getLastRunDuration(), is(delta)); + assertThat(service.getTimeBetweenStarts(), is(nullValue())); + + service.run(ClusterState.EMPTY_STATE); + assertThat(service.getLastRunDuration(), is(delta)); + assertThat(service.getTimeBetweenStarts(), is(2 * delta)); + } + /* * Creates a test cluster state with the given indexName. If customDataStreamLifecycleMetadata is not null, it is added as the value * of the index's custom metadata named "data_stream_lifecycle". diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/action/DataStreamLifecycleStatsResponseTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/action/DataStreamLifecycleStatsResponseTests.java new file mode 100644 index 0000000000000..b8e4b252645dd --- /dev/null +++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/action/DataStreamLifecycleStatsResponseTests.java @@ -0,0 +1,154 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.datastreams.lifecycle.action; + +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.core.TimeValue; +import org.elasticsearch.test.AbstractWireSerializingTestCase; +import org.elasticsearch.xcontent.ToXContent; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.XContentType; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.stream.IntStream; + +import static org.elasticsearch.xcontent.ToXContent.EMPTY_PARAMS; +import static org.hamcrest.Matchers.is; + +public class DataStreamLifecycleStatsResponseTests extends AbstractWireSerializingTestCase { + + @Override + protected GetDataStreamLifecycleStatsAction.Response createTestInstance() { + boolean hasRun = usually(); + var runDuration = hasRun ? randomLongBetween(10, 100000000) : null; + var timeBetweenStarts = hasRun && usually() ? randomLongBetween(10, 100000000) : null; + var dataStreams = IntStream.range(0, randomInt(10)) + .mapToObj( + ignored -> new GetDataStreamLifecycleStatsAction.Response.DataStreamStats( + randomAlphaOfLength(10), + randomIntBetween(1, 1000), + randomIntBetween(0, 100) + ) + ) + .toList(); + return new GetDataStreamLifecycleStatsAction.Response(runDuration, timeBetweenStarts, dataStreams); + } + + @Override + protected GetDataStreamLifecycleStatsAction.Response mutateInstance(GetDataStreamLifecycleStatsAction.Response instance) { + var runDuration = instance.getRunDuration(); + var timeBetweenStarts = instance.getTimeBetweenStarts(); + var dataStreams = instance.getDataStreamStats(); + switch (randomInt(2)) { + case 0 -> runDuration = runDuration != null && randomBoolean() + ? null + : randomValueOtherThan(runDuration, () -> randomLongBetween(10, 100000000)); + case 1 -> timeBetweenStarts = timeBetweenStarts != null && randomBoolean() + ? null + : randomValueOtherThan(timeBetweenStarts, () -> randomLongBetween(10, 100000000)); + default -> dataStreams = mutateDataStreamStats(dataStreams); + } + return new GetDataStreamLifecycleStatsAction.Response(runDuration, timeBetweenStarts, dataStreams); + } + + private List mutateDataStreamStats( + List dataStreamStats + ) { + // change the stats of a data stream + List mutated = new ArrayList<>(dataStreamStats); + if (randomBoolean() && dataStreamStats.isEmpty() == false) { + int i = randomInt(dataStreamStats.size() - 1); + GetDataStreamLifecycleStatsAction.Response.DataStreamStats instance = dataStreamStats.get(i); + mutated.set(i, switch (randomInt(2)) { + case 0 -> new GetDataStreamLifecycleStatsAction.Response.DataStreamStats( + instance.dataStreamName() + randomAlphaOfLength(2), + instance.backingIndicesInTotal(), + instance.backingIndicesInError() + ); + case 1 -> new GetDataStreamLifecycleStatsAction.Response.DataStreamStats( + instance.dataStreamName(), + instance.backingIndicesInTotal() + randomIntBetween(1, 10), + instance.backingIndicesInError() + ); + default -> new GetDataStreamLifecycleStatsAction.Response.DataStreamStats( + instance.dataStreamName(), + instance.backingIndicesInTotal(), + instance.backingIndicesInError() + randomIntBetween(1, 10) + ); + + }); + } else if (dataStreamStats.isEmpty() || randomBoolean()) { + mutated.add( + new GetDataStreamLifecycleStatsAction.Response.DataStreamStats( + randomAlphaOfLength(10), + randomIntBetween(1, 1000), + randomIntBetween(0, 100) + ) + ); + } else { + mutated.remove(randomInt(dataStreamStats.size() - 1)); + } + return mutated; + } + + @SuppressWarnings("unchecked") + public void testXContentSerialization() throws IOException { + GetDataStreamLifecycleStatsAction.Response testInstance = createTestInstance(); + try (XContentBuilder builder = XContentBuilder.builder(XContentType.JSON.xContent())) { + builder.humanReadable(true); + testInstance.toXContentChunked(ToXContent.EMPTY_PARAMS).forEachRemaining(xcontent -> { + try { + xcontent.toXContent(builder, EMPTY_PARAMS); + } catch (IOException e) { + logger.error(e.getMessage(), e); + fail(e.getMessage()); + } + }); + Map xContentMap = XContentHelper.convertToMap(BytesReference.bytes(builder), false, builder.contentType()).v2(); + assertThat(xContentMap.get("last_run_duration_in_millis"), is(testInstance.getRunDuration().intValue())); + assertThat( + xContentMap.get("last_run_duration"), + is(TimeValue.timeValueMillis(testInstance.getRunDuration()).toHumanReadableString(2)) + ); + assertThat(xContentMap.get("time_between_starts_in_millis"), is(testInstance.getTimeBetweenStarts().intValue())); + assertThat( + xContentMap.get("time_between_starts"), + is(TimeValue.timeValueMillis(testInstance.getTimeBetweenStarts()).toHumanReadableString(2)) + ); + assertThat(xContentMap.get("data_stream_count"), is(testInstance.getDataStreamStats().size())); + List> dataStreams = (List>) xContentMap.get("data_streams"); + if (testInstance.getDataStreamStats().isEmpty()) { + assertThat(dataStreams.isEmpty(), is(true)); + } else { + assertThat(dataStreams.size(), is(testInstance.getDataStreamStats().size())); + for (int i = 0; i < dataStreams.size(); i++) { + assertThat(dataStreams.get(i).get("name"), is(testInstance.getDataStreamStats().get(i).dataStreamName())); + assertThat( + dataStreams.get(i).get("backing_indices_in_total"), + is(testInstance.getDataStreamStats().get(i).backingIndicesInTotal()) + ); + assertThat( + dataStreams.get(i).get("backing_indices_in_error"), + is(testInstance.getDataStreamStats().get(i).backingIndicesInError()) + ); + } + } + } + } + + @Override + protected Writeable.Reader instanceReader() { + return GetDataStreamLifecycleStatsAction.Response::new; + } +} diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/action/TransportGetDataStreamLifecycleStatsActionTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/action/TransportGetDataStreamLifecycleStatsActionTests.java new file mode 100644 index 0000000000000..8c423107ea2f4 --- /dev/null +++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/action/TransportGetDataStreamLifecycleStatsActionTests.java @@ -0,0 +1,153 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.datastreams.lifecycle.action; + +import org.elasticsearch.action.admin.indices.rollover.MaxAgeCondition; +import org.elasticsearch.action.admin.indices.rollover.RolloverInfo; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.cluster.ClusterName; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.metadata.DataStream; +import org.elasticsearch.cluster.metadata.DataStreamLifecycle; +import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.metadata.Metadata; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.core.TimeValue; +import org.elasticsearch.datastreams.lifecycle.DataStreamLifecycleErrorStore; +import org.elasticsearch.datastreams.lifecycle.DataStreamLifecycleService; +import org.elasticsearch.index.Index; +import org.elasticsearch.index.IndexVersion; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportService; +import org.junit.Before; + +import java.time.Clock; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import static org.elasticsearch.cluster.metadata.DataStreamTestHelper.newInstance; +import static org.elasticsearch.datastreams.lifecycle.DataStreamLifecycleFixtures.createDataStream; +import static org.hamcrest.Matchers.is; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class TransportGetDataStreamLifecycleStatsActionTests extends ESTestCase { + + private final DataStreamLifecycleService dataStreamLifecycleService = mock(DataStreamLifecycleService.class); + private final DataStreamLifecycleErrorStore errorStore = mock(DataStreamLifecycleErrorStore.class); + private final TransportGetDataStreamLifecycleStatsAction action = new TransportGetDataStreamLifecycleStatsAction( + mock(TransportService.class), + mock(ClusterService.class), + mock(ThreadPool.class), + mock(ActionFilters.class), + mock(IndexNameExpressionResolver.class), + dataStreamLifecycleService + ); + private Long lastRunDuration; + private Long timeBetweenStarts; + + @Before + public void setUp() throws Exception { + super.setUp(); + lastRunDuration = randomBoolean() ? randomLongBetween(0, 100000) : null; + timeBetweenStarts = randomBoolean() ? randomLongBetween(0, 100000) : null; + when(dataStreamLifecycleService.getLastRunDuration()).thenReturn(lastRunDuration); + when(dataStreamLifecycleService.getTimeBetweenStarts()).thenReturn(timeBetweenStarts); + when(dataStreamLifecycleService.getErrorStore()).thenReturn(errorStore); + when(errorStore.getAllIndices()).thenReturn(Set.of()); + } + + public void testEmptyClusterState() { + GetDataStreamLifecycleStatsAction.Response response = action.collectStats(ClusterState.EMPTY_STATE); + assertThat(response.getRunDuration(), is(lastRunDuration)); + assertThat(response.getTimeBetweenStarts(), is(timeBetweenStarts)); + assertThat(response.getDataStreamStats().isEmpty(), is(true)); + } + + public void testMixedDataStreams() { + Set indicesInError = new HashSet<>(); + int numBackingIndices = 3; + Metadata.Builder builder = Metadata.builder(); + DataStream ilmDataStream = createDataStream( + builder, + "ilm-managed-index", + numBackingIndices, + Settings.builder() + .put(IndexMetadata.LIFECYCLE_NAME, "ILM_policy") + .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current()), + null, + Clock.systemUTC().millis() + ); + builder.put(ilmDataStream); + DataStream dslDataStream = createDataStream( + builder, + "dsl-managed-index", + numBackingIndices, + settings(IndexVersion.current()), + DataStreamLifecycle.newBuilder().dataRetention(TimeValue.timeValueDays(10)).build(), + Clock.systemUTC().millis() + ); + indicesInError.add(dslDataStream.getIndices().get(randomInt(numBackingIndices - 1)).getName()); + builder.put(dslDataStream); + { + String dataStreamName = "mixed"; + final List backingIndices = new ArrayList<>(); + for (int k = 1; k <= 2; k++) { + IndexMetadata.Builder indexMetaBuilder = IndexMetadata.builder(DataStream.getDefaultBackingIndexName(dataStreamName, k)) + .settings( + Settings.builder() + .put(IndexMetadata.LIFECYCLE_NAME, "ILM_policy") + .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current()) + ) + .numberOfShards(1) + .numberOfReplicas(1) + .creationDate(Clock.systemUTC().millis()); + + IndexMetadata indexMetadata = indexMetaBuilder.build(); + builder.put(indexMetadata, false); + backingIndices.add(indexMetadata.getIndex()); + } + // DSL managed write index + IndexMetadata.Builder indexMetaBuilder = IndexMetadata.builder(DataStream.getDefaultBackingIndexName(dataStreamName, 3)) + .settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current())) + .numberOfShards(1) + .numberOfReplicas(1) + .creationDate(Clock.systemUTC().millis()); + MaxAgeCondition rolloverCondition = new MaxAgeCondition(TimeValue.timeValueMillis(Clock.systemUTC().millis() - 2000L)); + indexMetaBuilder.putRolloverInfo( + new RolloverInfo(dataStreamName, List.of(rolloverCondition), Clock.systemUTC().millis() - 2000L) + ); + IndexMetadata indexMetadata = indexMetaBuilder.build(); + builder.put(indexMetadata, false); + backingIndices.add(indexMetadata.getIndex()); + builder.put(newInstance(dataStreamName, backingIndices, 3, null, false, DataStreamLifecycle.newBuilder().build())); + } + ClusterState state = ClusterState.builder(ClusterName.DEFAULT).metadata(builder).build(); + when(errorStore.getAllIndices()).thenReturn(indicesInError); + GetDataStreamLifecycleStatsAction.Response response = action.collectStats(state); + assertThat(response.getRunDuration(), is(lastRunDuration)); + assertThat(response.getTimeBetweenStarts(), is(timeBetweenStarts)); + assertThat(response.getDataStreamStats().size(), is(2)); + for (GetDataStreamLifecycleStatsAction.Response.DataStreamStats stats : response.getDataStreamStats()) { + if (stats.dataStreamName().equals("dsl-managed-index")) { + assertThat(stats.backingIndicesInTotal(), is(3)); + assertThat(stats.backingIndicesInError(), is(1)); + } + if (stats.dataStreamName().equals("mixed")) { + assertThat(stats.backingIndicesInTotal(), is(1)); + assertThat(stats.backingIndicesInError(), is(0)); + } + } + } +} diff --git a/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java b/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java index b77fc542a7af4..7a65a03277faf 100644 --- a/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java +++ b/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java @@ -294,6 +294,7 @@ public class Constants { "cluster:monitor/ccr/follow_info", "cluster:monitor/ccr/follow_stats", "cluster:monitor/ccr/stats", + "cluster:monitor/data_stream/lifecycle/stats", "cluster:monitor/eql/async/status", "cluster:monitor/fetch/health/info", "cluster:monitor/health", diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/DataStreamLifecycleDownsamplingSecurityIT.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/DataStreamLifecycleDownsamplingSecurityIT.java index 7cdc91b83afaf..1c696ffb9dd31 100644 --- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/DataStreamLifecycleDownsamplingSecurityIT.java +++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/DataStreamLifecycleDownsamplingSecurityIT.java @@ -244,7 +244,7 @@ private Map collectErrorsFromStoreAsMap() { Map indicesAndErrors = new HashMap<>(); for (DataStreamLifecycleService lifecycleService : lifecycleServices) { DataStreamLifecycleErrorStore errorStore = lifecycleService.getErrorStore(); - List allIndices = errorStore.getAllIndices(); + Set allIndices = errorStore.getAllIndices(); for (var index : allIndices) { ErrorEntry error = errorStore.getError(index); if (error != null) { diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/DataStreamLifecycleServiceRuntimeSecurityIT.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/DataStreamLifecycleServiceRuntimeSecurityIT.java index fbb2832461a7c..f5349cac99ed7 100644 --- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/DataStreamLifecycleServiceRuntimeSecurityIT.java +++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/DataStreamLifecycleServiceRuntimeSecurityIT.java @@ -48,6 +48,7 @@ import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.Set; import java.util.concurrent.ExecutionException; import static org.elasticsearch.cluster.metadata.DataStreamTestHelper.backingIndexEqualTo; @@ -168,7 +169,7 @@ private Map collectErrorsFromStoreAsMap() { Map indicesAndErrors = new HashMap<>(); for (DataStreamLifecycleService lifecycleService : lifecycleServices) { DataStreamLifecycleErrorStore errorStore = lifecycleService.getErrorStore(); - List allIndices = errorStore.getAllIndices(); + Set allIndices = errorStore.getAllIndices(); for (var index : allIndices) { ErrorEntry error = errorStore.getError(index); if (error != null) { From a5448ec2f1e0fdf98c314dea2b61c4747a953d50 Mon Sep 17 00:00:00 2001 From: David Turner Date: Mon, 20 Nov 2023 09:02:57 +0000 Subject: [PATCH 10/28] Improve suppressed exceptions in CloseFollowerIndexIT (#102346) This whole thing is trappy, but even more so that it doesn't show up in the logs or complete any normal cleanup steps. This commit adds logging and rethrows the expected `AssertionError` on a different thread to fix the missing cleanup. --- .../elasticsearch/index/shard/IndexShard.java | 18 +++++++++++++++++- ...oseFollowerIndexErrorSuppressionHelper.java | 14 ++++++++++++++ .../xpack/ccr/CloseFollowerIndexIT.java | 7 ++++++- 3 files changed, 37 insertions(+), 2 deletions(-) create mode 100644 x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/index/shard/CloseFollowerIndexErrorSuppressionHelper.java diff --git a/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java b/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java index 8b6f6afb72042..fedd84ee7392b 100644 --- a/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java +++ b/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java @@ -2006,7 +2006,7 @@ private void innerOpenEngineAndTranslog(LongSupplier globalCheckpointSupplier) t assert currentEngineReference.get() == null : "engine is running"; verifyNotClosed(); // we must create a new engine under mutex (see IndexShard#snapshotStoreMetadata). - final Engine newEngine = engineFactory.newReadWriteEngine(config); + final Engine newEngine = createEngine(config); onNewEngine(newEngine); currentEngineReference.set(newEngine); // We set active because we are now writing operations to the engine; this way, @@ -2021,6 +2021,22 @@ private void innerOpenEngineAndTranslog(LongSupplier globalCheckpointSupplier) t checkAndCallWaitForEngineOrClosedShardListeners(); } + // awful hack to work around problem in CloseFollowerIndexIT + static boolean suppressCreateEngineErrors; + + private Engine createEngine(EngineConfig config) { + if (suppressCreateEngineErrors) { + try { + return engineFactory.newReadWriteEngine(config); + } catch (Error e) { + ExceptionsHelper.maybeDieOnAnotherThread(e); + throw new RuntimeException("rethrowing suppressed error", e); + } + } else { + return engineFactory.newReadWriteEngine(config); + } + } + private boolean assertSequenceNumbersInCommit() throws IOException { final SegmentInfos segmentCommitInfos = SegmentInfos.readLatestCommit(store.directory()); final Map userData = segmentCommitInfos.getUserData(); diff --git a/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/index/shard/CloseFollowerIndexErrorSuppressionHelper.java b/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/index/shard/CloseFollowerIndexErrorSuppressionHelper.java new file mode 100644 index 0000000000000..89ba41317e0e3 --- /dev/null +++ b/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/index/shard/CloseFollowerIndexErrorSuppressionHelper.java @@ -0,0 +1,14 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.index.shard; + +public class CloseFollowerIndexErrorSuppressionHelper { + public static void setSuppressCreateEngineErrors(boolean value) { + IndexShard.suppressCreateEngineErrors = value; + } +} diff --git a/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/CloseFollowerIndexIT.java b/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/CloseFollowerIndexIT.java index 64ebb20538832..8e597c3992528 100644 --- a/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/CloseFollowerIndexIT.java +++ b/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/CloseFollowerIndexIT.java @@ -18,6 +18,7 @@ import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.engine.ReadOnlyEngine; +import org.elasticsearch.index.shard.CloseFollowerIndexErrorSuppressionHelper; import org.elasticsearch.xcontent.XContentType; import org.elasticsearch.xpack.CcrIntegTestCase; import org.elasticsearch.xpack.core.ccr.action.PutFollowAction; @@ -39,13 +40,16 @@ public class CloseFollowerIndexIT extends CcrIntegTestCase { @Before public void wrapUncaughtExceptionHandler() { + CloseFollowerIndexErrorSuppressionHelper.setSuppressCreateEngineErrors(true); uncaughtExceptionHandler = Thread.getDefaultUncaughtExceptionHandler(); AccessController.doPrivileged((PrivilegedAction) () -> { Thread.setDefaultUncaughtExceptionHandler((t, e) -> { - if (t.getThreadGroup().getName().contains(getTestClass().getSimpleName())) { + if (t.getThreadGroup().getName().contains(getTestClass().getSimpleName()) + && t.getName().equals("elasticsearch-error-rethrower")) { for (StackTraceElement element : e.getStackTrace()) { if (element.getClassName().equals(ReadOnlyEngine.class.getName())) { if (element.getMethodName().equals("assertMaxSeqNoEqualsToGlobalCheckpoint")) { + logger.error("HACK: suppressing uncaught exception thrown from assertMaxSeqNoEqualsToGlobalCheckpoint", e); return; } } @@ -59,6 +63,7 @@ public void wrapUncaughtExceptionHandler() { @After public void restoreUncaughtExceptionHandler() { + CloseFollowerIndexErrorSuppressionHelper.setSuppressCreateEngineErrors(false); AccessController.doPrivileged((PrivilegedAction) () -> { Thread.setDefaultUncaughtExceptionHandler(uncaughtExceptionHandler); return null; From b65e871c1bb99efa2c9699a8c03ff02ed421c6f6 Mon Sep 17 00:00:00 2001 From: eyalkoren <41850454+eyalkoren@users.noreply.github.com> Date: Mon, 20 Nov 2023 11:24:11 +0200 Subject: [PATCH 11/28] Fixing ECS tests - replace depracatoin property in template (#102356) --- .../org/elasticsearch/xpack/stack/EcsDynamicTemplatesIT.java | 3 +-- .../org/elasticsearch/xpack/stack/StackTemplateRegistry.java | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/x-pack/plugin/stack/src/javaRestTest/java/org/elasticsearch/xpack/stack/EcsDynamicTemplatesIT.java b/x-pack/plugin/stack/src/javaRestTest/java/org/elasticsearch/xpack/stack/EcsDynamicTemplatesIT.java index ea3286e96160c..25cea3b3f6e0a 100644 --- a/x-pack/plugin/stack/src/javaRestTest/java/org/elasticsearch/xpack/stack/EcsDynamicTemplatesIT.java +++ b/x-pack/plugin/stack/src/javaRestTest/java/org/elasticsearch/xpack/stack/EcsDynamicTemplatesIT.java @@ -32,7 +32,6 @@ import java.net.URL; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collections; import java.util.HashMap; import java.util.Iterator; import java.util.List; @@ -77,7 +76,7 @@ private static void prepareEcsDynamicTemplates() throws IOException { "/" + ECS_DYNAMIC_TEMPLATES_FILE, Integer.toString(1), StackTemplateRegistry.TEMPLATE_VERSION_VARIABLE, - Collections.emptyMap() + StackTemplateRegistry.ADDITIONAL_TEMPLATE_VARIABLES ); Map ecsDynamicTemplatesRaw; try ( diff --git a/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackTemplateRegistry.java b/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackTemplateRegistry.java index 36da14680c66a..8dc8238b8230b 100644 --- a/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackTemplateRegistry.java +++ b/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackTemplateRegistry.java @@ -57,7 +57,7 @@ public class StackTemplateRegistry extends IndexTemplateRegistry { private final FeatureService featureService; private volatile boolean stackTemplateEnabled; - private static final Map ADDITIONAL_TEMPLATE_VARIABLES = Map.of("xpack.stack.template.deprecated", "false"); + public static final Map ADDITIONAL_TEMPLATE_VARIABLES = Map.of("xpack.stack.template.deprecated", "false"); // General mappings conventions for any data that ends up in a data stream public static final String DATA_STREAMS_MAPPINGS_COMPONENT_TEMPLATE_NAME = "data-streams@mappings"; From e4291bb8f10736a359baf015965c3416399a75cf Mon Sep 17 00:00:00 2001 From: David Turner Date: Mon, 20 Nov 2023 11:06:59 +0000 Subject: [PATCH 12/28] AwaitsFix for #102373 --- .../lifecycle/action/DataStreamLifecycleStatsResponseTests.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/action/DataStreamLifecycleStatsResponseTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/action/DataStreamLifecycleStatsResponseTests.java index b8e4b252645dd..7242f72fe9f68 100644 --- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/action/DataStreamLifecycleStatsResponseTests.java +++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/action/DataStreamLifecycleStatsResponseTests.java @@ -8,6 +8,7 @@ package org.elasticsearch.datastreams.lifecycle.action; +import org.apache.lucene.tests.util.LuceneTestCase; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.xcontent.XContentHelper; @@ -26,6 +27,7 @@ import static org.elasticsearch.xcontent.ToXContent.EMPTY_PARAMS; import static org.hamcrest.Matchers.is; +@LuceneTestCase.AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/102373") public class DataStreamLifecycleStatsResponseTests extends AbstractWireSerializingTestCase { @Override From 7e669831e87220943907da6e731437340683ddc0 Mon Sep 17 00:00:00 2001 From: Luigi Dell'Aquila Date: Mon, 20 Nov 2023 12:43:54 +0100 Subject: [PATCH 13/28] ES|QL: Fix drop of renamed grouping (#102282) Fixes https://github.com/elastic/elasticsearch/issues/102121 Aggs groupings were not taken into account while merging aggs with projections, so they were wrongly removed in case of DROP --- docs/changelog/102282.yaml | 6 ++ .../src/main/resources/drop.csv-spec | 33 +++++++++++ .../esql/optimizer/LogicalPlanOptimizer.java | 38 ++++++++++++- .../optimizer/LogicalPlanOptimizerTests.java | 57 +++++++++++++++++++ 4 files changed, 133 insertions(+), 1 deletion(-) create mode 100644 docs/changelog/102282.yaml diff --git a/docs/changelog/102282.yaml b/docs/changelog/102282.yaml new file mode 100644 index 0000000000000..4860d70f99ccc --- /dev/null +++ b/docs/changelog/102282.yaml @@ -0,0 +1,6 @@ +pr: 102282 +summary: "ES|QL: Fix drop of renamed grouping" +area: ES|QL +type: bug +issues: + - 102121 diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/drop.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/drop.csv-spec index cd3afa25fc0a6..601b4f329f9d7 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/drop.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/drop.csv-spec @@ -54,3 +54,36 @@ c:l|mi:i|s:l 0 |null|null ; + +// see https://github.com/elastic/elasticsearch/issues/102121 +dropGrouping#[skip:-8.11.99, reason:planning bug fixed in v8.12] +row a = 1 | rename a AS foo | stats bar = count(*) by foo | drop foo; + +bar:long +1 +; + +dropGroupingMulti#[skip:-8.11.99] +row a = 1, b = 2 | rename a AS foo, b as bar | stats baz = count(*) by foo, bar | drop foo; + +baz:long | bar:integer +1 | 2 +; + +dropGroupingMulti2#[skip:-8.11.99] +row a = 1, b = 2 | rename a AS foo, b as bar | stats baz = count(*) by foo, bar | drop foo, bar; + +baz:long +1 +; + + +dropGroupingMultirow#[skip:-8.11.99] +from employees | rename gender AS foo | stats bar = count(*) by foo | drop foo | sort bar; + +bar:long +10 +33 +57 +; + diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizer.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizer.java index bfa9ea449ad46..29b61949b6778 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizer.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizer.java @@ -281,7 +281,10 @@ protected LogicalPlan rule(UnaryPlan plan) { // eliminate lower project but first replace the aliases in the upper one return p.withProjections(combineProjections(project.projections(), p.projections())); } else if (child instanceof Aggregate a) { - return new Aggregate(a.source(), a.child(), a.groupings(), combineProjections(project.projections(), a.aggregates())); + var aggs = a.aggregates(); + var newAggs = combineProjections(project.projections(), aggs); + var newGroups = replacePrunedAliasesUsedInGroupBy(a.groupings(), aggs, newAggs); + return new Aggregate(a.source(), a.child(), newGroups, newAggs); } } @@ -320,6 +323,39 @@ private List combineProjections(List return replaced; } + /** + * Replace grouping alias previously contained in the aggregations that might have been projected away. + */ + private List replacePrunedAliasesUsedInGroupBy( + List groupings, + List oldAggs, + List newAggs + ) { + AttributeMap removedAliases = new AttributeMap<>(); + AttributeSet currentAliases = new AttributeSet(Expressions.asAttributes(newAggs)); + + // record only removed aliases + for (NamedExpression ne : oldAggs) { + if (ne instanceof Alias alias) { + var attr = ne.toAttribute(); + if (currentAliases.contains(attr) == false) { + removedAliases.put(attr, alias.child()); + } + } + } + + if (removedAliases.isEmpty()) { + return groupings; + } + + var newGroupings = new ArrayList(groupings.size()); + for (Expression group : groupings) { + newGroupings.add(group.transformUp(Attribute.class, a -> removedAliases.resolve(a, a))); + } + + return newGroupings; + } + public static Expression trimNonTopLevelAliases(Expression e) { if (e instanceof Alias a) { return new Alias(a.source(), a.name(), a.qualifier(), trimAliases(a.child()), a.id()); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java index e825f1f96a8b3..b82bb46ec103e 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java @@ -2454,6 +2454,63 @@ public void testMvExpandFoldable() { var row = as(expand.child(), Row.class); } + /** + * Expected + * Limit[500[INTEGER]] + * \_Aggregate[[a{r}#2],[COUNT([2a][KEYWORD]) AS bar]] + * \_Row[[1[INTEGER] AS a]] + */ + public void testRenameStatsDropGroup() { + LogicalPlan plan = optimizedPlan(""" + row a = 1 + | rename a AS foo + | stats bar = count(*) by foo + | drop foo"""); + + var limit = as(plan, Limit.class); + var agg = as(limit.child(), Aggregate.class); + assertThat(Expressions.names(agg.groupings()), contains("a")); + var row = as(agg.child(), Row.class); + } + + /** + * Expected + * Limit[500[INTEGER]] + * \_Aggregate[[a{r}#2, bar{r}#8],[COUNT([2a][KEYWORD]) AS baz, b{r}#4 AS bar]] + * \_Row[[1[INTEGER] AS a, 2[INTEGER] AS b]] + */ + public void testMultipleRenameStatsDropGroup() { + LogicalPlan plan = optimizedPlan(""" + row a = 1, b = 2 + | rename a AS foo, b as bar + | stats baz = count(*) by foo, bar + | drop foo"""); + + var limit = as(plan, Limit.class); + var agg = as(limit.child(), Aggregate.class); + assertThat(Expressions.names(agg.groupings()), contains("a", "bar")); + var row = as(agg.child(), Row.class); + } + + /** + * Expected + * Limit[500[INTEGER]] + * \_Aggregate[[emp_no{f}#11, bar{r}#4],[MAX(salary{f}#16) AS baz, gender{f}#13 AS bar]] + * \_EsRelation[test][_meta_field{f}#17, emp_no{f}#11, first_name{f}#12, ..] + */ + public void testMultipleRenameStatsDropGroupMultirow() { + LogicalPlan plan = optimizedPlan(""" + from test + | rename emp_no AS foo, gender as bar + | stats baz = max(salary) by foo, bar + | drop foo"""); + + var limit = as(plan, Limit.class); + var agg = as(limit.child(), Aggregate.class); + assertThat(Expressions.names(agg.groupings()), contains("emp_no", "bar")); + var row = as(agg.child(), EsRelation.class); + } + private T aliased(Expression exp, Class clazz) { var alias = as(exp, Alias.class); return as(alias.child(), clazz); From e9dee63b5dba6ec27518ee96d293c66248054b0a Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Mon, 20 Nov 2023 07:46:50 -0500 Subject: [PATCH 14/28] ESQL: Fix rare bug with empty string (#102350) This fixes a rare bug that occurs when an empty value lands on the page boundary of the last page. We only allocate the last page if we need some bytes from it. So if you add an empty string to the as the last entry in a BytesRefBlock we don't allocate a whole new slab of bytes to hold the empty string. But, without this change, we attempt to read from the unallocated array. We try to read 0 bytes from it, but still. That's a read past the end of the array. Closes #101969 --- docs/changelog/102350.yaml | 6 ++ .../common/util/BigByteArray.java | 4 ++ .../common/util/BytesRefArrayTests.java | 61 +++++++++++++++++++ 3 files changed, 71 insertions(+) create mode 100644 docs/changelog/102350.yaml diff --git a/docs/changelog/102350.yaml b/docs/changelog/102350.yaml new file mode 100644 index 0000000000000..00a311c5d99f8 --- /dev/null +++ b/docs/changelog/102350.yaml @@ -0,0 +1,6 @@ +pr: 102350 +summary: "ESQL: Fix rare bug with empty string" +area: ES|QL +type: bug +issues: + - 101969 diff --git a/server/src/main/java/org/elasticsearch/common/util/BigByteArray.java b/server/src/main/java/org/elasticsearch/common/util/BigByteArray.java index 72a2fc41a9a12..2c623882afe14 100644 --- a/server/src/main/java/org/elasticsearch/common/util/BigByteArray.java +++ b/server/src/main/java/org/elasticsearch/common/util/BigByteArray.java @@ -64,6 +64,10 @@ public byte set(long index, byte value) { @Override public boolean get(long index, int len, BytesRef ref) { assert index + len <= size(); + if (len == 0) { + ref.length = 0; + return false; + } int pageIndex = pageIndex(index); final int indexInPage = indexInPage(index); if (indexInPage + len <= pageSize()) { diff --git a/server/src/test/java/org/elasticsearch/common/util/BytesRefArrayTests.java b/server/src/test/java/org/elasticsearch/common/util/BytesRefArrayTests.java index 0ca6bf86ceec7..e7868f442efd0 100644 --- a/server/src/test/java/org/elasticsearch/common/util/BytesRefArrayTests.java +++ b/server/src/test/java/org/elasticsearch/common/util/BytesRefArrayTests.java @@ -17,6 +17,9 @@ import org.elasticsearch.test.ESTestCase; import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.IntStream; import static org.hamcrest.Matchers.equalTo; @@ -115,6 +118,64 @@ public void testLookup() throws IOException { } } + public void testReadWritten() { + testReadWritten(false); + } + + public void testReadWrittenHalfEmpty() { + testReadWritten(true); + } + + private void testReadWritten(boolean halfEmpty) { + List values = new ArrayList<>(); + int bytes = PageCacheRecycler.PAGE_SIZE_IN_BYTES * between(2, 20); + int used = 0; + while (used < bytes) { + String str = halfEmpty && randomBoolean() ? "" : randomAlphaOfLengthBetween(0, 200); + BytesRef v = new BytesRef(str); + used += v.length; + values.add(v); + } + testReadWritten(values, randomBoolean() ? bytes : between(0, bytes)); + } + + public void testReadWrittenRepeated() { + testReadWrittenRepeated(false, between(2, 3000)); + } + + public void testReadWrittenRepeatedPowerOfTwo() { + testReadWrittenRepeated(false, 1024); + } + + public void testReadWrittenRepeatedHalfEmpty() { + testReadWrittenRepeated(true, between(1, 3000)); + } + + public void testReadWrittenRepeatedHalfEmptyPowerOfTwo() { + testReadWrittenRepeated(true, 1024); + } + + public void testReadWrittenRepeated(boolean halfEmpty, int listSize) { + List values = randomList(2, 10, () -> { + String str = halfEmpty && randomBoolean() ? "" : randomAlphaOfLengthBetween(0, 10); + return new BytesRef(str); + }); + testReadWritten(IntStream.range(0, listSize).mapToObj(i -> values).flatMap(List::stream).toList(), 10); + } + + private void testReadWritten(List values, int initialCapacity) { + try (BytesRefArray array = new BytesRefArray(initialCapacity, mockBigArrays())) { + for (BytesRef v : values) { + array.append(v); + } + BytesRef scratch = new BytesRef(); + for (int i = 0; i < values.size(); i++) { + array.get(i, scratch); + assertThat(scratch, equalTo(values.get(i))); + } + } + } + private static BigArrays mockBigArrays() { return new MockBigArrays(new MockPageCacheRecycler(Settings.EMPTY), new NoneCircuitBreakerService()); } From 3468730e0a7bfae1547090c3203aaec9af6638d8 Mon Sep 17 00:00:00 2001 From: David Turner Date: Mon, 20 Nov 2023 13:02:34 +0000 Subject: [PATCH 15/28] Visualize history on LinearizabilityChecker failure (#102376) It's a bit of a pain to have to re-run this checker in order to interpret a linearizability failure. With this commit we record the visualisation of the history (compressed) in the logs alongside the raw events. --- .../ConcurrentSeqNoVersioningIT.java | 15 ++++- .../LinearizabilityCheckerTests.java | 21 ++++++ .../coordination/LinearizabilityChecker.java | 65 ++++++++++++------- 3 files changed, 76 insertions(+), 25 deletions(-) diff --git a/server/src/internalClusterTest/java/org/elasticsearch/versioning/ConcurrentSeqNoVersioningIT.java b/server/src/internalClusterTest/java/org/elasticsearch/versioning/ConcurrentSeqNoVersioningIT.java index 0277c569fbe5d..cf59f5a8f6f3e 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/versioning/ConcurrentSeqNoVersioningIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/versioning/ConcurrentSeqNoVersioningIT.java @@ -33,6 +33,8 @@ import java.io.FileInputStream; import java.io.IOException; import java.io.OutputStream; +import java.io.OutputStreamWriter; +import java.nio.charset.StandardCharsets; import java.util.Arrays; import java.util.Base64; import java.util.List; @@ -448,13 +450,24 @@ public void assertLinearizable() { var chunkedLoggingStream = ChunkedLoggingStream.create( logger, Level.ERROR, - "unlinearizable history", + "raw unlinearizable history in partition " + id, ReferenceDocs.LOGGING // any old docs link will do ); var output = new OutputStreamStreamOutput(chunkedLoggingStream) ) { writeHistory(output, history); } + try ( + var chunkedLoggingStream = ChunkedLoggingStream.create( + logger, + Level.ERROR, + "visualisation of unlinearizable history in partition " + id, + ReferenceDocs.LOGGING // any old docs link will do + ); + var writer = new OutputStreamWriter(chunkedLoggingStream, StandardCharsets.UTF_8) + ) { + LinearizabilityChecker.writeVisualisation(spec, history, missingResponseGenerator(), writer); + } } } catch (IOException e) { logger.error("failure writing out history", e); diff --git a/server/src/test/java/org/elasticsearch/cluster/coordination/LinearizabilityCheckerTests.java b/server/src/test/java/org/elasticsearch/cluster/coordination/LinearizabilityCheckerTests.java index 34fe7eae32fcb..83df7e7d18f5c 100644 --- a/server/src/test/java/org/elasticsearch/cluster/coordination/LinearizabilityCheckerTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/coordination/LinearizabilityCheckerTests.java @@ -143,6 +143,27 @@ public void testRegisterWithLinearizableHistory() throws LinearizabilityCheckAbo assertTrue(LinearizabilityChecker.isLinearizable(registerSpec, history)); } + public void testRegisterHistoryVisualisation() { + final History history = new History(); + int write0 = history.invoke(42); // invoke write(42) + history.respond(history.invoke(null), 42); // read, returns 42 + history.respond(write0, null); // write(42) succeeds + + int write1 = history.invoke(24); // invoke write 24 + history.respond(history.invoke(null), 42); // read returns 42 + history.respond(history.invoke(null), 24); // subsequent read returns 24 + history.respond(write1, null); // write(24) succeeds + + assertEquals(""" + Partition 0 + 42 XXX null (0) + null X 42 (1) + 24 XXXXX null (2) + null X 42 (3) + null X 24 (4) + """, LinearizabilityChecker.visualize(registerSpec, history, o -> { throw new AssertionError("history was complete"); })); + } + public void testRegisterWithNonLinearizableHistory() throws LinearizabilityCheckAborted { final History history = new History(); int call0 = history.invoke(42); // 0: invoke write 42 diff --git a/test/framework/src/main/java/org/elasticsearch/cluster/coordination/LinearizabilityChecker.java b/test/framework/src/main/java/org/elasticsearch/cluster/coordination/LinearizabilityChecker.java index 223b0dc5a546b..6c43eff24be21 100644 --- a/test/framework/src/main/java/org/elasticsearch/cluster/coordination/LinearizabilityChecker.java +++ b/test/framework/src/main/java/org/elasticsearch/cluster/coordination/LinearizabilityChecker.java @@ -17,6 +17,9 @@ import org.elasticsearch.threadpool.Scheduler; import org.elasticsearch.threadpool.ThreadPool; +import java.io.IOException; +import java.io.StringWriter; +import java.io.Writer; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -35,7 +38,6 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.function.BooleanSupplier; -import java.util.function.Consumer; import java.util.function.Function; /** @@ -308,25 +310,38 @@ private static boolean isLinearizable(SequentialSpec spec, List history, * Return a visual representation of the history */ public static String visualize(SequentialSpec spec, History history, Function missingResponseGenerator) { + final var writer = new StringWriter(); + writeVisualisation(spec, history, missingResponseGenerator, writer); + return writer.toString(); + } + + /** + * Write a visual representation of the history to the given writer + */ + public static void writeVisualisation( + SequentialSpec spec, + History history, + Function missingResponseGenerator, + Writer writer + ) { history = history.clone(); history.complete(missingResponseGenerator); final Collection> partitions = spec.partition(history.copyEvents()); - StringBuilder builder = new StringBuilder(); - partitions.forEach(new Consumer>() { + try { int index = 0; - - @Override - public void accept(List events) { - builder.append("Partition ").append(index++).append("\n"); - builder.append(visualizePartition(events)); + for (List partition : partitions) { + writer.write("Partition "); + writer.write(Integer.toString(index++)); + writer.append('\n'); + visualizePartition(partition, writer); } - }); - - return builder.toString(); + } catch (IOException e) { + logger.error("unexpected writeVisualisation failure", e); + assert false : e; // not really doing any IO + } } - private static String visualizePartition(List events) { - StringBuilder builder = new StringBuilder(); + private static void visualizePartition(List events, Writer writer) throws IOException { Entry entry = createLinkedEntries(events).next; Map, Integer> eventToPosition = new HashMap<>(); for (Event event : events) { @@ -334,28 +349,30 @@ private static String visualizePartition(List events) { } while (entry != null) { if (entry.match != null) { - builder.append(visualizeEntry(entry, eventToPosition)).append("\n"); + visualizeEntry(entry, eventToPosition, writer); + writer.append('\n'); } entry = entry.next; } - return builder.toString(); } - private static String visualizeEntry(Entry entry, Map, Integer> eventToPosition) { + private static void visualizeEntry(Entry entry, Map, Integer> eventToPosition, Writer writer) + throws IOException { + String input = String.valueOf(entry.event.value); String output = String.valueOf(entry.match.event.value); int id = entry.event.id; int beginIndex = eventToPosition.get(Tuple.tuple(EventType.INVOCATION, id)); int endIndex = eventToPosition.get(Tuple.tuple(EventType.RESPONSE, id)); input = input.substring(0, Math.min(beginIndex + 25, input.length())); - return Strings.padStart(input, beginIndex + 25, ' ') - + " " - + Strings.padStart("", endIndex - beginIndex, 'X') - + " " - + output - + " (" - + entry.event.id - + ")"; + writer.write(Strings.padStart(input, beginIndex + 25, ' ')); + writer.write(" "); + writer.write(Strings.padStart("", endIndex - beginIndex, 'X')); + writer.write(" "); + writer.write(output); + writer.write(" ("); + writer.write(Integer.toString(entry.event.id)); + writer.write(")"); } /** From 92fb7780f9f8f289e96c7b0b5569f61d3c4a7279 Mon Sep 17 00:00:00 2001 From: Alexander Spies Date: Mon, 20 Nov 2023 14:09:15 +0100 Subject: [PATCH 16/28] ESQL: Make blocks ref counted (#100408) This allows to replace deep copying of blocks by simply calling Block::incRef - the block then has to be closed (or decRefed) one additional time for each call to incRef (and tryIncRef, if successfull). --- docs/changelog/100408.yaml | 5 + .../org/elasticsearch/core/RefCounted.java | 2 +- .../compute/data/BooleanArrayBlock.java | 6 +- .../compute/data/BooleanVectorBlock.java | 12 +- .../compute/data/BytesRefArrayBlock.java | 6 +- .../compute/data/BytesRefVectorBlock.java | 12 +- .../compute/data/DoubleArrayBlock.java | 6 +- .../compute/data/DoubleVectorBlock.java | 12 +- .../compute/data/IntArrayBlock.java | 6 +- .../compute/data/IntVectorBlock.java | 12 +- .../compute/data/LongArrayBlock.java | 6 +- .../compute/data/LongVectorBlock.java | 12 +- .../compute/data/AbstractBlock.java | 54 +++++- .../org/elasticsearch/compute/data/Block.java | 28 ++- .../compute/data/ConstantNullBlock.java | 6 +- .../elasticsearch/compute/data/DocBlock.java | 12 +- .../elasticsearch/compute/data/DocVector.java | 1 + .../org/elasticsearch/compute/data/Page.java | 36 +--- .../compute/data/X-ArrayBlock.java.st | 6 +- .../compute/data/X-VectorBlock.java.st | 12 +- .../lucene/LuceneTopNSourceOperator.java | 2 +- .../compute/operator/AggregationOperator.java | 9 +- .../compute/operator/EvalOperator.java | 7 +- .../compute/operator/FilterOperator.java | 2 +- .../operator/HashAggregationOperator.java | 2 +- .../compute/operator/LimitOperator.java | 7 +- .../compute/operator/ProjectOperator.java | 7 +- .../compute/data/BasicBlockTests.java | 159 +++++++++++++++++- .../compute/data/BasicPageTests.java | 10 -- .../esql/planner/LocalExecutionPlanner.java | 7 +- 30 files changed, 313 insertions(+), 151 deletions(-) create mode 100644 docs/changelog/100408.yaml diff --git a/docs/changelog/100408.yaml b/docs/changelog/100408.yaml new file mode 100644 index 0000000000000..275c3b4a0de48 --- /dev/null +++ b/docs/changelog/100408.yaml @@ -0,0 +1,5 @@ +pr: 100408 +summary: "ESQL: Make blocks ref counted" +area: ES|QL +type: enhancement +issues: [] diff --git a/libs/core/src/main/java/org/elasticsearch/core/RefCounted.java b/libs/core/src/main/java/org/elasticsearch/core/RefCounted.java index 0f7dec4968ba7..49c030609951b 100644 --- a/libs/core/src/main/java/org/elasticsearch/core/RefCounted.java +++ b/libs/core/src/main/java/org/elasticsearch/core/RefCounted.java @@ -38,7 +38,7 @@ public interface RefCounted { void incRef(); /** - * Tries to increment the refCount of this instance. This method will return {@code true} iff the refCount was + * Tries to increment the refCount of this instance. This method will return {@code true} iff the refCount was successfully incremented. * * @see #decRef() * @see #incRef() diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BooleanArrayBlock.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BooleanArrayBlock.java index eb9364c57e755..ed38d3139dd4a 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BooleanArrayBlock.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BooleanArrayBlock.java @@ -135,11 +135,7 @@ public String toString() { } @Override - public void close() { - if (released) { - throw new IllegalStateException("can't release already released block [" + this + "]"); - } - released = true; + public void closeInternal() { blockFactory.adjustBreaker(-ramBytesUsed(), true); } } diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BooleanVectorBlock.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BooleanVectorBlock.java index 1a7a5b4aa6e7e..c5c3a24736c16 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BooleanVectorBlock.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BooleanVectorBlock.java @@ -17,6 +17,9 @@ public final class BooleanVectorBlock extends AbstractVectorBlock implements Boo private final BooleanVector vector; + /** + * @param vector considered owned by the current block; must not be used in any other {@code Block} + */ BooleanVectorBlock(BooleanVector vector) { super(vector.getPositionCount(), vector.blockFactory()); this.vector = vector; @@ -72,15 +75,12 @@ public String toString() { @Override public boolean isReleased() { - return released || vector.isReleased(); + return super.isReleased() || vector.isReleased(); } @Override - public void close() { - if (released || vector.isReleased()) { - throw new IllegalStateException("can't release already released block [" + this + "]"); - } - released = true; + public void closeInternal() { + assert (vector.isReleased() == false) : "can't release block [" + this + "] containing already released vector"; Releasables.closeExpectNoException(vector); } } diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BytesRefArrayBlock.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BytesRefArrayBlock.java index b2729ed370b32..6aef8fa54b134 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BytesRefArrayBlock.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BytesRefArrayBlock.java @@ -139,11 +139,7 @@ public String toString() { } @Override - public void close() { - if (released) { - throw new IllegalStateException("can't release already released block [" + this + "]"); - } - released = true; + public void closeInternal() { blockFactory.adjustBreaker(-ramBytesUsed() + values.bigArraysRamBytesUsed(), true); Releasables.closeExpectNoException(values); } diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BytesRefVectorBlock.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BytesRefVectorBlock.java index 5b0f2f2331fbe..d8c2c615a3dfb 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BytesRefVectorBlock.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/BytesRefVectorBlock.java @@ -18,6 +18,9 @@ public final class BytesRefVectorBlock extends AbstractVectorBlock implements By private final BytesRefVector vector; + /** + * @param vector considered owned by the current block; must not be used in any other {@code Block} + */ BytesRefVectorBlock(BytesRefVector vector) { super(vector.getPositionCount(), vector.blockFactory()); this.vector = vector; @@ -73,15 +76,12 @@ public String toString() { @Override public boolean isReleased() { - return released || vector.isReleased(); + return super.isReleased() || vector.isReleased(); } @Override - public void close() { - if (released || vector.isReleased()) { - throw new IllegalStateException("can't release already released block [" + this + "]"); - } - released = true; + public void closeInternal() { + assert (vector.isReleased() == false) : "can't release block [" + this + "] containing already released vector"; Releasables.closeExpectNoException(vector); } } diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/DoubleArrayBlock.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/DoubleArrayBlock.java index b6b1fae0ded03..6a5af2d7ca6de 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/DoubleArrayBlock.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/DoubleArrayBlock.java @@ -135,11 +135,7 @@ public String toString() { } @Override - public void close() { - if (released) { - throw new IllegalStateException("can't release already released block [" + this + "]"); - } - released = true; + public void closeInternal() { blockFactory.adjustBreaker(-ramBytesUsed(), true); } } diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/DoubleVectorBlock.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/DoubleVectorBlock.java index d05be62744bc8..ac4c826b5f2d2 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/DoubleVectorBlock.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/DoubleVectorBlock.java @@ -17,6 +17,9 @@ public final class DoubleVectorBlock extends AbstractVectorBlock implements Doub private final DoubleVector vector; + /** + * @param vector considered owned by the current block; must not be used in any other {@code Block} + */ DoubleVectorBlock(DoubleVector vector) { super(vector.getPositionCount(), vector.blockFactory()); this.vector = vector; @@ -72,15 +75,12 @@ public String toString() { @Override public boolean isReleased() { - return released || vector.isReleased(); + return super.isReleased() || vector.isReleased(); } @Override - public void close() { - if (released || vector.isReleased()) { - throw new IllegalStateException("can't release already released block [" + this + "]"); - } - released = true; + public void closeInternal() { + assert (vector.isReleased() == false) : "can't release block [" + this + "] containing already released vector"; Releasables.closeExpectNoException(vector); } } diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/IntArrayBlock.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/IntArrayBlock.java index 31f71d292f95d..284520a5f3bd6 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/IntArrayBlock.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/IntArrayBlock.java @@ -135,11 +135,7 @@ public String toString() { } @Override - public void close() { - if (released) { - throw new IllegalStateException("can't release already released block [" + this + "]"); - } - released = true; + public void closeInternal() { blockFactory.adjustBreaker(-ramBytesUsed(), true); } } diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/IntVectorBlock.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/IntVectorBlock.java index 472475d0662d7..60280ebb13064 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/IntVectorBlock.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/IntVectorBlock.java @@ -17,6 +17,9 @@ public final class IntVectorBlock extends AbstractVectorBlock implements IntBloc private final IntVector vector; + /** + * @param vector considered owned by the current block; must not be used in any other {@code Block} + */ IntVectorBlock(IntVector vector) { super(vector.getPositionCount(), vector.blockFactory()); this.vector = vector; @@ -72,15 +75,12 @@ public String toString() { @Override public boolean isReleased() { - return released || vector.isReleased(); + return super.isReleased() || vector.isReleased(); } @Override - public void close() { - if (released || vector.isReleased()) { - throw new IllegalStateException("can't release already released block [" + this + "]"); - } - released = true; + public void closeInternal() { + assert (vector.isReleased() == false) : "can't release block [" + this + "] containing already released vector"; Releasables.closeExpectNoException(vector); } } diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/LongArrayBlock.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/LongArrayBlock.java index 8a71703441ebb..fccad0ec1f09b 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/LongArrayBlock.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/LongArrayBlock.java @@ -135,11 +135,7 @@ public String toString() { } @Override - public void close() { - if (released) { - throw new IllegalStateException("can't release already released block [" + this + "]"); - } - released = true; + public void closeInternal() { blockFactory.adjustBreaker(-ramBytesUsed(), true); } } diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/LongVectorBlock.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/LongVectorBlock.java index b94cd4e875dc3..c9b65ba3e9029 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/LongVectorBlock.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/LongVectorBlock.java @@ -17,6 +17,9 @@ public final class LongVectorBlock extends AbstractVectorBlock implements LongBl private final LongVector vector; + /** + * @param vector considered owned by the current block; must not be used in any other {@code Block} + */ LongVectorBlock(LongVector vector) { super(vector.getPositionCount(), vector.blockFactory()); this.vector = vector; @@ -72,15 +75,12 @@ public String toString() { @Override public boolean isReleased() { - return released || vector.isReleased(); + return super.isReleased() || vector.isReleased(); } @Override - public void close() { - if (released || vector.isReleased()) { - throw new IllegalStateException("can't release already released block [" + this + "]"); - } - released = true; + public void closeInternal() { + assert (vector.isReleased() == false) : "can't release block [" + this + "] containing already released vector"; Releasables.closeExpectNoException(vector); } } diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/AbstractBlock.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/AbstractBlock.java index cbe74c814594d..39f17cfecab1a 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/AbstractBlock.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/AbstractBlock.java @@ -12,7 +12,7 @@ import java.util.BitSet; abstract class AbstractBlock implements Block { - + private int references = 1; private final int positionCount; @Nullable @@ -23,8 +23,6 @@ abstract class AbstractBlock implements Block { protected final BlockFactory blockFactory; - protected boolean released = false; - /** * @param positionCount the number of values in this block */ @@ -99,6 +97,54 @@ public BlockFactory blockFactory() { @Override public boolean isReleased() { - return released; + return hasReferences() == false; + } + + @Override + public final void incRef() { + if (isReleased()) { + throw new IllegalStateException("can't increase refCount on already released block [" + this + "]"); + } + references++; + } + + @Override + public final boolean tryIncRef() { + if (isReleased()) { + return false; + } + references++; + return true; + } + + @Override + public final boolean decRef() { + if (isReleased()) { + throw new IllegalStateException("can't release already released block [" + this + "]"); + } + + references--; + + if (references <= 0) { + closeInternal(); + return true; + } + return false; + } + + @Override + public final boolean hasReferences() { + return references >= 1; } + + @Override + public final void close() { + decRef(); + } + + /** + * This is called when the number of references reaches zero. + * It must release any resources held by the block (adjusting circuit breakers if needed). + */ + protected abstract void closeInternal(); } diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/Block.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/Block.java index 75b02ff911df7..ee9889d7d3be8 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/Block.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/Block.java @@ -11,6 +11,7 @@ import org.elasticsearch.common.io.stream.NamedWriteable; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.core.Nullable; +import org.elasticsearch.core.RefCounted; import org.elasticsearch.core.Releasable; import org.elasticsearch.core.Releasables; import org.elasticsearch.index.mapper.BlockLoader; @@ -23,12 +24,19 @@ * position. * *

Blocks can represent various shapes of underlying data. A Block can represent either sparse - * or dense data. A Block can represent either single or multi valued data. A Block that represents + * or dense data. A Block can represent either single or multivalued data. A Block that represents * dense single-valued data can be viewed as a {@link Vector}. * - *

Block are immutable and can be passed between threads. + *

Blocks are reference counted; to make a shallow copy of a block (e.g. if a {@link Page} contains + * the same column twice), use {@link Block#incRef()}. Before a block is garbage collected, + * {@link Block#close()} must be called to release a block's resources; it must also be called one + * additional time for each time {@link Block#incRef()} was called. Calls to {@link Block#decRef()} and + * {@link Block#close()} are equivalent. + * + *

Block are immutable and can be passed between threads as long as no two threads hold a reference to + * the same block at the same time. */ -public interface Block extends Accountable, BlockLoader.Block, NamedWriteable, Releasable { +public interface Block extends Accountable, BlockLoader.Block, NamedWriteable, RefCounted, Releasable { /** * {@return an efficient dense single-value view of this block}. @@ -57,14 +65,15 @@ public interface Block extends Accountable, BlockLoader.Block, NamedWriteable, R /** The block factory associated with this block. */ BlockFactory blockFactory(); - /** Tells if this block has been released. A block is released by calling its {@link Block#close()} method. */ + /** + * Tells if this block has been released. A block is released by calling its {@link Block#close()} or {@link Block#decRef()} methods. + * @return true iff the block's reference count is zero. + * */ boolean isReleased(); /** - * Returns true if the value stored at the given position is null, false otherwise. - * * @param position the position - * @return true or false + * @return true if the value stored at the given position is null, false otherwise */ boolean isNull(int position); @@ -91,6 +100,7 @@ public interface Block extends Accountable, BlockLoader.Block, NamedWriteable, R /** * Creates a new block that only exposes the positions provided. Materialization of the selected positions is avoided. + * The new block may hold a reference to this block, increasing this block's reference count. * @param positions the positions to retain * @return a filtered block */ @@ -137,6 +147,7 @@ default boolean mvSortedAscending() { * Expand multivalued fields into one row per value. Returns the * block if there aren't any multivalued fields to expand. */ + // TODO: We should use refcounting instead of either deep copies or returning the same identical block. Block expand(); /** @@ -231,7 +242,7 @@ static Block[] buildAll(Block.Builder... builders) { /** * A reference to a {@link Block}. This is {@link Releasable} and - * {@link Ref#close closing} it will {@link Block#close release} + * {@link Ref#close closing} it will {@link Block#close() release} * the underlying {@link Block} if it wasn't borrowed from a {@link Page}. * * The usual way to use this is: @@ -248,6 +259,7 @@ static Block[] buildAll(Block.Builder... builders) { * @param block the block referenced * @param containedIn the page containing it or null, if it is "free floating". */ + // We probably want to remove this; instead, we could incRef and decRef consistently in the EvalOperator. record Ref(Block block, @Nullable Page containedIn) implements Releasable { /** * Create a "free floating" {@link Ref}. diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/ConstantNullBlock.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/ConstantNullBlock.java index 9437bdd35e21f..5823a4b98d52c 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/ConstantNullBlock.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/ConstantNullBlock.java @@ -126,11 +126,7 @@ public String toString() { } @Override - public void close() { - if (isReleased()) { - throw new IllegalStateException("can't release already released block [" + this + "]"); - } - released = true; + public void closeInternal() { blockFactory.adjustBreaker(-ramBytesUsed(), true); } diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/DocBlock.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/DocBlock.java index ed7e317bfc4c7..9dc27196bd128 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/DocBlock.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/DocBlock.java @@ -71,11 +71,13 @@ public long ramBytesUsed() { } @Override - public void close() { - if (released) { - throw new IllegalStateException("can't release already released block [" + this + "]"); - } - released = true; + public boolean isReleased() { + return super.isReleased() || vector.isReleased(); + } + + @Override + public void closeInternal() { + assert (vector.isReleased() == false) : "can't release block [" + this + "] containing already released vector"; Releasables.closeExpectNoException(vector); } diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/DocVector.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/DocVector.java index b6ba42f953609..24c656404e89f 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/DocVector.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/DocVector.java @@ -225,6 +225,7 @@ public long ramBytesUsed() { @Override public void close() { + released = true; Releasables.closeExpectNoException(shards.asBlock(), segments.asBlock(), docs.asBlock()); // Ugh! we always close blocks } } diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/Page.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/Page.java index 451a0b540f308..94f27e9e55f33 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/Page.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/Page.java @@ -14,8 +14,6 @@ import java.io.IOException; import java.util.Arrays; -import java.util.Collections; -import java.util.IdentityHashMap; import java.util.Objects; /** @@ -235,39 +233,7 @@ public void releaseBlocks() { blocksReleased = true; - // blocks can be used as multiple columns - var map = new IdentityHashMap(mapSize(blocks.length)); - for (Block b : blocks) { - if (map.putIfAbsent(b, Boolean.TRUE) == null) { - Releasables.closeExpectNoException(b); - } - } - } - - /** - * Returns a Page from the given blocks and closes all blocks that are not included, from the current Page. - * That is, allows clean-up of the current page _after_ external manipulation of the blocks. - * The current page should no longer be used and be considered closed. - */ - public Page newPageAndRelease(Block... keep) { - if (blocksReleased) { - throw new IllegalStateException("can't create new page from already released page"); - } - - blocksReleased = true; - - var newPage = new Page(positionCount, keep); - var set = Collections.newSetFromMap(new IdentityHashMap(mapSize(keep.length))); - set.addAll(Arrays.asList(keep)); - - // close blocks that have been left out - for (Block b : blocks) { - if (set.contains(b) == false) { - Releasables.closeExpectNoException(b); - } - } - - return newPage; + Releasables.closeExpectNoException(blocks); } static int mapSize(int expectedSize) { diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-ArrayBlock.java.st b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-ArrayBlock.java.st index 49a4c43709cde..86a8dfc78450d 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-ArrayBlock.java.st +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-ArrayBlock.java.st @@ -168,11 +168,7 @@ $endif$ } @Override - public void close() { - if (released) { - throw new IllegalStateException("can't release already released block [" + this + "]"); - } - released = true; + public void closeInternal() { $if(BytesRef)$ blockFactory.adjustBreaker(-ramBytesUsed() + values.bigArraysRamBytesUsed(), true); Releasables.closeExpectNoException(values); diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-VectorBlock.java.st b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-VectorBlock.java.st index 3ef4251f80684..89bc84d551b63 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-VectorBlock.java.st +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-VectorBlock.java.st @@ -20,6 +20,9 @@ public final class $Type$VectorBlock extends AbstractVectorBlock implements $Typ private final $Type$Vector vector; + /** + * @param vector considered owned by the current block; must not be used in any other {@code Block} + */ $Type$VectorBlock($Type$Vector vector) { super(vector.getPositionCount(), vector.blockFactory()); this.vector = vector; @@ -80,15 +83,12 @@ $endif$ @Override public boolean isReleased() { - return released || vector.isReleased(); + return super.isReleased() || vector.isReleased(); } @Override - public void close() { - if (released || vector.isReleased()) { - throw new IllegalStateException("can't release already released block [" + this + "]"); - } - released = true; + public void closeInternal() { + assert (vector.isReleased() == false) : "can't release block [" + this + "] containing already released vector"; Releasables.closeExpectNoException(vector); } } diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/LuceneTopNSourceOperator.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/LuceneTopNSourceOperator.java index 4ce0af3bd0ffe..9624fa48ef20d 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/LuceneTopNSourceOperator.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/LuceneTopNSourceOperator.java @@ -219,7 +219,7 @@ private Page emit(boolean startEmitting) { page = new Page(size, new DocVector(shard.asVector(), segments, docs, null).asBlock()); } finally { if (page == null) { - Releasables.close(shard, segments, docs); + Releasables.closeExpectNoException(shard, segments, docs); } } pagesEmitted++; diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/AggregationOperator.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/AggregationOperator.java index 3e653b1a19750..07d1809262c9b 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/AggregationOperator.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/AggregationOperator.java @@ -124,10 +124,11 @@ public boolean isFinished() { @Override public void close() { - if (output != null) { - Releasables.closeExpectNoException(() -> output.releaseBlocks()); - } - Releasables.close(aggregators); + Releasables.closeExpectNoException(() -> { + if (output != null) { + Releasables.closeExpectNoException(() -> output.releaseBlocks()); + } + }, Releasables.wrap(aggregators)); } private static void checkState(boolean condition, String msg) { diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/EvalOperator.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/EvalOperator.java index 65efdc4266b28..1612a2b2ece18 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/EvalOperator.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/EvalOperator.java @@ -9,7 +9,6 @@ import org.elasticsearch.compute.data.Block; import org.elasticsearch.compute.data.BlockFactory; -import org.elasticsearch.compute.data.BlockUtils; import org.elasticsearch.compute.data.Page; import org.elasticsearch.core.Releasable; import org.elasticsearch.core.Releasables; @@ -44,7 +43,11 @@ public EvalOperator(BlockFactory blockFactory, ExpressionEvaluator evaluator) { @Override protected Page process(Page page) { Block.Ref ref = evaluator.eval(page); - Block block = ref.floating() ? ref.block() : BlockUtils.deepCopyOf(ref.block(), blockFactory); + Block block = ref.block(); + if (ref.floating() == false) { + // We take ownership of this block, so we need to shallow copy (incRef) to avoid double releases. + block.incRef(); + } return page.appendBlock(block); } diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/FilterOperator.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/FilterOperator.java index be4996e129d7b..d3e86352d1f29 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/FilterOperator.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/FilterOperator.java @@ -78,7 +78,7 @@ protected Page process(Page page) { } success = true; } finally { - Releasables.closeExpectNoException(page::releaseBlocks); + page.releaseBlocks(); if (success == false) { Releasables.closeExpectNoException(filteredBlocks); } diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/HashAggregationOperator.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/HashAggregationOperator.java index 4b26b74b42a1d..39068787f3c9e 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/HashAggregationOperator.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/HashAggregationOperator.java @@ -161,7 +161,7 @@ public void finish() { } finally { // selected should always be closed if (selected != null) { - Releasables.closeExpectNoException(selected.asBlock()); // we always close blocks, not vectors + selected.close(); } if (success == false && blocks != null) { Releasables.closeExpectNoException(blocks); diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/LimitOperator.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/LimitOperator.java index a41057386d365..bcd2ffa1f3855 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/LimitOperator.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/LimitOperator.java @@ -100,11 +100,12 @@ public Page getOutput() { } success = true; } finally { - Releasables.closeExpectNoException(lastInput::releaseBlocks); - lastInput = null; if (success == false) { - Releasables.closeExpectNoException(blocks); + Releasables.closeExpectNoException(lastInput::releaseBlocks, Releasables.wrap(blocks)); + } else { + lastInput.releaseBlocks(); } + lastInput = null; } result = new Page(blocks); limitRemaining = 0; diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/ProjectOperator.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/ProjectOperator.java index 6e52a5351de58..4f2790d1d1e53 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/ProjectOperator.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/ProjectOperator.java @@ -67,8 +67,13 @@ protected Page process(Page page) { } var block = page.getBlock(source); blocks[b++] = block; + block.incRef(); } - return page.newPageAndRelease(blocks); + int positionCount = page.getPositionCount(); + page.releaseBlocks(); + // Use positionCount explicitly to avoid re-computing - also, if the projection is empty, there may be + // no more blocks left to determine the positionCount from. + return new Page(positionCount, blocks); } @Override diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/BasicBlockTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/BasicBlockTests.java index 0a36617f35b18..2a49feeab9a30 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/BasicBlockTests.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/BasicBlockTests.java @@ -39,7 +39,6 @@ import static org.mockito.Mockito.when; public class BasicBlockTests extends ESTestCase { - final CircuitBreaker breaker = new MockBigArrays.LimitedBreaker("esql-test-breaker", ByteSizeValue.ofGb(1)); final BigArrays bigArrays = new MockBigArrays(PageCacheRecycler.NON_RECYCLING_INSTANCE, mockBreakerService(breaker)); final BlockFactory blockFactory = BlockFactory.getInstance(breaker, bigArrays); @@ -1004,6 +1003,12 @@ static void assertCannotDoubleRelease(Block block) { assertThat(ex.getMessage(), containsString("can't release already released block")); } + static void assertCannotReleaseIfVectorAlreadyReleased(Block block) { + var ex = expectThrows(IllegalStateException.class, () -> block.close()); + assertThat(ex.getMessage(), containsString("can't release block")); + assertThat(ex.getMessage(), containsString("containing already released vector")); + } + static void assertCannotReadFromPage(Page page) { var e = expectThrows(IllegalStateException.class, () -> page.getBlock(0)); assertThat(e.getMessage(), containsString("can't read released block")); @@ -1028,4 +1033,156 @@ static CircuitBreakerService mockBreakerService(CircuitBreaker breaker) { when(breakerService.getBreaker(CircuitBreaker.REQUEST)).thenReturn(breaker); return breakerService; } + + public void testRefCountingArrayBlock() { + Block block = randomArrayBlock(); + assertThat(breaker.getUsed(), greaterThan(0L)); + assertRefCountingBehavior(block); + assertThat(breaker.getUsed(), is(0L)); + } + + public void testRefCountingConstantNullBlock() { + Block block = blockFactory.newConstantNullBlock(10); + assertThat(breaker.getUsed(), greaterThan(0L)); + assertRefCountingBehavior(block); + assertThat(breaker.getUsed(), is(0L)); + } + + public void testRefCountingDocBlock() { + int positionCount = randomIntBetween(0, 100); + DocBlock block = new DocVector(intVector(positionCount), intVector(positionCount), intVector(positionCount), true).asBlock(); + assertThat(breaker.getUsed(), greaterThan(0L)); + assertRefCountingBehavior(block); + assertThat(breaker.getUsed(), is(0L)); + } + + public void testRefCountingVectorBlock() { + Block block = randomNonDocVector().asBlock(); + assertThat(breaker.getUsed(), greaterThan(0L)); + assertRefCountingBehavior(block); + assertThat(breaker.getUsed(), is(0L)); + } + + // Take a block with exactly 1 reference and assert that ref counting works fine. + static void assertRefCountingBehavior(Block b) { + assertTrue(b.hasReferences()); + int numShallowCopies = randomIntBetween(0, 15); + for (int i = 0; i < numShallowCopies; i++) { + if (randomBoolean()) { + b.incRef(); + } else { + assertTrue(b.tryIncRef()); + } + } + + for (int i = 0; i < numShallowCopies; i++) { + if (randomBoolean()) { + b.close(); + } else { + // closing and decRef'ing must be equivalent + assertFalse(b.decRef()); + } + assertTrue(b.hasReferences()); + } + + if (randomBoolean()) { + b.close(); + } else { + assertTrue(b.decRef()); + } + + assertFalse(b.hasReferences()); + assertFalse(b.tryIncRef()); + + expectThrows(IllegalStateException.class, b::close); + expectThrows(IllegalStateException.class, b::incRef); + } + + public void testReleasedVectorInvalidatesBlockState() { + Vector vector = randomNonDocVector(); + Block block = vector.asBlock(); + + int numRefs = randomIntBetween(1, 10); + for (int i = 0; i < numRefs - 1; i++) { + block.incRef(); + } + + vector.close(); + assertEquals(false, block.tryIncRef()); + expectThrows(IllegalStateException.class, block::close); + expectThrows(IllegalStateException.class, block::incRef); + } + + public void testReleasedDocVectorInvalidatesBlockState() { + int positionCount = randomIntBetween(0, 100); + DocVector vector = new DocVector(intVector(positionCount), intVector(positionCount), intVector(positionCount), true); + DocBlock block = vector.asBlock(); + + int numRefs = randomIntBetween(1, 10); + for (int i = 0; i < numRefs - 1; i++) { + block.incRef(); + } + + vector.close(); + assertEquals(false, block.tryIncRef()); + expectThrows(IllegalStateException.class, block::close); + expectThrows(IllegalStateException.class, block::incRef); + } + + private IntVector intVector(int positionCount) { + return blockFactory.newIntArrayVector(IntStream.range(0, positionCount).toArray(), positionCount); + } + + private Vector randomNonDocVector() { + int positionCount = randomIntBetween(0, 100); + int vectorType = randomIntBetween(0, 4); + + return switch (vectorType) { + case 0 -> blockFactory.newConstantBooleanVector(true, positionCount); + case 1 -> blockFactory.newConstantBytesRefVector(new BytesRef(), positionCount); + case 2 -> blockFactory.newConstantDoubleVector(1.0, positionCount); + case 3 -> blockFactory.newConstantIntVector(1, positionCount); + default -> blockFactory.newConstantLongVector(1L, positionCount); + }; + } + + private Block randomArrayBlock() { + int positionCount = randomIntBetween(0, 100); + int arrayType = randomIntBetween(0, 4); + + return switch (arrayType) { + case 0 -> { + boolean[] values = new boolean[positionCount]; + Arrays.fill(values, true); + + yield blockFactory.newBooleanArrayBlock(values, positionCount, new int[] {}, new BitSet(), randomOrdering()); + } + case 1 -> { + BytesRefArray values = new BytesRefArray(positionCount, BigArrays.NON_RECYCLING_INSTANCE); + for (int i = 0; i < positionCount; i++) { + values.append(new BytesRef(randomByteArrayOfLength(between(1, 20)))); + } + + yield blockFactory.newBytesRefArrayBlock(values, positionCount, new int[] {}, new BitSet(), randomOrdering()); + } + case 2 -> { + double[] values = new double[positionCount]; + Arrays.fill(values, 1.0); + + yield blockFactory.newDoubleArrayBlock(values, positionCount, new int[] {}, new BitSet(), randomOrdering()); + } + case 3 -> { + int[] values = new int[positionCount]; + Arrays.fill(values, 1); + + yield blockFactory.newIntArrayBlock(values, positionCount, new int[] {}, new BitSet(), randomOrdering()); + } + default -> { + long[] values = new long[positionCount]; + Arrays.fill(values, 1L); + + yield blockFactory.newLongArrayBlock(values, positionCount, new int[] {}, new BitSet(), randomOrdering()); + } + }; + } } diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/BasicPageTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/BasicPageTests.java index 23a257e7afbbe..25cd9ed5b9fe5 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/BasicPageTests.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/BasicPageTests.java @@ -205,16 +205,6 @@ public void testPageMultiRelease() { page.releaseBlocks(); } - public void testNewPageAndRelease() { - int positions = randomInt(1024); - var blockA = new IntArrayVector(IntStream.range(0, positions).toArray(), positions).asBlock(); - var blockB = new IntArrayVector(IntStream.range(0, positions).toArray(), positions).asBlock(); - Page page = new Page(blockA, blockB); - Page newPage = page.newPageAndRelease(blockA); - assertThat(blockA.isReleased(), is(false)); - assertThat(blockB.isReleased(), is(true)); - } - BytesRefArray bytesRefArrayOf(String... values) { var array = new BytesRefArray(values.length, bigArrays); Arrays.stream(values).map(BytesRef::new).forEach(array::append); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/LocalExecutionPlanner.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/LocalExecutionPlanner.java index 9a76bc0865865..c749f505b86a7 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/LocalExecutionPlanner.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/LocalExecutionPlanner.java @@ -335,8 +335,10 @@ private static Function alignPageToAttributes(List attrs, var blocks = new Block[mappedPosition.length]; for (int i = 0; i < blocks.length; i++) { blocks[i] = p.getBlock(mappedPosition[i]); + blocks[i].incRef(); } - return p.newPageAndRelease(blocks); + p.releaseBlocks(); + return new Page(blocks); } : Function.identity(); return transformer; @@ -360,11 +362,10 @@ private PhysicalOperation planExchangeSink(ExchangeSinkExec exchangeSink, LocalE // the outputs are going to be similar except for the bool "seen" flags which are added in below List blocks = new ArrayList<>(asList(localExec.supplier().get())); if (blocks.size() > 0) { - Block boolBlock = BooleanBlock.newConstantBlockWith(true, 1); for (int i = 0, s = output.size(); i < s; i++) { var out = output.get(i); if (out.dataType() == DataTypes.BOOLEAN) { - blocks.add(i, boolBlock); + blocks.add(i, BooleanBlock.newConstantBlockWith(true, 1)); } } } From 3d03f82d1aea148233ae2fb1ced2f51e1146b4a1 Mon Sep 17 00:00:00 2001 From: David Turner Date: Mon, 20 Nov 2023 13:10:51 +0000 Subject: [PATCH 17/28] AwaitsFix for #102255 --- .../elasticsearch/versioning/ConcurrentSeqNoVersioningIT.java | 1 + 1 file changed, 1 insertion(+) diff --git a/server/src/internalClusterTest/java/org/elasticsearch/versioning/ConcurrentSeqNoVersioningIT.java b/server/src/internalClusterTest/java/org/elasticsearch/versioning/ConcurrentSeqNoVersioningIT.java index cf59f5a8f6f3e..0f9b63f2757cf 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/versioning/ConcurrentSeqNoVersioningIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/versioning/ConcurrentSeqNoVersioningIT.java @@ -118,6 +118,7 @@ public class ConcurrentSeqNoVersioningIT extends AbstractDisruptionTestCase { // multiple threads doing CAS updates. // Wait up to 1 minute (+10s in thread to ensure it does not time out) for threads to complete previous round before initiating next // round. + @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/102255") public void testSeqNoCASLinearizability() { final int disruptTimeSeconds = scaledRandomIntBetween(1, 8); From 21d63ce0c559e6eb6277d9a549e6bcd2b0d4ce82 Mon Sep 17 00:00:00 2001 From: David Turner Date: Mon, 20 Nov 2023 13:27:12 +0000 Subject: [PATCH 18/28] AwaitsFix for #102381 --- .../xpack/ml/integration/TestFeatureLicenseTrackingIT.java | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/TestFeatureLicenseTrackingIT.java b/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/TestFeatureLicenseTrackingIT.java index 4ad4dfdbaccf8..f7bf94e0479e8 100644 --- a/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/TestFeatureLicenseTrackingIT.java +++ b/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/TestFeatureLicenseTrackingIT.java @@ -118,6 +118,7 @@ public void testFeatureTrackingAnomalyJob() throws Exception { }); } + @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/102381") public void testFeatureTrackingInferenceModelPipeline() throws Exception { String modelId = "test-load-models-classification-license-tracking"; Map oneHotEncoding = new HashMap<>(); From 4567d397fa3b20b1d57f596bd3ee2b046482fb8b Mon Sep 17 00:00:00 2001 From: Kathleen DeRusso Date: Mon, 20 Nov 2023 08:56:26 -0500 Subject: [PATCH 19/28] Clarify text expansion query docs to not suggest enabling track_total_hits for performance (#102102) --- .../query-dsl/text-expansion-query.asciidoc | 30 +------ .../search-your-data/search-api.asciidoc | 11 +++ .../semantic-search-elser.asciidoc | 79 ++++++++++--------- 3 files changed, 55 insertions(+), 65 deletions(-) diff --git a/docs/reference/query-dsl/text-expansion-query.asciidoc b/docs/reference/query-dsl/text-expansion-query.asciidoc index d15fd40846529..e924cc05376d9 100644 --- a/docs/reference/query-dsl/text-expansion-query.asciidoc +++ b/docs/reference/query-dsl/text-expansion-query.asciidoc @@ -78,29 +78,7 @@ GET my-index/_search ---- // TEST[skip: TBD] -[discrete] -[[optimizing-text-expansion]] -=== Optimizing the search performance of the text_expansion query - -https://www.elastic.co/blog/faster-retrieval-of-top-hits-in-elasticsearch-with-block-max-wand[Max WAND] -is an optimization technique used by {es} to skip documents that cannot score -competitively against the current best matching documents. However, the tokens -generated by the ELSER model don't work well with the Max WAND optimization. -Consequently, enabling Max WAND can actually increase query latency for -`text_expansion`. For datasets of a significant size, disabling Max -WAND leads to lower query latencies. - -Max WAND is controlled by the -<> query parameter. Setting track_total_hits -to true forces {es} to consider all documents, resulting in lower query -latencies for the `text_expansion` query. However, other {es} queries run slower -when Max WAND is disabled. - -If you are combining the `text_expansion` query with standard text queries in a -compound search, it is recommended to measure the query performance before -deciding which setting to use. - -NOTE: The `track_total_hits` option applies to all queries in the search request -and may be optimal for some queries but not for others. Take into account the -characteristics of all your queries to determine the most suitable -configuration. +[NOTE] +==== +Depending on your data, the text expansion query may be faster with `track_total_hits: false`. +==== diff --git a/docs/reference/search/search-your-data/search-api.asciidoc b/docs/reference/search/search-your-data/search-api.asciidoc index f3e271918b9b2..496812a0cedb4 100644 --- a/docs/reference/search/search-your-data/search-api.asciidoc +++ b/docs/reference/search/search-your-data/search-api.asciidoc @@ -440,6 +440,17 @@ GET my-index-000001/_search Finally you can force an accurate count by setting `"track_total_hits"` to `true` in the request. +[TIP] +========================================= +The `track_total_hits` parameter allows you to trade hit count accuracy for performance. +In general the lower the value of `track_total_hits` the faster the query will be, +with `false` returning the fastest results. +Setting `track_total_hits` to true will cause {es} to return exact hit counts, which could +hurt query performance because it disables the +https://www.elastic.co/blog/faster-retrieval-of-top-hits-in-elasticsearch-with-block-max-wand[Max WAND] +optimization. +========================================= + [discrete] [[quickly-check-for-matching-docs]] === Quickly check for matching docs diff --git a/docs/reference/search/search-your-data/semantic-search-elser.asciidoc b/docs/reference/search/search-your-data/semantic-search-elser.asciidoc index 164beb221cd4f..0bee9533cd358 100644 --- a/docs/reference/search/search-your-data/semantic-search-elser.asciidoc +++ b/docs/reference/search/search-your-data/semantic-search-elser.asciidoc @@ -45,7 +45,7 @@ you must provide suitably sized nodes yourself. First, the mapping of the destination index - the index that contains the tokens that the model created based on your text - must be created. The destination index must have a field with the -<> or <> field +<> or <> field type to index the ELSER output. NOTE: ELSER output must be ingested into a field with the `sparse_vector` or @@ -72,11 +72,11 @@ PUT my-index } ---- // TEST[skip:TBD] -<1> The name of the field to contain the generated tokens. It must be refrenced +<1> The name of the field to contain the generated tokens. It must be refrenced in the {infer} pipeline configuration in the next step. <2> The field to contain the tokens is a `sparse_vector` field. -<3> The name of the field from which to create the sparse vector representation. -In this example, the name of the field is `content`. It must be referenced in the +<3> The name of the field from which to create the sparse vector representation. +In this example, the name of the field is `content`. It must be referenced in the {infer} pipeline configuration in the next step. <4> The field type which is text in this example. @@ -93,24 +93,24 @@ that is being ingested in the pipeline. [source,console] ---- -PUT _ingest/pipeline/elser-v2-test -{ - "processors": [ - { - "inference": { - "model_id": ".elser_model_2", - "input_output": [ <1> - { - "input_field": "content", - "output_field": "content_embedding" - } - ] - } - } - ] +PUT _ingest/pipeline/elser-v2-test +{ + "processors": [ + { + "inference": { + "model_id": ".elser_model_2", + "input_output": [ <1> + { + "input_field": "content", + "output_field": "content_embedding" + } + ] + } + } + ] } ---- -<1> Configuration object that defines the `input_field` for the {infer} process +<1> Configuration object that defines the `input_field` for the {infer} process and the `output_field` that will contain the {infer} results. //// @@ -137,8 +137,8 @@ https://github.com/elastic/stack-docs/blob/main/docs/en/stack/ml/nlp/data/msmarc Download the file and upload it to your cluster using the {kibana-ref}/connect-to-elasticsearch.html#upload-data-kibana[Data Visualizer] -in the {ml-app} UI. Assign the name `id` to the first column and `content` to -the second column. The index name is `test-data`. Once the upload is complete, +in the {ml-app} UI. Assign the name `id` to the first column and `content` to +the second column. The index name is `test-data`. Once the upload is complete, you can see an index named `test-data` with 182469 documents. @@ -184,9 +184,9 @@ follow the progress. [[text-expansion-query]] ==== Semantic search by using the `text_expansion` query -To perform semantic search, use the `text_expansion` query, and provide the -query text and the ELSER model ID. The example below uses the query text "How to -avoid muscle soreness after running?", the `content_embedding` field contains +To perform semantic search, use the `text_expansion` query, and provide the +query text and the ELSER model ID. The example below uses the query text "How to +avoid muscle soreness after running?", the `content_embedding` field contains the generated ELSER output: [source,console] @@ -208,9 +208,9 @@ GET my-index/_search The result is the top 10 documents that are closest in meaning to your query text from the `my-index` index sorted by their relevancy. The result also contains the extracted tokens for each of the relevant search results with their -weights. Tokens are learned associations capturing relevance, they are not -synonyms. To learn more about what tokens are, refer to -{ml-docs}/ml-nlp-elser.html#elser-tokens[this page]. It is possible to exclude +weights. Tokens are learned associations capturing relevance, they are not +synonyms. To learn more about what tokens are, refer to +{ml-docs}/ml-nlp-elser.html#elser-tokens[this page]. It is possible to exclude tokens from source, refer to <> to learn more. [source,consol-result] @@ -253,9 +253,6 @@ tokens from source, refer to <> to learn more. ---- // NOTCONSOLE -To learn about optimizing your `text_expansion` query, refer to -<>. - [discrete] [[text-expansion-compound-query]] @@ -281,7 +278,7 @@ GET my-index/_search "bool": { <1> "should": [ { - "text_expansion": { + "text_expansion": { "content_embedding": { "model_text": "How to avoid muscle soreness after running?", "model_id": ".elser_model_2", @@ -333,12 +330,12 @@ WARNING: Reindex uses the document source to populate the destination index. space-saving optimsation that should only be applied if you are certain that reindexing will not be required in the future! It's important to carefully consider this trade-off and make sure that excluding the ELSER terms from the -source aligns with your specific requirements and use case. Review the -<> and <> sections carefully to learn +source aligns with your specific requirements and use case. Review the +<> and <> sections carefully to learn more about the possible consequences of excluding the tokens from the `_source`. -The mapping that excludes `content_embedding` from the `_source` field can be -created by the following API call: +The mapping that excludes `content_embedding` from the `_source` field can be +created by the following API call: [source,console] ---- @@ -352,10 +349,10 @@ PUT my-index }, "properties": { "content_embedding": { - "type": "sparse_vector" + "type": "sparse_vector" }, - "content": { - "type": "text" + "content": { + "type": "text" } } } @@ -363,6 +360,10 @@ PUT my-index ---- // TEST[skip:TBD] +[NOTE] +==== +Depending on your data, the text expansion query may be faster with `track_total_hits: false`. +==== [discrete] [[further-reading]] From fd300cffcf67785da2a1330dc19597ec5333ec1c Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Mon, 20 Nov 2023 09:44:04 -0500 Subject: [PATCH 20/28] ESQL: Load more than one field at once (#102192) This modifies ESQL to load a list of fields at one time which is especially effective when loading from stored fields or _source because it allows visiting the stored fields one time. Part of #101322 --- .../operator/ValuesSourceReaderBenchmark.java | 197 +++- docs/changelog/102192.yaml | 5 + .../extras/MatchOnlyTextFieldMapper.java | 4 +- .../mapper/extras/ScaledFloatFieldMapper.java | 6 +- .../index/mapper/BlockDocValuesReader.java | 551 +++++----- .../index/mapper/BlockLoader.java | 304 +++++- ...BlockLoaderStoredFieldsFromLeafLoader.java | 54 + .../index/mapper/BlockSourceReader.java | 366 ++++--- .../index/mapper/BlockStoredFieldsReader.java | 156 +-- .../index/mapper/BooleanFieldMapper.java | 4 +- .../BooleanScriptBlockDocValuesReader.java | 34 +- .../index/mapper/BooleanScriptFieldType.java | 2 +- .../index/mapper/DateFieldMapper.java | 4 +- .../DateScriptBlockDocValuesReader.java | 35 +- .../index/mapper/DateScriptFieldType.java | 2 +- .../DoubleScriptBlockDocValuesReader.java | 35 +- .../index/mapper/DoubleScriptFieldType.java | 2 +- .../index/mapper/IndexFieldMapper.java | 37 +- .../index/mapper/IpFieldMapper.java | 2 +- .../mapper/IpScriptBlockDocValuesReader.java | 35 +- .../index/mapper/IpScriptFieldType.java | 2 +- .../index/mapper/KeywordFieldMapper.java | 6 +- .../KeywordScriptBlockDocValuesReader.java | 35 +- .../index/mapper/KeywordScriptFieldType.java | 2 +- .../LongScriptBlockDocValuesReader.java | 35 +- .../index/mapper/LongScriptFieldType.java | 2 +- .../index/mapper/NumberFieldMapper.java | 30 +- .../index/mapper/ProvidedIdFieldMapper.java | 2 +- .../index/mapper/TextFieldMapper.java | 17 +- .../mapper/TsidExtractingIdFieldMapper.java | 2 +- .../index/mapper/VersionFieldMapper.java | 2 +- .../search/fetch/StoredFieldsSpec.java | 3 + .../mapper/BooleanScriptFieldTypeTests.java | 4 +- .../mapper/DateScriptFieldTypeTests.java | 7 +- .../mapper/DoubleScriptFieldTypeTests.java | 4 +- .../index/mapper/IpScriptFieldTypeTests.java | 4 +- .../mapper/KeywordScriptFieldTypeTests.java | 7 +- .../mapper/LongScriptFieldTypeTests.java | 4 +- .../index/mapper/TextFieldMapperTests.java | 6 + .../AbstractScriptFieldTypeTestCase.java | 15 +- .../index/mapper/MapperTestCase.java | 41 +- .../elasticsearch/index/mapper/TestBlock.java | 202 ++-- .../compute/lucene/BlockReaderFactories.java | 46 +- .../lucene/ValuesSourceReaderOperator.java | 313 ++++-- .../operator/OrdinalsGroupingOperator.java | 37 +- .../elasticsearch/compute/OperatorTests.java | 6 +- .../ValuesSourceReaderOperatorTests.java | 954 ++++++++++++++++-- .../resources/rest-api-spec/test/20_aggs.yml | 112 +- .../rest-api-spec/test/50_index_patterns.yml | 30 +- .../xpack/esql/action/EsqlActionTaskIT.java | 5 +- .../esql/enrich/EnrichLookupService.java | 13 +- .../planner/EsPhysicalOperationProviders.java | 19 +- .../mapper/ConstantKeywordFieldMapper.java | 40 +- .../ConstantKeywordFieldMapperTests.java | 34 +- .../unsignedlong/UnsignedLongFieldMapper.java | 6 +- .../VersionStringFieldMapper.java | 2 +- .../wildcard/mapper/WildcardFieldMapper.java | 4 +- 57 files changed, 2727 insertions(+), 1161 deletions(-) create mode 100644 docs/changelog/102192.yaml create mode 100644 server/src/main/java/org/elasticsearch/index/mapper/BlockLoaderStoredFieldsFromLeafLoader.java diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/ValuesSourceReaderBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/ValuesSourceReaderBenchmark.java index 9fa876a00c35c..40edc0b8b9b7f 100644 --- a/benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/ValuesSourceReaderBenchmark.java +++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/ValuesSourceReaderBenchmark.java @@ -8,8 +8,11 @@ package org.elasticsearch.benchmark.compute.operator; +import org.apache.lucene.document.FieldType; import org.apache.lucene.document.NumericDocValuesField; +import org.apache.lucene.document.StoredField; import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexWriterConfig; @@ -19,6 +22,7 @@ import org.apache.lucene.store.Directory; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.NumericUtils; +import org.elasticsearch.common.lucene.Lucene; import org.elasticsearch.compute.data.BlockFactory; import org.elasticsearch.compute.data.BytesRefBlock; import org.elasticsearch.compute.data.BytesRefVector; @@ -30,14 +34,16 @@ import org.elasticsearch.compute.data.LongBlock; import org.elasticsearch.compute.data.LongVector; import org.elasticsearch.compute.data.Page; -import org.elasticsearch.compute.lucene.BlockReaderFactories; import org.elasticsearch.compute.lucene.LuceneSourceOperator; import org.elasticsearch.compute.lucene.ValuesSourceReaderOperator; import org.elasticsearch.compute.operator.topn.TopNOperator; import org.elasticsearch.core.IOUtils; +import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.mapper.BlockLoader; import org.elasticsearch.index.mapper.KeywordFieldMapper; +import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.NumberFieldMapper; +import org.elasticsearch.search.lookup.SearchLookup; import org.openjdk.jmh.annotations.Benchmark; import org.openjdk.jmh.annotations.BenchmarkMode; import org.openjdk.jmh.annotations.Fork; @@ -56,7 +62,9 @@ import java.util.ArrayList; import java.util.Iterator; import java.util.List; +import java.util.Map; import java.util.PrimitiveIterator; +import java.util.Set; import java.util.concurrent.TimeUnit; import java.util.stream.IntStream; @@ -93,18 +101,113 @@ public class ValuesSourceReaderBenchmark { } } - private static BlockLoader blockLoader(String name) { + private static List fields(String name) { return switch (name) { - case "long" -> numericBlockLoader(name, NumberFieldMapper.NumberType.LONG); - case "int" -> numericBlockLoader(name, NumberFieldMapper.NumberType.INTEGER); - case "double" -> numericBlockLoader(name, NumberFieldMapper.NumberType.DOUBLE); - case "keyword" -> new KeywordFieldMapper.KeywordFieldType(name).blockLoader(null); - default -> throw new IllegalArgumentException("can't read [" + name + "]"); + case "3_stored_keywords" -> List.of( + new ValuesSourceReaderOperator.FieldInfo("keyword_1", List.of(blockLoader("stored_keyword_1"))), + new ValuesSourceReaderOperator.FieldInfo("keyword_2", List.of(blockLoader("stored_keyword_2"))), + new ValuesSourceReaderOperator.FieldInfo("keyword_3", List.of(blockLoader("stored_keyword_3"))) + ); + default -> List.of(new ValuesSourceReaderOperator.FieldInfo(name, List.of(blockLoader(name)))); }; } - private static BlockLoader numericBlockLoader(String name, NumberFieldMapper.NumberType numberType) { - return new NumberFieldMapper.NumberFieldType(name, numberType).blockLoader(null); + enum Where { + DOC_VALUES, + SOURCE, + STORED; + } + + private static BlockLoader blockLoader(String name) { + Where where = Where.DOC_VALUES; + if (name.startsWith("stored_")) { + name = name.substring("stored_".length()); + where = Where.STORED; + } else if (name.startsWith("source_")) { + name = name.substring("source_".length()); + where = Where.SOURCE; + } + switch (name) { + case "long": + return numericBlockLoader(name, where, NumberFieldMapper.NumberType.LONG); + case "int": + return numericBlockLoader(name, where, NumberFieldMapper.NumberType.INTEGER); + case "double": + return numericBlockLoader(name, where, NumberFieldMapper.NumberType.DOUBLE); + case "keyword": + name = "keyword_1"; + } + if (name.startsWith("keyword")) { + boolean syntheticSource = false; + FieldType ft = new FieldType(KeywordFieldMapper.Defaults.FIELD_TYPE); + switch (where) { + case DOC_VALUES: + break; + case SOURCE: + ft.setDocValuesType(DocValuesType.NONE); + break; + case STORED: + ft.setStored(true); + ft.setDocValuesType(DocValuesType.NONE); + syntheticSource = true; + break; + } + ft.freeze(); + return new KeywordFieldMapper.KeywordFieldType( + name, + ft, + Lucene.KEYWORD_ANALYZER, + Lucene.KEYWORD_ANALYZER, + Lucene.KEYWORD_ANALYZER, + new KeywordFieldMapper.Builder(name, IndexVersion.current()).docValues(ft.docValuesType() != DocValuesType.NONE), + syntheticSource + ).blockLoader(new MappedFieldType.BlockLoaderContext() { + @Override + public String indexName() { + return "benchmark"; + } + + @Override + public SearchLookup lookup() { + throw new UnsupportedOperationException(); + } + + @Override + public Set sourcePaths(String name) { + return Set.of(name); + } + }); + } + throw new IllegalArgumentException("can't read [" + name + "]"); + } + + private static BlockLoader numericBlockLoader(String name, Where where, NumberFieldMapper.NumberType numberType) { + boolean stored = false; + boolean docValues = true; + switch (where) { + case DOC_VALUES: + break; + case SOURCE: + stored = true; + docValues = false; + break; + case STORED: + throw new UnsupportedOperationException(); + } + return new NumberFieldMapper.NumberFieldType( + name, + numberType, + true, + stored, + docValues, + true, + null, + Map.of(), + null, + false, + null, + null + ).blockLoader(null); } /** @@ -122,7 +225,7 @@ private static BlockLoader numericBlockLoader(String name, NumberFieldMapper.Num @Param({ "in_order", "shuffled", "shuffled_singles" }) public String layout; - @Param({ "long", "int", "double", "keyword" }) + @Param({ "long", "int", "double", "keyword", "stored_keyword", "3_stored_keywords" }) public String name; private Directory directory; @@ -134,9 +237,9 @@ private static BlockLoader numericBlockLoader(String name, NumberFieldMapper.Num public void benchmark() { ValuesSourceReaderOperator op = new ValuesSourceReaderOperator( BlockFactory.getNonBreakingInstance(), - List.of(BlockReaderFactories.loaderToFactory(reader, blockLoader(name))), - 0, - name + fields(name), + List.of(reader), + 0 ); long sum = 0; for (Page page : pages) { @@ -160,7 +263,7 @@ public void benchmark() { sum += (long) values.getDouble(p); } } - case "keyword" -> { + case "keyword", "stored_keyword" -> { BytesRef scratch = new BytesRef(); BytesRefVector values = op.getOutput().getBlock(1).asVector(); for (int p = 0; p < values.getPositionCount(); p++) { @@ -170,21 +273,59 @@ public void benchmark() { sum += Integer.parseInt(r.utf8ToString()); } } + case "3_stored_keywords" -> { + BytesRef scratch = new BytesRef(); + Page out = op.getOutput(); + for (BytesRefVector values : new BytesRefVector[] { + out.getBlock(1).asVector(), + out.getBlock(2).asVector(), + out.getBlock(3).asVector() }) { + + for (int p = 0; p < values.getPositionCount(); p++) { + BytesRef r = values.getBytesRef(p, scratch); + r.offset++; + r.length--; + sum += Integer.parseInt(r.utf8ToString()); + } + } + } } } - long expected; - if (name.equals("keyword")) { - expected = 0; - for (int i = 0; i < INDEX_SIZE; i++) { - expected += i % 1000; - } - } else { - expected = INDEX_SIZE; - expected = expected * (expected - 1) / 2; + long expected = 0; + switch (name) { + case "keyword", "stored_keyword": + for (int i = 0; i < INDEX_SIZE; i++) { + expected += i % 1000; + } + break; + case "3_stored_keywords": + for (int i = 0; i < INDEX_SIZE; i++) { + expected += 3 * (i % 1000); + } + break; + default: + expected = INDEX_SIZE; + expected = expected * (expected - 1) / 2; } if (expected != sum) { throw new AssertionError("[" + layout + "][" + name + "] expected [" + expected + "] but was [" + sum + "]"); } + boolean foundStoredFieldLoader = false; + ValuesSourceReaderOperator.Status status = (ValuesSourceReaderOperator.Status) op.status(); + for (Map.Entry e : status.readersBuilt().entrySet()) { + if (e.getKey().indexOf("stored_fields") >= 0) { + foundStoredFieldLoader = true; + } + } + if (name.indexOf("stored") >= 0) { + if (foundStoredFieldLoader == false) { + throw new AssertionError("expected to use a stored field loader but only had: " + status.readersBuilt()); + } + } else { + if (foundStoredFieldLoader) { + throw new AssertionError("expected not to use a stored field loader but only had: " + status.readersBuilt()); + } + } } @Setup @@ -195,15 +336,23 @@ public void setup() throws IOException { private void setupIndex() throws IOException { directory = new ByteBuffersDirectory(); + FieldType keywordFieldType = new FieldType(KeywordFieldMapper.Defaults.FIELD_TYPE); + keywordFieldType.setStored(true); + keywordFieldType.freeze(); try (IndexWriter iw = new IndexWriter(directory, new IndexWriterConfig().setMergePolicy(NoMergePolicy.INSTANCE))) { for (int i = 0; i < INDEX_SIZE; i++) { String c = Character.toString('a' - ((i % 1000) % 26) + 26); iw.addDocument( List.of( new NumericDocValuesField("long", i), + new StoredField("long", i), new NumericDocValuesField("int", i), + new StoredField("int", i), new NumericDocValuesField("double", NumericUtils.doubleToSortableLong(i)), - new KeywordFieldMapper.KeywordField("keyword", new BytesRef(c + i % 1000), KeywordFieldMapper.Defaults.FIELD_TYPE) + new StoredField("double", (double) i), + new KeywordFieldMapper.KeywordField("keyword_1", new BytesRef(c + i % 1000), keywordFieldType), + new KeywordFieldMapper.KeywordField("keyword_2", new BytesRef(c + i % 1000), keywordFieldType), + new KeywordFieldMapper.KeywordField("keyword_3", new BytesRef(c + i % 1000), keywordFieldType) ) ); if (i % COMMIT_INTERVAL == 0) { diff --git a/docs/changelog/102192.yaml b/docs/changelog/102192.yaml new file mode 100644 index 0000000000000..531aa943c9e36 --- /dev/null +++ b/docs/changelog/102192.yaml @@ -0,0 +1,5 @@ +pr: 102192 +summary: "ESQL: Load more than one field at once" +area: ES|QL +type: enhancement +issues: [] diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapper.java index ee04346591009..161cb1674a7b9 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapper.java @@ -324,9 +324,9 @@ public Query phrasePrefixQuery(TokenStream stream, int slop, int maxExpansions, @Override public BlockLoader blockLoader(BlockLoaderContext blContext) { if (textFieldType.isSyntheticSource()) { - return BlockStoredFieldsReader.bytesRefsFromStrings(storedFieldNameForSyntheticSource()); + return new BlockStoredFieldsReader.BytesFromStringsBlockLoader(storedFieldNameForSyntheticSource()); } - return BlockSourceReader.bytesRefs(SourceValueFetcher.toString(blContext.sourcePaths(name()))); + return new BlockSourceReader.BytesRefsBlockLoader(SourceValueFetcher.toString(blContext.sourcePaths(name()))); } @Override diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java index abed23621d5e9..b35fb09c2d053 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java @@ -310,13 +310,13 @@ public Query rangeQuery( public BlockLoader blockLoader(BlockLoaderContext blContext) { if (indexMode == IndexMode.TIME_SERIES && metricType == TimeSeriesParams.MetricType.COUNTER) { // Counters are not supported by ESQL so we load them in null - return BlockDocValuesReader.nulls(); + return BlockLoader.CONSTANT_NULLS; } if (hasDocValues()) { double scalingFactorInverse = 1d / scalingFactor; - return BlockDocValuesReader.doubles(name(), l -> l * scalingFactorInverse); + return new BlockDocValuesReader.DoublesBlockLoader(name(), l -> l * scalingFactorInverse); } - return BlockSourceReader.doubles(sourceValueFetcher(blContext.sourcePaths(name()))); + return new BlockSourceReader.DoublesBlockLoader(sourceValueFetcher(blContext.sourcePaths(name()))); } @Override diff --git a/server/src/main/java/org/elasticsearch/index/mapper/BlockDocValuesReader.java b/server/src/main/java/org/elasticsearch/index/mapper/BlockDocValuesReader.java index 90a295e5a25f2..6e572eceeafc4 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/BlockDocValuesReader.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/BlockDocValuesReader.java @@ -8,6 +8,7 @@ package org.elasticsearch.index.mapper; +import org.apache.lucene.index.BinaryDocValues; import org.apache.lucene.index.DocValues; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.NumericDocValues; @@ -15,171 +16,94 @@ import org.apache.lucene.index.SortedNumericDocValues; import org.apache.lucene.index.SortedSetDocValues; import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.UnicodeUtil; -import org.elasticsearch.core.CheckedFunction; -import org.elasticsearch.index.fielddata.SortedBinaryDocValues; +import org.elasticsearch.common.io.stream.ByteArrayStreamInput; +import org.elasticsearch.index.mapper.BlockLoader.BlockFactory; import org.elasticsearch.index.mapper.BlockLoader.BooleanBuilder; import org.elasticsearch.index.mapper.BlockLoader.Builder; -import org.elasticsearch.index.mapper.BlockLoader.BuilderFactory; import org.elasticsearch.index.mapper.BlockLoader.BytesRefBuilder; import org.elasticsearch.index.mapper.BlockLoader.Docs; import org.elasticsearch.index.mapper.BlockLoader.DoubleBuilder; import org.elasticsearch.index.mapper.BlockLoader.IntBuilder; import org.elasticsearch.index.mapper.BlockLoader.LongBuilder; +import org.elasticsearch.search.fetch.StoredFieldsSpec; import java.io.IOException; /** * A reader that supports reading doc-values from a Lucene segment in Block fashion. */ -public abstract class BlockDocValuesReader { - public interface Factory { - BlockDocValuesReader build(int segment) throws IOException; - - boolean supportsOrdinals(); - - SortedSetDocValues ordinals(int segment) throws IOException; - } - - protected final Thread creationThread; +public abstract class BlockDocValuesReader implements BlockLoader.AllReader { + private final Thread creationThread; public BlockDocValuesReader() { this.creationThread = Thread.currentThread(); } - /** - * Returns the current doc that this reader is on. - */ - public abstract int docID(); - - /** - * The {@link BlockLoader.Builder} for data of this type. - */ - public abstract Builder builder(BuilderFactory factory, int expectedCount); - - /** - * Reads the values of the given documents specified in the input block - */ - public abstract BlockLoader.Block readValues(BuilderFactory factory, Docs docs) throws IOException; - - /** - * Reads the values of the given document into the builder - */ - public abstract void readValuesFromSingleDoc(int docId, Builder builder) throws IOException; + protected abstract int docId(); /** * Checks if the reader can be used to read a range documents starting with the given docID by the current thread. */ - public static boolean canReuse(BlockDocValuesReader reader, int startingDocID) { - return reader != null && reader.creationThread == Thread.currentThread() && reader.docID() <= startingDocID; + @Override + public final boolean canReuse(int startingDocID) { + return creationThread == Thread.currentThread() && docId() <= startingDocID; } - public static BlockLoader booleans(String fieldName) { - return context -> { - SortedNumericDocValues docValues = DocValues.getSortedNumeric(context.reader(), fieldName); - NumericDocValues singleton = DocValues.unwrapSingleton(docValues); - if (singleton != null) { - return new SingletonBooleans(singleton); - } - return new Booleans(docValues); - }; - } + @Override + public abstract String toString(); - public static BlockLoader bytesRefsFromOrds(String fieldName) { - return new BlockLoader() { - @Override - public BlockDocValuesReader reader(LeafReaderContext context) throws IOException { - SortedSetDocValues docValues = ordinals(context); - SortedDocValues singleton = DocValues.unwrapSingleton(docValues); - if (singleton != null) { - return new SingletonOrdinals(singleton); - } - return new Ordinals(docValues); - } + public abstract static class DocValuesBlockLoader implements BlockLoader { + public abstract AllReader reader(LeafReaderContext context) throws IOException; - @Override - public boolean supportsOrdinals() { - return true; - } + @Override + public final ColumnAtATimeReader columnAtATimeReader(LeafReaderContext context) throws IOException { + return reader(context); + } - @Override - public SortedSetDocValues ordinals(LeafReaderContext context) throws IOException { - return DocValues.getSortedSet(context.reader(), fieldName); - } - }; - } + @Override + public final RowStrideReader rowStrideReader(LeafReaderContext context) throws IOException { + return reader(context); + } - /** - * Load {@link BytesRef} values from doc values. Prefer {@link #bytesRefsFromOrds} if - * doc values are indexed with ordinals because that's generally much faster. It's - * possible to use this with field data, but generally should be avoided because field - * data has higher per invocation overhead. - */ - public static BlockLoader bytesRefsFromDocValues(CheckedFunction fieldData) { - return context -> new Bytes(fieldData.apply(context)); - } + @Override + public final StoredFieldsSpec rowStrideStoredFieldSpec() { + return StoredFieldsSpec.NO_REQUIREMENTS; + } - /** - * Convert from the stored {@link long} into the {@link double} to load. - * Sadly, this will go megamorphic pretty quickly and slow us down, - * but it gets the job done for now. - */ - public interface ToDouble { - double convert(long v); - } + @Override + public boolean supportsOrdinals() { + return false; + } - /** - * Load {@code double} values from doc values. - */ - public static BlockLoader doubles(String fieldName, ToDouble toDouble) { - return context -> { - SortedNumericDocValues docValues = DocValues.getSortedNumeric(context.reader(), fieldName); - NumericDocValues singleton = DocValues.unwrapSingleton(docValues); - if (singleton != null) { - return new SingletonDoubles(singleton, toDouble); - } - return new Doubles(docValues, toDouble); - }; + @Override + public SortedSetDocValues ordinals(LeafReaderContext context) throws IOException { + throw new UnsupportedOperationException(); + } } - /** - * Load {@code int} values from doc values. - */ - public static BlockLoader ints(String fieldName) { - return context -> { - SortedNumericDocValues docValues = DocValues.getSortedNumeric(context.reader(), fieldName); - NumericDocValues singleton = DocValues.unwrapSingleton(docValues); - if (singleton != null) { - return new SingletonInts(singleton); - } - return new Ints(docValues); - }; - } + public static class LongsBlockLoader extends DocValuesBlockLoader { + private final String fieldName; - /** - * Load a block of {@code long}s from doc values. - */ - public static BlockLoader longs(String fieldName) { - return context -> { + public LongsBlockLoader(String fieldName) { + this.fieldName = fieldName; + } + + @Override + public Builder builder(BlockFactory factory, int expectedCount) { + return factory.longs(expectedCount); + } + + @Override + public AllReader reader(LeafReaderContext context) throws IOException { SortedNumericDocValues docValues = DocValues.getSortedNumeric(context.reader(), fieldName); NumericDocValues singleton = DocValues.unwrapSingleton(docValues); if (singleton != null) { return new SingletonLongs(singleton); } return new Longs(docValues); - }; - } - - /** - * Load blocks with only null. - */ - public static BlockLoader nulls() { - return context -> new Nulls(); + } } - @Override - public abstract String toString(); - private static class SingletonLongs extends BlockDocValuesReader { private final NumericDocValues numericDocValues; @@ -188,13 +112,8 @@ private static class SingletonLongs extends BlockDocValuesReader { } @Override - public BlockLoader.LongBuilder builder(BuilderFactory factory, int expectedCount) { - return factory.longsFromDocValues(expectedCount); - } - - @Override - public BlockLoader.Block readValues(BuilderFactory factory, Docs docs) throws IOException { - try (BlockLoader.LongBuilder builder = builder(factory, docs.count())) { + public BlockLoader.Block read(BlockFactory factory, Docs docs) throws IOException { + try (BlockLoader.LongBuilder builder = factory.longsFromDocValues(docs.count())) { int lastDoc = -1; for (int i = 0; i < docs.count(); i++) { int doc = docs.get(i); @@ -213,7 +132,7 @@ public BlockLoader.Block readValues(BuilderFactory factory, Docs docs) throws IO } @Override - public void readValuesFromSingleDoc(int docId, Builder builder) throws IOException { + public void read(int docId, BlockLoader.StoredFields storedFields, Builder builder) throws IOException { BlockLoader.LongBuilder blockBuilder = (BlockLoader.LongBuilder) builder; if (numericDocValues.advanceExact(docId)) { blockBuilder.appendLong(numericDocValues.longValue()); @@ -223,13 +142,13 @@ public void readValuesFromSingleDoc(int docId, Builder builder) throws IOExcepti } @Override - public int docID() { + public int docId() { return numericDocValues.docID(); } @Override public String toString() { - return "SingletonLongs"; + return "BlockDocValuesReader.SingletonLongs"; } } @@ -242,13 +161,8 @@ private static class Longs extends BlockDocValuesReader { } @Override - public BlockLoader.LongBuilder builder(BuilderFactory factory, int expectedCount) { - return factory.longsFromDocValues(expectedCount); - } - - @Override - public BlockLoader.Block readValues(BuilderFactory factory, Docs docs) throws IOException { - try (BlockLoader.LongBuilder builder = builder(factory, docs.count())) { + public BlockLoader.Block read(BlockFactory factory, Docs docs) throws IOException { + try (BlockLoader.LongBuilder builder = factory.longsFromDocValues(docs.count())) { for (int i = 0; i < docs.count(); i++) { int doc = docs.get(i); if (doc < this.docID) { @@ -261,7 +175,7 @@ public BlockLoader.Block readValues(BuilderFactory factory, Docs docs) throws IO } @Override - public void readValuesFromSingleDoc(int docId, Builder builder) throws IOException { + public void read(int docId, BlockLoader.StoredFields storedFields, Builder builder) throws IOException { read(docId, (LongBuilder) builder); } @@ -284,14 +198,37 @@ private void read(int doc, LongBuilder builder) throws IOException { } @Override - public int docID() { + public int docId() { // There is a .docID on the numericDocValues but it is often not implemented. return docID; } @Override public String toString() { - return "Longs"; + return "BlockDocValuesReader.Longs"; + } + } + + public static class IntsBlockLoader extends DocValuesBlockLoader { + private final String fieldName; + + public IntsBlockLoader(String fieldName) { + this.fieldName = fieldName; + } + + @Override + public Builder builder(BlockFactory factory, int expectedCount) { + return factory.ints(expectedCount); + } + + @Override + public AllReader reader(LeafReaderContext context) throws IOException { + SortedNumericDocValues docValues = DocValues.getSortedNumeric(context.reader(), fieldName); + NumericDocValues singleton = DocValues.unwrapSingleton(docValues); + if (singleton != null) { + return new SingletonInts(singleton); + } + return new Ints(docValues); } } @@ -303,13 +240,8 @@ private static class SingletonInts extends BlockDocValuesReader { } @Override - public IntBuilder builder(BuilderFactory factory, int expectedCount) { - return factory.intsFromDocValues(expectedCount); - } - - @Override - public BlockLoader.Block readValues(BuilderFactory factory, Docs docs) throws IOException { - try (BlockLoader.IntBuilder builder = builder(factory, docs.count())) { + public BlockLoader.Block read(BlockFactory factory, Docs docs) throws IOException { + try (BlockLoader.IntBuilder builder = factory.intsFromDocValues(docs.count())) { int lastDoc = -1; for (int i = 0; i < docs.count(); i++) { int doc = docs.get(i); @@ -328,7 +260,7 @@ public BlockLoader.Block readValues(BuilderFactory factory, Docs docs) throws IO } @Override - public void readValuesFromSingleDoc(int docId, Builder builder) throws IOException { + public void read(int docId, BlockLoader.StoredFields storedFields, Builder builder) throws IOException { IntBuilder blockBuilder = (IntBuilder) builder; if (numericDocValues.advanceExact(docId)) { blockBuilder.appendInt(Math.toIntExact(numericDocValues.longValue())); @@ -338,13 +270,13 @@ public void readValuesFromSingleDoc(int docId, Builder builder) throws IOExcepti } @Override - public int docID() { + public int docId() { return numericDocValues.docID(); } @Override public String toString() { - return "SingletonInts"; + return "BlockDocValuesReader.SingletonInts"; } } @@ -357,13 +289,8 @@ private static class Ints extends BlockDocValuesReader { } @Override - public IntBuilder builder(BuilderFactory factory, int expectedCount) { - return factory.intsFromDocValues(expectedCount); - } - - @Override - public BlockLoader.Block readValues(BuilderFactory factory, Docs docs) throws IOException { - try (BlockLoader.IntBuilder builder = builder(factory, docs.count())) { + public BlockLoader.Block read(BlockFactory factory, Docs docs) throws IOException { + try (BlockLoader.IntBuilder builder = factory.intsFromDocValues(docs.count())) { for (int i = 0; i < docs.count(); i++) { int doc = docs.get(i); if (doc < this.docID) { @@ -376,7 +303,7 @@ public BlockLoader.Block readValues(BuilderFactory factory, Docs docs) throws IO } @Override - public void readValuesFromSingleDoc(int docId, Builder builder) throws IOException { + public void read(int docId, BlockLoader.StoredFields storedFields, Builder builder) throws IOException { read(docId, (IntBuilder) builder); } @@ -399,14 +326,48 @@ private void read(int doc, IntBuilder builder) throws IOException { } @Override - public int docID() { - // There is a .docID on on the numericDocValues but it is often not implemented. + public int docId() { + // There is a .docID on the numericDocValues but it is often not implemented. return docID; } @Override public String toString() { - return "Ints"; + return "BlockDocValuesReader.Ints"; + } + } + + /** + * Convert from the stored {@link long} into the {@link double} to load. + * Sadly, this will go megamorphic pretty quickly and slow us down, + * but it gets the job done for now. + */ + public interface ToDouble { + double convert(long v); + } + + public static class DoublesBlockLoader extends DocValuesBlockLoader { + private final String fieldName; + private final ToDouble toDouble; + + public DoublesBlockLoader(String fieldName, ToDouble toDouble) { + this.fieldName = fieldName; + this.toDouble = toDouble; + } + + @Override + public Builder builder(BlockFactory factory, int expectedCount) { + return factory.doubles(expectedCount); + } + + @Override + public AllReader reader(LeafReaderContext context) throws IOException { + SortedNumericDocValues docValues = DocValues.getSortedNumeric(context.reader(), fieldName); + NumericDocValues singleton = DocValues.unwrapSingleton(docValues); + if (singleton != null) { + return new SingletonDoubles(singleton, toDouble); + } + return new Doubles(docValues, toDouble); } } @@ -421,13 +382,8 @@ private static class SingletonDoubles extends BlockDocValuesReader { } @Override - public DoubleBuilder builder(BuilderFactory factory, int expectedCount) { - return factory.doublesFromDocValues(expectedCount); - } - - @Override - public BlockLoader.Block readValues(BuilderFactory factory, Docs docs) throws IOException { - try (BlockLoader.DoubleBuilder builder = builder(factory, docs.count())) { + public BlockLoader.Block read(BlockFactory factory, Docs docs) throws IOException { + try (BlockLoader.DoubleBuilder builder = factory.doublesFromDocValues(docs.count())) { int lastDoc = -1; for (int i = 0; i < docs.count(); i++) { int doc = docs.get(i); @@ -447,7 +403,7 @@ public BlockLoader.Block readValues(BuilderFactory factory, Docs docs) throws IO } @Override - public void readValuesFromSingleDoc(int docId, Builder builder) throws IOException { + public void read(int docId, BlockLoader.StoredFields storedFields, Builder builder) throws IOException { this.docID = docId; DoubleBuilder blockBuilder = (DoubleBuilder) builder; if (docValues.advanceExact(this.docID)) { @@ -458,13 +414,13 @@ public void readValuesFromSingleDoc(int docId, Builder builder) throws IOExcepti } @Override - public int docID() { + public int docId() { return docID; } @Override public String toString() { - return "SingletonDoubles"; + return "BlockDocValuesReader.SingletonDoubles"; } } @@ -479,13 +435,8 @@ private static class Doubles extends BlockDocValuesReader { } @Override - public DoubleBuilder builder(BuilderFactory factory, int expectedCount) { - return factory.doublesFromDocValues(expectedCount); - } - - @Override - public BlockLoader.Block readValues(BuilderFactory factory, Docs docs) throws IOException { - try (BlockLoader.DoubleBuilder builder = builder(factory, docs.count())) { + public BlockLoader.Block read(BlockFactory factory, Docs docs) throws IOException { + try (BlockLoader.DoubleBuilder builder = factory.doublesFromDocValues(docs.count())) { for (int i = 0; i < docs.count(); i++) { int doc = docs.get(i); if (doc < this.docID) { @@ -498,7 +449,7 @@ public BlockLoader.Block readValues(BuilderFactory factory, Docs docs) throws IO } @Override - public void readValuesFromSingleDoc(int docId, Builder builder) throws IOException { + public void read(int docId, BlockLoader.StoredFields storedFields, Builder builder) throws IOException { read(docId, (DoubleBuilder) builder); } @@ -521,13 +472,46 @@ private void read(int doc, DoubleBuilder builder) throws IOException { } @Override - public int docID() { + public int docId() { return docID; } @Override public String toString() { - return "Doubles"; + return "BlockDocValuesReader.Doubles"; + } + } + + public static class BytesRefsFromOrdsBlockLoader extends DocValuesBlockLoader { + private final String fieldName; + + public BytesRefsFromOrdsBlockLoader(String fieldName) { + this.fieldName = fieldName; + } + + @Override + public BytesRefBuilder builder(BlockFactory factory, int expectedCount) { + return factory.bytesRefs(expectedCount); + } + + @Override + public AllReader reader(LeafReaderContext context) throws IOException { + SortedSetDocValues docValues = ordinals(context); + SortedDocValues singleton = DocValues.unwrapSingleton(docValues); + if (singleton != null) { + return new SingletonOrdinals(singleton); + } + return new Ordinals(docValues); + } + + @Override + public boolean supportsOrdinals() { + return true; + } + + @Override + public SortedSetDocValues ordinals(LeafReaderContext context) throws IOException { + return DocValues.getSortedSet(context.reader(), fieldName); } } @@ -539,12 +523,7 @@ private static class SingletonOrdinals extends BlockDocValuesReader { } @Override - public BytesRefBuilder builder(BuilderFactory factory, int expectedCount) { - return factory.bytesRefsFromDocValues(expectedCount); - } - - @Override - public BlockLoader.Block readValues(BuilderFactory factory, Docs docs) throws IOException { + public BlockLoader.Block read(BlockFactory factory, Docs docs) throws IOException { try (BlockLoader.SingletonOrdinalsBuilder builder = factory.singletonOrdinalsBuilder(ordinals, docs.count())) { for (int i = 0; i < docs.count(); i++) { int doc = docs.get(i); @@ -562,8 +541,8 @@ public BlockLoader.Block readValues(BuilderFactory factory, Docs docs) throws IO } @Override - public void readValuesFromSingleDoc(int doc, Builder builder) throws IOException { - if (ordinals.advanceExact(doc)) { + public void read(int docId, BlockLoader.StoredFields storedFields, Builder builder) throws IOException { + if (ordinals.advanceExact(docId)) { ((BytesRefBuilder) builder).appendBytesRef(ordinals.lookupOrd(ordinals.ordValue())); } else { builder.appendNull(); @@ -571,13 +550,13 @@ public void readValuesFromSingleDoc(int doc, Builder builder) throws IOException } @Override - public int docID() { + public int docId() { return ordinals.docID(); } @Override public String toString() { - return "SingletonOrdinals"; + return "BlockDocValuesReader.SingletonOrdinals"; } } @@ -589,13 +568,8 @@ private static class Ordinals extends BlockDocValuesReader { } @Override - public BytesRefBuilder builder(BuilderFactory factory, int expectedCount) { - return factory.bytesRefsFromDocValues(expectedCount); - } - - @Override - public BlockLoader.Block readValues(BuilderFactory factory, Docs docs) throws IOException { - try (BytesRefBuilder builder = builder(factory, docs.count())) { + public BlockLoader.Block read(BlockFactory factory, Docs docs) throws IOException { + try (BytesRefBuilder builder = factory.bytesRefsFromDocValues(docs.count())) { for (int i = 0; i < docs.count(); i++) { int doc = docs.get(i); if (doc < ordinals.docID()) { @@ -608,12 +582,12 @@ public BlockLoader.Block readValues(BuilderFactory factory, Docs docs) throws IO } @Override - public void readValuesFromSingleDoc(int doc, Builder builder) throws IOException { - read(doc, (BytesRefBuilder) builder); + public void read(int docId, BlockLoader.StoredFields storedFields, Builder builder) throws IOException { + read(docId, (BytesRefBuilder) builder); } - private void read(int doc, BytesRefBuilder builder) throws IOException { - if (false == ordinals.advanceExact(doc)) { + private void read(int docId, BytesRefBuilder builder) throws IOException { + if (false == ordinals.advanceExact(docId)) { builder.appendNull(); return; } @@ -630,32 +604,52 @@ private void read(int doc, BytesRefBuilder builder) throws IOException { } @Override - public int docID() { + public int docId() { return ordinals.docID(); } @Override public String toString() { - return "Ordinals"; + return "BlockDocValuesReader.Ordinals"; } } - private static class Bytes extends BlockDocValuesReader { - private final SortedBinaryDocValues docValues; - private int docID = -1; + public static class BytesRefsFromBinaryBlockLoader extends DocValuesBlockLoader { + private final String fieldName; - Bytes(SortedBinaryDocValues docValues) { - this.docValues = docValues; + public BytesRefsFromBinaryBlockLoader(String fieldName) { + this.fieldName = fieldName; } @Override - public BytesRefBuilder builder(BuilderFactory factory, int expectedCount) { - return factory.bytesRefsFromDocValues(expectedCount); + public Builder builder(BlockFactory factory, int expectedCount) { + return factory.bytesRefs(expectedCount); } @Override - public BlockLoader.Block readValues(BuilderFactory factory, Docs docs) throws IOException { - try (BlockLoader.BytesRefBuilder builder = builder(factory, docs.count())) { + public AllReader reader(LeafReaderContext context) throws IOException { + BinaryDocValues docValues = context.reader().getBinaryDocValues(fieldName); + if (docValues == null) { + return new ConstantNullsReader(); + } + return new BytesRefsFromBinary(docValues); + } + } + + private static class BytesRefsFromBinary extends BlockDocValuesReader { + private final BinaryDocValues docValues; + private final ByteArrayStreamInput in = new ByteArrayStreamInput(); + private final BytesRef scratch = new BytesRef(); + + private int docID = -1; + + BytesRefsFromBinary(BinaryDocValues docValues) { + this.docValues = docValues; + } + + @Override + public BlockLoader.Block read(BlockFactory factory, Docs docs) throws IOException { + try (BlockLoader.BytesRefBuilder builder = factory.bytesRefs(docs.count())) { for (int i = 0; i < docs.count(); i++) { int doc = docs.get(i); if (doc < docID) { @@ -668,7 +662,7 @@ public BlockLoader.Block readValues(BuilderFactory factory, Docs docs) throws IO } @Override - public void readValuesFromSingleDoc(int docId, Builder builder) throws IOException { + public void read(int docId, BlockLoader.StoredFields storedFields, Builder builder) throws IOException { read(docId, (BytesRefBuilder) builder); } @@ -678,27 +672,59 @@ private void read(int doc, BytesRefBuilder builder) throws IOException { builder.appendNull(); return; } - int count = docValues.docValueCount(); + BytesRef bytes = docValues.binaryValue(); + assert bytes.length > 0; + in.reset(bytes.bytes, bytes.offset, bytes.length); + int count = in.readVInt(); + scratch.bytes = bytes.bytes; + if (count == 1) { - // TODO read ords in ascending order. Buffers and stuff. - builder.appendBytesRef(docValues.nextValue()); + scratch.length = in.readVInt(); + scratch.offset = in.getPosition(); + builder.appendBytesRef(scratch); return; } builder.beginPositionEntry(); for (int v = 0; v < count; v++) { - builder.appendBytesRef(docValues.nextValue()); + scratch.length = in.readVInt(); + scratch.offset = in.getPosition(); + in.setPosition(scratch.offset + scratch.length); + builder.appendBytesRef(scratch); } builder.endPositionEntry(); } @Override - public int docID() { + public int docId() { return docID; } @Override public String toString() { - return "Bytes"; + return "BlockDocValuesReader.Bytes"; + } + } + + public static class BooleansBlockLoader extends DocValuesBlockLoader { + private final String fieldName; + + public BooleansBlockLoader(String fieldName) { + this.fieldName = fieldName; + } + + @Override + public BooleanBuilder builder(BlockFactory factory, int expectedCount) { + return factory.booleans(expectedCount); + } + + @Override + public AllReader reader(LeafReaderContext context) throws IOException { + SortedNumericDocValues docValues = DocValues.getSortedNumeric(context.reader(), fieldName); + NumericDocValues singleton = DocValues.unwrapSingleton(docValues); + if (singleton != null) { + return new SingletonBooleans(singleton); + } + return new Booleans(docValues); } } @@ -710,13 +736,8 @@ private static class SingletonBooleans extends BlockDocValuesReader { } @Override - public BooleanBuilder builder(BuilderFactory factory, int expectedCount) { - return factory.booleansFromDocValues(expectedCount); - } - - @Override - public BlockLoader.Block readValues(BuilderFactory factory, Docs docs) throws IOException { - try (BlockLoader.BooleanBuilder builder = builder(factory, docs.count())) { + public BlockLoader.Block read(BlockFactory factory, Docs docs) throws IOException { + try (BlockLoader.BooleanBuilder builder = factory.booleansFromDocValues(docs.count())) { int lastDoc = -1; for (int i = 0; i < docs.count(); i++) { int doc = docs.get(i); @@ -735,7 +756,7 @@ public BlockLoader.Block readValues(BuilderFactory factory, Docs docs) throws IO } @Override - public void readValuesFromSingleDoc(int docId, Builder builder) throws IOException { + public void read(int docId, BlockLoader.StoredFields storedFields, Builder builder) throws IOException { BooleanBuilder blockBuilder = (BooleanBuilder) builder; if (numericDocValues.advanceExact(docId)) { blockBuilder.appendBoolean(numericDocValues.longValue() != 0); @@ -745,13 +766,13 @@ public void readValuesFromSingleDoc(int docId, Builder builder) throws IOExcepti } @Override - public int docID() { + public int docId() { return numericDocValues.docID(); } @Override public String toString() { - return "SingletonBooleans"; + return "BlockDocValuesReader.SingletonBooleans"; } } @@ -764,13 +785,8 @@ private static class Booleans extends BlockDocValuesReader { } @Override - public BooleanBuilder builder(BuilderFactory factory, int expectedCount) { - return factory.booleansFromDocValues(expectedCount); - } - - @Override - public BlockLoader.Block readValues(BuilderFactory factory, Docs docs) throws IOException { - try (BlockLoader.BooleanBuilder builder = builder(factory, docs.count())) { + public BlockLoader.Block read(BlockFactory factory, Docs docs) throws IOException { + try (BlockLoader.BooleanBuilder builder = factory.booleansFromDocValues(docs.count())) { for (int i = 0; i < docs.count(); i++) { int doc = docs.get(i); if (doc < this.docID) { @@ -783,7 +799,7 @@ public BlockLoader.Block readValues(BuilderFactory factory, Docs docs) throws IO } @Override - public void readValuesFromSingleDoc(int docId, Builder builder) throws IOException { + public void read(int docId, BlockLoader.StoredFields storedFields, Builder builder) throws IOException { read(docId, (BooleanBuilder) builder); } @@ -806,61 +822,14 @@ private void read(int doc, BooleanBuilder builder) throws IOException { } @Override - public int docID() { + public int docId() { // There is a .docID on the numericDocValues but it is often not implemented. return docID; } @Override public String toString() { - return "Booleans"; - } - } - - private static class Nulls extends BlockDocValuesReader { - private int docID = -1; - - @Override - public BlockLoader.Builder builder(BuilderFactory factory, int expectedCount) { - return factory.nulls(expectedCount); - } - - @Override - public BlockLoader.Block readValues(BuilderFactory factory, Docs docs) throws IOException { - try (BlockLoader.Builder builder = builder(factory, docs.count())) { - for (int i = 0; i < docs.count(); i++) { - builder.appendNull(); - } - return builder.build(); - } - } - - @Override - public void readValuesFromSingleDoc(int docId, Builder builder) { - this.docID = docId; - builder.appendNull(); - } - - @Override - public int docID() { - return docID; - } - - @Override - public String toString() { - return "Nulls"; - } - } - - /** - * Convert a {@link String} into a utf-8 {@link BytesRef}. - */ - protected static BytesRef toBytesRef(BytesRef scratch, String v) { - int len = UnicodeUtil.maxUTF8Length(v.length()); - if (scratch.bytes.length < len) { - scratch.bytes = new byte[len]; + return "BlockDocValuesReader.Booleans"; } - scratch.length = UnicodeUtil.UTF16toUTF8(v, 0, v.length(), scratch.bytes); - return scratch; } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/BlockLoader.java b/server/src/main/java/org/elasticsearch/index/mapper/BlockLoader.java index af53ab42d35d9..a8f3b919f33cc 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/BlockLoader.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/BlockLoader.java @@ -13,8 +13,12 @@ import org.apache.lucene.index.SortedSetDocValues; import org.apache.lucene.util.BytesRef; import org.elasticsearch.core.Releasable; +import org.elasticsearch.search.fetch.StoredFieldsSpec; +import org.elasticsearch.search.lookup.Source; import java.io.IOException; +import java.util.List; +import java.util.Map; /** * Interface for loading data in a block shape. Instances of this class @@ -22,26 +26,292 @@ */ public interface BlockLoader { /** - * Build a {@link LeafReaderContext leaf} level reader. + * The {@link BlockLoader.Builder} for data of this type. Called when + * loading from a multi-segment or unsorted block. */ - BlockDocValuesReader reader(LeafReaderContext context) throws IOException; + Builder builder(BlockFactory factory, int expectedCount); + + interface Reader { + /** + * Checks if the reader can be used to read a range documents starting with the given docID by the current thread. + */ + boolean canReuse(int startingDocID); + } + + interface ColumnAtATimeReader extends Reader { + /** + * Reads the values of all documents in {@code docs}. + */ + BlockLoader.Block read(BlockFactory factory, Docs docs) throws IOException; + } + + interface RowStrideReader extends Reader { + /** + * Reads the values of the given document into the builder. + */ + void read(int docId, StoredFields storedFields, Builder builder) throws IOException; + } + + interface AllReader extends ColumnAtATimeReader, RowStrideReader {} + + interface StoredFields { + Source source(); + + /** + * @return the ID for the current document + */ + String id(); + + /** + * @return the routing path for the current document + */ + String routing(); + + /** + * @return stored fields for the current document + */ + Map> storedFields(); + } + + ColumnAtATimeReader columnAtATimeReader(LeafReaderContext context) throws IOException; + + RowStrideReader rowStrideReader(LeafReaderContext context) throws IOException; + + StoredFieldsSpec rowStrideStoredFieldSpec(); /** * Does this loader support loading bytes via calling {@link #ordinals}. */ - default boolean supportsOrdinals() { - return false; - } + boolean supportsOrdinals(); /** * Load ordinals for the provided context. */ - default SortedSetDocValues ordinals(LeafReaderContext context) throws IOException { - throw new IllegalStateException("ordinals not supported"); + SortedSetDocValues ordinals(LeafReaderContext context) throws IOException; + + /** + * Load blocks with only null. + */ + BlockLoader CONSTANT_NULLS = new BlockLoader() { + @Override + public Builder builder(BlockFactory factory, int expectedCount) { + return factory.nulls(expectedCount); + } + + @Override + public ColumnAtATimeReader columnAtATimeReader(LeafReaderContext context) { + return new ConstantNullsReader(); + } + + @Override + public RowStrideReader rowStrideReader(LeafReaderContext context) { + return new ConstantNullsReader(); + } + + @Override + public StoredFieldsSpec rowStrideStoredFieldSpec() { + return StoredFieldsSpec.NO_REQUIREMENTS; + } + + @Override + public boolean supportsOrdinals() { + return false; + } + + @Override + public SortedSetDocValues ordinals(LeafReaderContext context) { + throw new UnsupportedOperationException(); + } + + @Override + public String toString() { + return "ConstantNull"; + } + }; + + /** + * Implementation of {@link ColumnAtATimeReader} and {@link RowStrideReader} that always + * loads {@code null}. + */ + class ConstantNullsReader implements AllReader { + @Override + public Block read(BlockFactory factory, Docs docs) throws IOException { + return factory.constantNulls(docs.count()); + } + + @Override + public void read(int docId, StoredFields storedFields, Builder builder) throws IOException { + builder.appendNull(); + } + + @Override + public boolean canReuse(int startingDocID) { + return true; + } + + @Override + public String toString() { + return "constant_nulls"; + } } /** - * A list of documents to load. + * Load blocks with only {@code value}. + */ + static BlockLoader constantBytes(BytesRef value) { + return new BlockLoader() { + @Override + public Builder builder(BlockFactory factory, int expectedCount) { + return factory.bytesRefs(expectedCount); + } + + @Override + public ColumnAtATimeReader columnAtATimeReader(LeafReaderContext context) { + return new ColumnAtATimeReader() { + @Override + public Block read(BlockFactory factory, Docs docs) { + return factory.constantBytes(value, docs.count()); + } + + @Override + public boolean canReuse(int startingDocID) { + return true; + } + + @Override + public String toString() { + return "constant[" + value + "]"; + } + }; + } + + @Override + public RowStrideReader rowStrideReader(LeafReaderContext context) { + return new RowStrideReader() { + @Override + public void read(int docId, StoredFields storedFields, Builder builder) { + ((BlockLoader.BytesRefBuilder) builder).appendBytesRef(value); + } + + @Override + public boolean canReuse(int startingDocID) { + return true; + } + + @Override + public String toString() { + return "constant[" + value + "]"; + } + }; + } + + @Override + public StoredFieldsSpec rowStrideStoredFieldSpec() { + return StoredFieldsSpec.NO_REQUIREMENTS; + } + + @Override + public boolean supportsOrdinals() { + return false; + } + + @Override + public SortedSetDocValues ordinals(LeafReaderContext context) { + throw new UnsupportedOperationException(); + } + + @Override + public String toString() { + return "ConstantBytes[" + value + "]"; + } + }; + } + + abstract class Delegating implements BlockLoader { + protected final BlockLoader delegate; + + protected Delegating(BlockLoader delegate) { + this.delegate = delegate; + } + + @Override + public Builder builder(BlockFactory factory, int expectedCount) { + return delegate.builder(factory, expectedCount); + } + + @Override + public ColumnAtATimeReader columnAtATimeReader(LeafReaderContext context) throws IOException { + ColumnAtATimeReader reader = delegate.columnAtATimeReader(context); + if (reader == null) { + return null; + } + return new ColumnAtATimeReader() { + @Override + public Block read(BlockFactory factory, Docs docs) throws IOException { + return reader.read(factory, docs); + } + + @Override + public boolean canReuse(int startingDocID) { + return reader.canReuse(startingDocID); + } + + @Override + public String toString() { + return "Delegating[to=" + delegatingTo() + ", impl=" + reader + "]"; + } + }; + } + + @Override + public RowStrideReader rowStrideReader(LeafReaderContext context) throws IOException { + RowStrideReader reader = delegate.rowStrideReader(context); + if (reader == null) { + return null; + } + return new RowStrideReader() { + @Override + public void read(int docId, StoredFields storedFields, Builder builder) throws IOException { + reader.read(docId, storedFields, builder); + } + + @Override + public boolean canReuse(int startingDocID) { + return reader.canReuse(startingDocID); + } + + @Override + public String toString() { + return "Delegating[to=" + delegatingTo() + ", impl=" + reader + "]"; + } + }; + } + + @Override + public StoredFieldsSpec rowStrideStoredFieldSpec() { + return delegate.rowStrideStoredFieldSpec(); + } + + @Override + public boolean supportsOrdinals() { + return delegate.supportsOrdinals(); + } + + @Override + public SortedSetDocValues ordinals(LeafReaderContext context) throws IOException { + return delegate.ordinals(context); + } + + protected abstract String delegatingTo(); + + @Override + public final String toString() { + return "Delegating[to=" + delegatingTo() + ", impl=" + delegate + "]"; + } + } + + /** + * A list of documents to load. Documents are always in non-decreasing order. */ interface Docs { int count(); @@ -55,7 +325,7 @@ interface Docs { * production code. That implementation sits in the "compute" project. The is * also a test implementation, but there may be no more other implementations. */ - interface BuilderFactory { + interface BlockFactory { /** * Build a builder to load booleans as loaded from doc values. Doc values * load booleans deduplicated and in sorted order. @@ -112,11 +382,21 @@ interface BuilderFactory { LongBuilder longs(int expectedCount); /** - * Build a builder that can only load null values. - * TODO this should return a block directly instead of a builder + * Build a builder to load only {@code null}s. */ Builder nulls(int expectedCount); + /** + * Build a block that contains only {@code null}. + */ + Block constantNulls(int size); + + /** + * Build a block that contains {@code value} repeated + * {@code size} times. + */ + Block constantBytes(BytesRef value, int size); + /** * Build a reader for reading keyword ordinals. */ @@ -129,7 +409,7 @@ interface BuilderFactory { * Marker interface for block results. The compute engine has a fleshed * out implementation. */ - interface Block {} + interface Block extends Releasable {} /** * A builder for typed values. For each document you may either call diff --git a/server/src/main/java/org/elasticsearch/index/mapper/BlockLoaderStoredFieldsFromLeafLoader.java b/server/src/main/java/org/elasticsearch/index/mapper/BlockLoaderStoredFieldsFromLeafLoader.java new file mode 100644 index 0000000000000..8b1b794f1df55 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/index/mapper/BlockLoaderStoredFieldsFromLeafLoader.java @@ -0,0 +1,54 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.index.mapper; + +import org.elasticsearch.index.fieldvisitor.LeafStoredFieldLoader; +import org.elasticsearch.search.lookup.Source; + +import java.io.IOException; +import java.util.List; +import java.util.Map; + +public class BlockLoaderStoredFieldsFromLeafLoader implements BlockLoader.StoredFields { + private final LeafStoredFieldLoader loader; + private final boolean loadSource; + private Source source; + + public BlockLoaderStoredFieldsFromLeafLoader(LeafStoredFieldLoader loader, boolean loadSource) { + this.loader = loader; + this.loadSource = loadSource; + } + + public void advanceTo(int doc) throws IOException { + loader.advanceTo(doc); + if (loadSource) { + source = Source.fromBytes(loader.source()); + } + } + + @Override + public Source source() { + return source; + } + + @Override + public String id() { + return loader.id(); + } + + @Override + public String routing() { + return loader.routing(); + } + + @Override + public Map> storedFields() { + return loader.storedFields(); + } +} diff --git a/server/src/main/java/org/elasticsearch/index/mapper/BlockSourceReader.java b/server/src/main/java/org/elasticsearch/index/mapper/BlockSourceReader.java index 1261a3612d3cb..289b28949cdab 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/BlockSourceReader.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/BlockSourceReader.java @@ -8,172 +8,32 @@ package org.elasticsearch.index.mapper; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.SortedSetDocValues; import org.apache.lucene.util.BytesRef; -import org.elasticsearch.index.fieldvisitor.LeafStoredFieldLoader; -import org.elasticsearch.index.fieldvisitor.StoredFieldLoader; -import org.elasticsearch.search.lookup.Source; +import org.apache.lucene.util.UnicodeUtil; +import org.elasticsearch.search.fetch.StoredFieldsSpec; import java.io.IOException; import java.util.ArrayList; import java.util.List; -import java.util.Set; /** * Loads values from {@code _source}. This whole process is very slow and cast-tastic, * so it doesn't really try to avoid megamorphic invocations. It's just going to be * slow. - * - * Note that this extends {@link BlockDocValuesReader} because it pretends to load - * doc values because, for now, ESQL only knows how to load things in a doc values - * order. */ -public abstract class BlockSourceReader extends BlockDocValuesReader { - /** - * Read {@code boolean}s from {@code _source}. - */ - public static BlockLoader booleans(ValueFetcher fetcher) { - StoredFieldLoader loader = StoredFieldLoader.create(true, Set.of()); - return context -> new BlockSourceReader(fetcher, loader.getLoader(context, null)) { - @Override - public BlockLoader.Builder builder(BlockLoader.BuilderFactory factory, int expectedCount) { - return factory.booleans(expectedCount); - } - - @Override - protected void append(BlockLoader.Builder builder, Object v) { - ((BlockLoader.BooleanBuilder) builder).appendBoolean((Boolean) v); - } - - @Override - public String toString() { - return "SourceBooleans"; - } - }; - } - - /** - * Read {@link BytesRef}s from {@code _source}. - */ - public static BlockLoader bytesRefs(ValueFetcher fetcher) { - StoredFieldLoader loader = StoredFieldLoader.create(true, Set.of()); - return context -> new BlockSourceReader(fetcher, loader.getLoader(context, null)) { - BytesRef scratch = new BytesRef(); - - @Override - public BlockLoader.Builder builder(BlockLoader.BuilderFactory factory, int expectedCount) { - return factory.bytesRefs(expectedCount); - } - - @Override - protected void append(BlockLoader.Builder builder, Object v) { - ((BlockLoader.BytesRefBuilder) builder).appendBytesRef(toBytesRef(scratch, (String) v)); - } - - @Override - public String toString() { - return "SourceBytes"; - } - }; - } - - /** - * Read {@code double}s from {@code _source}. - */ - public static BlockLoader doubles(ValueFetcher fetcher) { - StoredFieldLoader loader = StoredFieldLoader.create(true, Set.of()); - return context -> new BlockSourceReader(fetcher, loader.getLoader(context, null)) { - @Override - public BlockLoader.Builder builder(BlockLoader.BuilderFactory factory, int expectedCount) { - return factory.doubles(expectedCount); - } - - @Override - protected void append(BlockLoader.Builder builder, Object v) { - ((BlockLoader.DoubleBuilder) builder).appendDouble(((Number) v).doubleValue()); - } - - @Override - public String toString() { - return "SourceDoubles"; - } - }; - } - - /** - * Read {@code int}s from {@code _source}. - */ - public static BlockLoader ints(ValueFetcher fetcher) { - StoredFieldLoader loader = StoredFieldLoader.create(true, Set.of()); - return context -> new BlockSourceReader(fetcher, loader.getLoader(context, null)) { - @Override - public BlockLoader.Builder builder(BlockLoader.BuilderFactory factory, int expectedCount) { - return factory.ints(expectedCount); - } - - @Override - protected void append(BlockLoader.Builder builder, Object v) { - ((BlockLoader.IntBuilder) builder).appendInt(((Number) v).intValue()); - } - - @Override - public String toString() { - return "SourceInts"; - } - }; - } - - /** - * Read {@code long}s from {@code _source}. - */ - public static BlockLoader longs(ValueFetcher fetcher) { - StoredFieldLoader loader = StoredFieldLoader.create(true, Set.of()); - return context -> new BlockSourceReader(fetcher, loader.getLoader(context, null)) { - @Override - public BlockLoader.Builder builder(BlockLoader.BuilderFactory factory, int expectedCount) { - return factory.longs(expectedCount); - } - - @Override - protected void append(BlockLoader.Builder builder, Object v) { - ((BlockLoader.LongBuilder) builder).appendLong(((Number) v).longValue()); - } - - @Override - public String toString() { - return "SourceLongs"; - } - }; - } - +public abstract class BlockSourceReader implements BlockLoader.RowStrideReader { private final ValueFetcher fetcher; - private final LeafStoredFieldLoader loader; private final List ignoredValues = new ArrayList<>(); - private int docID = -1; - BlockSourceReader(ValueFetcher fetcher, LeafStoredFieldLoader loader) { + BlockSourceReader(ValueFetcher fetcher) { this.fetcher = fetcher; - this.loader = loader; - } - - @Override - public BlockLoader.Block readValues(BlockLoader.BuilderFactory factory, BlockLoader.Docs docs) throws IOException { - try (BlockLoader.Builder builder = builder(factory, docs.count())) { - for (int i = 0; i < docs.count(); i++) { - int doc = docs.get(i); - if (doc < this.docID) { - throw new IllegalStateException("docs within same block must be in order"); - } - readValuesFromSingleDoc(doc, builder); - } - return builder.build(); - } } @Override - public void readValuesFromSingleDoc(int doc, BlockLoader.Builder builder) throws IOException { - this.docID = doc; - loader.advanceTo(doc); - List values = fetcher.fetchValues(Source.fromBytes(loader.source()), doc, ignoredValues); + public final void read(int docId, BlockLoader.StoredFields storedFields, BlockLoader.Builder builder) throws IOException { + List values = fetcher.fetchValues(storedFields.source(), docId, ignoredValues); ignoredValues.clear(); // TODO do something with these? if (values == null) { builder.appendNull(); @@ -193,7 +53,213 @@ public void readValuesFromSingleDoc(int doc, BlockLoader.Builder builder) throws protected abstract void append(BlockLoader.Builder builder, Object v); @Override - public int docID() { - return docID; + public boolean canReuse(int startingDocID) { + return true; + } + + private abstract static class SourceBlockLoader implements BlockLoader { + @Override + public final ColumnAtATimeReader columnAtATimeReader(LeafReaderContext context) throws IOException { + return null; + } + + @Override + public final StoredFieldsSpec rowStrideStoredFieldSpec() { + return StoredFieldsSpec.NEEDS_SOURCE; + } + + @Override + public final boolean supportsOrdinals() { + return false; + } + + @Override + public final SortedSetDocValues ordinals(LeafReaderContext context) { + throw new UnsupportedOperationException(); + } + } + + public static class BooleansBlockLoader extends SourceBlockLoader { + private final ValueFetcher fetcher; + + public BooleansBlockLoader(ValueFetcher fetcher) { + this.fetcher = fetcher; + } + + @Override + public Builder builder(BlockFactory factory, int expectedCount) { + return factory.booleans(expectedCount); + } + + @Override + public RowStrideReader rowStrideReader(LeafReaderContext context) { + return new Booleans(fetcher); + } + } + + private static class Booleans extends BlockSourceReader { + Booleans(ValueFetcher fetcher) { + super(fetcher); + } + + @Override + protected void append(BlockLoader.Builder builder, Object v) { + ((BlockLoader.BooleanBuilder) builder).appendBoolean((Boolean) v); + } + + @Override + public String toString() { + return "BlockSourceReader.Booleans"; + } + } + + public static class BytesRefsBlockLoader extends SourceBlockLoader { + private final ValueFetcher fetcher; + + public BytesRefsBlockLoader(ValueFetcher fetcher) { + this.fetcher = fetcher; + } + + @Override + public Builder builder(BlockFactory factory, int expectedCount) { + return factory.bytesRefs(expectedCount); + } + + @Override + public RowStrideReader rowStrideReader(LeafReaderContext context) { + return new BytesRefs(fetcher); + } + } + + private static class BytesRefs extends BlockSourceReader { + BytesRef scratch = new BytesRef(); + + BytesRefs(ValueFetcher fetcher) { + super(fetcher); + } + + @Override + protected void append(BlockLoader.Builder builder, Object v) { + ((BlockLoader.BytesRefBuilder) builder).appendBytesRef(toBytesRef(scratch, (String) v)); + } + + @Override + public String toString() { + return "BlockSourceReader.Bytes"; + } + } + + public static class DoublesBlockLoader extends SourceBlockLoader { + private final ValueFetcher fetcher; + + public DoublesBlockLoader(ValueFetcher fetcher) { + this.fetcher = fetcher; + } + + @Override + public Builder builder(BlockFactory factory, int expectedCount) { + return factory.doubles(expectedCount); + } + + @Override + public RowStrideReader rowStrideReader(LeafReaderContext context) { + return new Doubles(fetcher); + } + } + + private static class Doubles extends BlockSourceReader { + Doubles(ValueFetcher fetcher) { + super(fetcher); + } + + @Override + protected void append(BlockLoader.Builder builder, Object v) { + ((BlockLoader.DoubleBuilder) builder).appendDouble(((Number) v).doubleValue()); + } + + @Override + public String toString() { + return "BlockSourceReader.Doubles"; + } + } + + public static class IntsBlockLoader extends SourceBlockLoader { + private final ValueFetcher fetcher; + + public IntsBlockLoader(ValueFetcher fetcher) { + this.fetcher = fetcher; + } + + @Override + public Builder builder(BlockFactory factory, int expectedCount) { + return factory.ints(expectedCount); + } + + @Override + public RowStrideReader rowStrideReader(LeafReaderContext context) { + return new Ints(fetcher); + } + } + + private static class Ints extends BlockSourceReader { + Ints(ValueFetcher fetcher) { + super(fetcher); + } + + @Override + protected void append(BlockLoader.Builder builder, Object v) { + ((BlockLoader.IntBuilder) builder).appendInt(((Number) v).intValue()); + } + + @Override + public String toString() { + return "BlockSourceReader.Ints"; + } + } + + public static class LongsBlockLoader extends SourceBlockLoader { + private final ValueFetcher fetcher; + + public LongsBlockLoader(ValueFetcher fetcher) { + this.fetcher = fetcher; + } + + @Override + public Builder builder(BlockFactory factory, int expectedCount) { + return factory.longs(expectedCount); + } + + @Override + public RowStrideReader rowStrideReader(LeafReaderContext context) { + return new Longs(fetcher); + } + } + + private static class Longs extends BlockSourceReader { + Longs(ValueFetcher fetcher) { + super(fetcher); + } + + @Override + protected void append(BlockLoader.Builder builder, Object v) { + ((BlockLoader.LongBuilder) builder).appendLong(((Number) v).longValue()); + } + + @Override + public String toString() { + return "BlockSourceReader.Longs"; + } + } + + /** + * Convert a {@link String} into a utf-8 {@link BytesRef}. + */ + static BytesRef toBytesRef(BytesRef scratch, String v) { + int len = UnicodeUtil.maxUTF8Length(v.length()); + if (scratch.bytes.length < len) { + scratch.bytes = new byte[len]; + } + scratch.length = UnicodeUtil.UTF16toUTF8(v, 0, v.length(), scratch.bytes); + return scratch; } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/BlockStoredFieldsReader.java b/server/src/main/java/org/elasticsearch/index/mapper/BlockStoredFieldsReader.java index 5984482fd9441..043ca38b1c78b 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/BlockStoredFieldsReader.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/BlockStoredFieldsReader.java @@ -9,10 +9,11 @@ package org.elasticsearch.index.mapper; import org.apache.lucene.index.LeafReader; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.SortedSetDocValues; import org.apache.lucene.util.BytesRef; -import org.elasticsearch.index.fieldvisitor.LeafStoredFieldLoader; -import org.elasticsearch.index.fieldvisitor.StoredFieldLoader; import org.elasticsearch.index.mapper.BlockLoader.BytesRefBuilder; +import org.elasticsearch.search.fetch.StoredFieldsSpec; import java.io.IOException; import java.util.List; @@ -27,86 +28,101 @@ * doc values because, for now, ESQL only knows how to load things in a doc values * order. */ -public abstract class BlockStoredFieldsReader extends BlockDocValuesReader { - public static BlockLoader bytesRefsFromBytesRefs(String field) { - StoredFieldLoader loader = StoredFieldLoader.create(false, Set.of(field)); - return context -> new Bytes(loader.getLoader(context, null), field) { - @Override - protected BytesRef toBytesRef(Object v) { - return (BytesRef) v; - } - }; +public abstract class BlockStoredFieldsReader implements BlockLoader.RowStrideReader { + @Override + public boolean canReuse(int startingDocID) { + return true; } - public static BlockLoader bytesRefsFromStrings(String field) { - StoredFieldLoader loader = StoredFieldLoader.create(false, Set.of(field)); - return context -> new Bytes(loader.getLoader(context, null), field) { - private final BytesRef scratch = new BytesRef(); + private abstract static class StoredFieldsBlockLoader implements BlockLoader { + protected final String field; - @Override - protected BytesRef toBytesRef(Object v) { - return toBytesRef(scratch, (String) v); - } - }; - } + StoredFieldsBlockLoader(String field) { + this.field = field; + } - public static BlockLoader id() { - StoredFieldLoader loader = StoredFieldLoader.create(false, Set.of(IdFieldMapper.NAME)); - return context -> new Id(loader.getLoader(context, null)); - } + @Override + public final ColumnAtATimeReader columnAtATimeReader(LeafReaderContext context) throws IOException { + return null; + } - private final LeafStoredFieldLoader loader; - private int docID = -1; + @Override + public final StoredFieldsSpec rowStrideStoredFieldSpec() { + return new StoredFieldsSpec(false, false, Set.of(field)); + } - protected BlockStoredFieldsReader(LeafStoredFieldLoader loader) { - this.loader = loader; - } + @Override + public final boolean supportsOrdinals() { + return false; + } - @Override - public final BlockLoader.Block readValues(BlockLoader.BuilderFactory factory, BlockLoader.Docs docs) throws IOException { - try (BlockLoader.Builder builder = builder(factory, docs.count())) { - for (int i = 0; i < docs.count(); i++) { - readValuesFromSingleDoc(docs.get(i), builder); - } - return builder.build(); + @Override + public final SortedSetDocValues ordinals(LeafReaderContext context) { + throw new UnsupportedOperationException(); } } - @Override - public final void readValuesFromSingleDoc(int docId, BlockLoader.Builder builder) throws IOException { - if (docId < this.docID) { - throw new IllegalStateException("docs within same block must be in order"); + /** + * Load {@link BytesRef} blocks from stored {@link BytesRef}s. + */ + public static class BytesFromBytesRefsBlockLoader extends StoredFieldsBlockLoader { + public BytesFromBytesRefsBlockLoader(String field) { + super(field); + } + + @Override + public Builder builder(BlockFactory factory, int expectedCount) { + return factory.bytesRefs(expectedCount); + } + + @Override + public RowStrideReader rowStrideReader(LeafReaderContext context) throws IOException { + return new Bytes(field) { + @Override + protected BytesRef toBytesRef(Object v) { + return (BytesRef) v; + } + }; } - this.docID = docId; - loader.advanceTo(docId); - read(loader, builder); } - protected abstract void read(LeafStoredFieldLoader loader, BlockLoader.Builder builder) throws IOException; + /** + * Load {@link BytesRef} blocks from stored {@link String}s. + */ + public static class BytesFromStringsBlockLoader extends StoredFieldsBlockLoader { + public BytesFromStringsBlockLoader(String field) { + super(field); + } - @Override - public final int docID() { - return docID; + public Builder builder(BlockFactory factory, int expectedCount) { + return factory.bytesRefs(expectedCount); + } + + @Override + public RowStrideReader rowStrideReader(LeafReaderContext context) throws IOException { + return new Bytes(field) { + private final BytesRef scratch = new BytesRef(); + + @Override + protected BytesRef toBytesRef(Object v) { + return BlockSourceReader.toBytesRef(scratch, (String) v); + } + }; + } } private abstract static class Bytes extends BlockStoredFieldsReader { private final String field; - Bytes(LeafStoredFieldLoader loader, String field) { - super(loader); + Bytes(String field) { this.field = field; } - @Override - public BytesRefBuilder builder(BlockLoader.BuilderFactory factory, int expectedCount) { - return factory.bytesRefs(expectedCount); - } - protected abstract BytesRef toBytesRef(Object v); @Override - protected void read(LeafStoredFieldLoader loader, BlockLoader.Builder builder) throws IOException { - List values = loader.storedFields().get(field); + public void read(int docId, BlockLoader.StoredFields storedFields, BlockLoader.Builder builder) throws IOException { + List values = storedFields.storedFields().get(field); if (values == null) { builder.appendNull(); return; @@ -128,21 +144,31 @@ public String toString() { } } - private static class Id extends BlockStoredFieldsReader { - private final BytesRef scratch = new BytesRef(); - - Id(LeafStoredFieldLoader loader) { - super(loader); + /** + * Load {@link BytesRef} blocks from stored {@link String}s. + */ + public static class IdBlockLoader extends StoredFieldsBlockLoader { + public IdBlockLoader() { + super(IdFieldMapper.NAME); } @Override - public BlockLoader.BytesRefBuilder builder(BlockLoader.BuilderFactory factory, int expectedCount) { + public Builder builder(BlockFactory factory, int expectedCount) { return factory.bytesRefs(expectedCount); } @Override - protected void read(LeafStoredFieldLoader loader, BlockLoader.Builder builder) throws IOException { - ((BytesRefBuilder) builder).appendBytesRef(toBytesRef(scratch, loader.id())); + public RowStrideReader rowStrideReader(LeafReaderContext context) throws IOException { + return new Id(); + } + } + + private static class Id extends BlockStoredFieldsReader { + private final BytesRef scratch = new BytesRef(); + + @Override + public void read(int docId, BlockLoader.StoredFields storedFields, BlockLoader.Builder builder) throws IOException { + ((BytesRefBuilder) builder).appendBytesRef(BlockSourceReader.toBytesRef(scratch, storedFields.id())); } @Override diff --git a/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java index a5793df3b82e0..7f175982dc28e 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java @@ -257,9 +257,9 @@ public Boolean valueForDisplay(Object value) { @Override public BlockLoader blockLoader(BlockLoaderContext blContext) { if (hasDocValues()) { - return BlockDocValuesReader.booleans(name()); + return new BlockDocValuesReader.BooleansBlockLoader(name()); } - return BlockSourceReader.booleans(sourceValueFetcher(blContext.sourcePaths(name()))); + return new BlockSourceReader.BooleansBlockLoader(sourceValueFetcher(blContext.sourcePaths(name()))); } @Override diff --git a/server/src/main/java/org/elasticsearch/index/mapper/BooleanScriptBlockDocValuesReader.java b/server/src/main/java/org/elasticsearch/index/mapper/BooleanScriptBlockDocValuesReader.java index b59df56791fbe..953e13dc69eb0 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/BooleanScriptBlockDocValuesReader.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/BooleanScriptBlockDocValuesReader.java @@ -8,14 +8,31 @@ package org.elasticsearch.index.mapper; +import org.apache.lucene.index.LeafReaderContext; import org.elasticsearch.script.BooleanFieldScript; +import java.io.IOException; + /** * {@link BlockDocValuesReader} implementation for {@code boolean} scripts. */ public class BooleanScriptBlockDocValuesReader extends BlockDocValuesReader { - public static BlockLoader blockLoader(BooleanFieldScript.LeafFactory factory) { - return context -> new BooleanScriptBlockDocValuesReader(factory.newInstance(context)); + static class BooleanScriptBlockLoader extends DocValuesBlockLoader { + private final BooleanFieldScript.LeafFactory factory; + + BooleanScriptBlockLoader(BooleanFieldScript.LeafFactory factory) { + this.factory = factory; + } + + @Override + public Builder builder(BlockFactory factory, int expectedCount) { + return factory.booleans(expectedCount); + } + + @Override + public AllReader reader(LeafReaderContext context) throws IOException { + return new BooleanScriptBlockDocValuesReader(factory.newInstance(context)); + } } private final BooleanFieldScript script; @@ -26,19 +43,14 @@ public static BlockLoader blockLoader(BooleanFieldScript.LeafFactory factory) { } @Override - public int docID() { + public int docId() { return docId; } @Override - public BlockLoader.BooleanBuilder builder(BlockLoader.BuilderFactory factory, int expectedCount) { + public BlockLoader.Block read(BlockLoader.BlockFactory factory, BlockLoader.Docs docs) throws IOException { // Note that we don't emit falses before trues so we conform to the doc values contract and can use booleansFromDocValues - return factory.booleansFromDocValues(expectedCount); - } - - @Override - public BlockLoader.Block readValues(BlockLoader.BuilderFactory factory, BlockLoader.Docs docs) { - try (BlockLoader.BooleanBuilder builder = builder(factory, docs.count())) { + try (BlockLoader.BooleanBuilder builder = factory.booleans(docs.count())) { for (int i = 0; i < docs.count(); i++) { read(docs.get(i), builder); } @@ -47,7 +59,7 @@ public BlockLoader.Block readValues(BlockLoader.BuilderFactory factory, BlockLoa } @Override - public void readValuesFromSingleDoc(int docId, BlockLoader.Builder builder) { + public void read(int docId, BlockLoader.StoredFields storedFields, BlockLoader.Builder builder) throws IOException { this.docId = docId; read(docId, (BlockLoader.BooleanBuilder) builder); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/BooleanScriptFieldType.java b/server/src/main/java/org/elasticsearch/index/mapper/BooleanScriptFieldType.java index 6e3876644567f..749bb279cfed4 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/BooleanScriptFieldType.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/BooleanScriptFieldType.java @@ -112,7 +112,7 @@ public DocValueFormat docValueFormat(String format, ZoneId timeZone) { @Override public BlockLoader blockLoader(BlockLoaderContext blContext) { - return BooleanScriptBlockDocValuesReader.blockLoader(leafFactory(blContext.lookup())); + return new BooleanScriptBlockDocValuesReader.BooleanScriptBlockLoader(leafFactory(blContext.lookup())); } @Override diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java index 9d12fc6910d66..e90bea103c4cb 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java @@ -775,9 +775,9 @@ public Function pointReaderIfPossible() { @Override public BlockLoader blockLoader(BlockLoaderContext blContext) { if (hasDocValues()) { - return BlockDocValuesReader.longs(name()); + return new BlockDocValuesReader.LongsBlockLoader(name()); } - return BlockSourceReader.longs(sourceValueFetcher(blContext.sourcePaths(name()))); + return new BlockSourceReader.LongsBlockLoader(sourceValueFetcher(blContext.sourcePaths(name()))); } @Override diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DateScriptBlockDocValuesReader.java b/server/src/main/java/org/elasticsearch/index/mapper/DateScriptBlockDocValuesReader.java index ad630a71870a4..a5303f27573eb 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DateScriptBlockDocValuesReader.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DateScriptBlockDocValuesReader.java @@ -8,14 +8,31 @@ package org.elasticsearch.index.mapper; +import org.apache.lucene.index.LeafReaderContext; import org.elasticsearch.script.DateFieldScript; +import java.io.IOException; + /** * {@link BlockDocValuesReader} implementation for date scripts. */ public class DateScriptBlockDocValuesReader extends BlockDocValuesReader { - public static BlockLoader blockLoader(DateFieldScript.LeafFactory factory) { - return context -> new DateScriptBlockDocValuesReader(factory.newInstance(context)); + static class DateScriptBlockLoader extends DocValuesBlockLoader { + private final DateFieldScript.LeafFactory factory; + + DateScriptBlockLoader(DateFieldScript.LeafFactory factory) { + this.factory = factory; + } + + @Override + public Builder builder(BlockFactory factory, int expectedCount) { + return factory.longs(expectedCount); + } + + @Override + public AllReader reader(LeafReaderContext context) throws IOException { + return new DateScriptBlockDocValuesReader(factory.newInstance(context)); + } } private final DateFieldScript script; @@ -26,18 +43,14 @@ public static BlockLoader blockLoader(DateFieldScript.LeafFactory factory) { } @Override - public int docID() { + public int docId() { return docId; } @Override - public BlockLoader.LongBuilder builder(BlockLoader.BuilderFactory factory, int expectedCount) { - return factory.longs(expectedCount); // Note that we don't pre-sort our output so we can't use longsFromDocValues - } - - @Override - public BlockLoader.Block readValues(BlockLoader.BuilderFactory factory, BlockLoader.Docs docs) { - try (BlockLoader.LongBuilder builder = builder(factory, docs.count())) { + public BlockLoader.Block read(BlockLoader.BlockFactory factory, BlockLoader.Docs docs) throws IOException { + // Note that we don't sort the values sort, so we can't use factory.longsFromDocValues + try (BlockLoader.LongBuilder builder = factory.longs(docs.count())) { for (int i = 0; i < docs.count(); i++) { read(docs.get(i), builder); } @@ -46,7 +59,7 @@ public BlockLoader.Block readValues(BlockLoader.BuilderFactory factory, BlockLoa } @Override - public void readValuesFromSingleDoc(int docId, BlockLoader.Builder builder) { + public void read(int docId, BlockLoader.StoredFields storedFields, BlockLoader.Builder builder) throws IOException { this.docId = docId; read(docId, (BlockLoader.LongBuilder) builder); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DateScriptFieldType.java b/server/src/main/java/org/elasticsearch/index/mapper/DateScriptFieldType.java index 8252d571dce68..238f7488f6b54 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DateScriptFieldType.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DateScriptFieldType.java @@ -181,7 +181,7 @@ public DocValueFormat docValueFormat(@Nullable String format, ZoneId timeZone) { @Override public BlockLoader blockLoader(BlockLoaderContext blContext) { - return DateScriptBlockDocValuesReader.blockLoader(leafFactory(blContext.lookup())); + return new DateScriptBlockDocValuesReader.DateScriptBlockLoader(leafFactory(blContext.lookup())); } @Override diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DoubleScriptBlockDocValuesReader.java b/server/src/main/java/org/elasticsearch/index/mapper/DoubleScriptBlockDocValuesReader.java index 4e317a3ed11cb..a98f5ff661a78 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DoubleScriptBlockDocValuesReader.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DoubleScriptBlockDocValuesReader.java @@ -8,14 +8,31 @@ package org.elasticsearch.index.mapper; +import org.apache.lucene.index.LeafReaderContext; import org.elasticsearch.script.DoubleFieldScript; +import java.io.IOException; + /** * {@link BlockDocValuesReader} implementation for {@code double} scripts. */ public class DoubleScriptBlockDocValuesReader extends BlockDocValuesReader { - public static BlockLoader blockLoader(DoubleFieldScript.LeafFactory factory) { - return context -> new DoubleScriptBlockDocValuesReader(factory.newInstance(context)); + static class DoubleScriptBlockLoader extends DocValuesBlockLoader { + private final DoubleFieldScript.LeafFactory factory; + + DoubleScriptBlockLoader(DoubleFieldScript.LeafFactory factory) { + this.factory = factory; + } + + @Override + public Builder builder(BlockFactory factory, int expectedCount) { + return factory.doubles(expectedCount); + } + + @Override + public AllReader reader(LeafReaderContext context) throws IOException { + return new DoubleScriptBlockDocValuesReader(factory.newInstance(context)); + } } private final DoubleFieldScript script; @@ -26,18 +43,14 @@ public static BlockLoader blockLoader(DoubleFieldScript.LeafFactory factory) { } @Override - public int docID() { + public int docId() { return docId; } @Override - public BlockLoader.DoubleBuilder builder(BlockLoader.BuilderFactory factory, int expectedCount) { - return factory.doubles(expectedCount); // Note that we don't pre-sort our output so we can't use doublesFromDocValues - } - - @Override - public BlockLoader.Block readValues(BlockLoader.BuilderFactory factory, BlockLoader.Docs docs) { - try (BlockLoader.DoubleBuilder builder = builder(factory, docs.count())) { + public BlockLoader.Block read(BlockLoader.BlockFactory factory, BlockLoader.Docs docs) throws IOException { + // Note that we don't sort the values sort, so we can't use factory.doublesFromDocValues + try (BlockLoader.DoubleBuilder builder = factory.doubles(docs.count())) { for (int i = 0; i < docs.count(); i++) { read(docs.get(i), builder); } @@ -46,7 +59,7 @@ public BlockLoader.Block readValues(BlockLoader.BuilderFactory factory, BlockLoa } @Override - public void readValuesFromSingleDoc(int docId, BlockLoader.Builder builder) { + public void read(int docId, BlockLoader.StoredFields storedFields, BlockLoader.Builder builder) throws IOException { this.docId = docId; read(docId, (BlockLoader.DoubleBuilder) builder); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DoubleScriptFieldType.java b/server/src/main/java/org/elasticsearch/index/mapper/DoubleScriptFieldType.java index ef5c112ef212a..c3f7e782c219a 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DoubleScriptFieldType.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DoubleScriptFieldType.java @@ -107,7 +107,7 @@ public DocValueFormat docValueFormat(String format, ZoneId timeZone) { @Override public BlockLoader blockLoader(BlockLoaderContext blContext) { - return DoubleScriptBlockDocValuesReader.blockLoader(leafFactory(blContext.lookup())); + return new DoubleScriptBlockDocValuesReader.DoubleScriptBlockLoader(leafFactory(blContext.lookup())); } @Override diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IndexFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IndexFieldMapper.java index 5f987fd96ca66..1b2667fe9d2ea 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IndexFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IndexFieldMapper.java @@ -80,42 +80,7 @@ public IndexFieldData.Builder fielddataBuilder(FieldDataContext fieldDataContext @Override public BlockLoader blockLoader(BlockLoaderContext blContext) { - // TODO build a constant block directly - BytesRef bytes = new BytesRef(blContext.indexName()); - return context -> new BlockDocValuesReader() { - private int docId; - - @Override - public int docID() { - return docId; - } - - @Override - public BlockLoader.BytesRefBuilder builder(BlockLoader.BuilderFactory factory, int expectedCount) { - return factory.bytesRefs(expectedCount); - } - - @Override - public BlockLoader.Block readValues(BlockLoader.BuilderFactory factory, BlockLoader.Docs docs) { - try (BlockLoader.BytesRefBuilder builder = builder(factory, docs.count())) { - for (int i = 0; i < docs.count(); i++) { - builder.appendBytesRef(bytes); - } - return builder.build(); - } - } - - @Override - public void readValuesFromSingleDoc(int docId, BlockLoader.Builder builder) { - this.docId = docId; - ((BlockLoader.BytesRefBuilder) builder).appendBytesRef(bytes); - } - - @Override - public String toString() { - return "Index"; - } - }; + return BlockLoader.constantBytes(new BytesRef(blContext.indexName())); } @Override diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java index 80fd384f15fb7..56a50c2dee0aa 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java @@ -408,7 +408,7 @@ public static Query rangeQuery( @Override public BlockLoader blockLoader(BlockLoaderContext blContext) { if (hasDocValues()) { - return BlockDocValuesReader.bytesRefsFromOrds(name()); + return new BlockDocValuesReader.BytesRefsFromOrdsBlockLoader(name()); } return null; } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IpScriptBlockDocValuesReader.java b/server/src/main/java/org/elasticsearch/index/mapper/IpScriptBlockDocValuesReader.java index 23229a6533cdb..ff063555ff05d 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IpScriptBlockDocValuesReader.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IpScriptBlockDocValuesReader.java @@ -8,14 +8,31 @@ package org.elasticsearch.index.mapper; +import org.apache.lucene.index.LeafReaderContext; import org.elasticsearch.script.IpFieldScript; +import java.io.IOException; + /** * {@link BlockDocValuesReader} implementation for keyword scripts. */ public class IpScriptBlockDocValuesReader extends BlockDocValuesReader { - public static BlockLoader blockLoader(IpFieldScript.LeafFactory factory) { - return context -> new IpScriptBlockDocValuesReader(factory.newInstance(context)); + static class IpScriptBlockLoader extends DocValuesBlockLoader { + private final IpFieldScript.LeafFactory factory; + + IpScriptBlockLoader(IpFieldScript.LeafFactory factory) { + this.factory = factory; + } + + @Override + public Builder builder(BlockFactory factory, int expectedCount) { + return factory.bytesRefs(expectedCount); + } + + @Override + public AllReader reader(LeafReaderContext context) throws IOException { + return new IpScriptBlockDocValuesReader(factory.newInstance(context)); + } } private final IpFieldScript script; @@ -26,18 +43,14 @@ public static BlockLoader blockLoader(IpFieldScript.LeafFactory factory) { } @Override - public int docID() { + public int docId() { return docId; } @Override - public BlockLoader.BytesRefBuilder builder(BlockLoader.BuilderFactory factory, int expectedCount) { - return factory.bytesRefs(expectedCount); // Note that we don't pre-sort our output so we can't use bytesRefsFromDocValues - } - - @Override - public BlockLoader.Block readValues(BlockLoader.BuilderFactory factory, BlockLoader.Docs docs) { - try (BlockLoader.BytesRefBuilder builder = builder(factory, docs.count())) { + public BlockLoader.Block read(BlockLoader.BlockFactory factory, BlockLoader.Docs docs) throws IOException { + // Note that we don't pre-sort our output so we can't use bytesRefsFromDocValues + try (BlockLoader.BytesRefBuilder builder = factory.bytesRefs(docs.count())) { for (int i = 0; i < docs.count(); i++) { read(docs.get(i), builder); } @@ -46,7 +59,7 @@ public BlockLoader.Block readValues(BlockLoader.BuilderFactory factory, BlockLoa } @Override - public void readValuesFromSingleDoc(int docId, BlockLoader.Builder builder) { + public void read(int docId, BlockLoader.StoredFields storedFields, BlockLoader.Builder builder) throws IOException { this.docId = docId; read(docId, (BlockLoader.BytesRefBuilder) builder); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IpScriptFieldType.java b/server/src/main/java/org/elasticsearch/index/mapper/IpScriptFieldType.java index 0e56b30e2d5d9..4a64184d5d164 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IpScriptFieldType.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IpScriptFieldType.java @@ -211,6 +211,6 @@ private Query cidrQuery(String term, SearchExecutionContext context) { @Override public BlockLoader blockLoader(BlockLoaderContext blContext) { - return IpScriptBlockDocValuesReader.blockLoader(leafFactory(blContext.lookup())); + return new IpScriptBlockDocValuesReader.IpScriptBlockLoader(leafFactory(blContext.lookup())); } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java index f15bb0069570f..caac5b7f3bfe0 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java @@ -580,7 +580,7 @@ NamedAnalyzer normalizer() { @Override public BlockLoader blockLoader(BlockLoaderContext blContext) { if (hasDocValues()) { - return BlockDocValuesReader.bytesRefsFromOrds(name()); + return new BlockDocValuesReader.BytesRefsFromOrdsBlockLoader(name()); } if (isSyntheticSource) { if (false == isStored()) { @@ -590,9 +590,9 @@ public BlockLoader blockLoader(BlockLoaderContext blContext) { + "] is only supported in synthetic _source index if it creates doc values or stored fields" ); } - return BlockStoredFieldsReader.bytesRefsFromBytesRefs(name()); + return new BlockStoredFieldsReader.BytesFromBytesRefsBlockLoader(name()); } - return BlockSourceReader.bytesRefs(sourceValueFetcher(blContext.sourcePaths(name()))); + return new BlockSourceReader.BytesRefsBlockLoader(sourceValueFetcher(blContext.sourcePaths(name()))); } @Override diff --git a/server/src/main/java/org/elasticsearch/index/mapper/KeywordScriptBlockDocValuesReader.java b/server/src/main/java/org/elasticsearch/index/mapper/KeywordScriptBlockDocValuesReader.java index 6afbcae50d31f..df5ba51755c2a 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/KeywordScriptBlockDocValuesReader.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/KeywordScriptBlockDocValuesReader.java @@ -8,15 +8,32 @@ package org.elasticsearch.index.mapper; +import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.util.BytesRefBuilder; import org.elasticsearch.script.StringFieldScript; +import java.io.IOException; + /** * {@link BlockDocValuesReader} implementation for keyword scripts. */ public class KeywordScriptBlockDocValuesReader extends BlockDocValuesReader { - public static BlockLoader blockLoader(StringFieldScript.LeafFactory factory) { - return context -> new KeywordScriptBlockDocValuesReader(factory.newInstance(context)); + static class KeywordScriptBlockLoader extends DocValuesBlockLoader { + private final StringFieldScript.LeafFactory factory; + + KeywordScriptBlockLoader(StringFieldScript.LeafFactory factory) { + this.factory = factory; + } + + @Override + public Builder builder(BlockFactory factory, int expectedCount) { + return factory.bytesRefs(expectedCount); + } + + @Override + public AllReader reader(LeafReaderContext context) throws IOException { + return new KeywordScriptBlockDocValuesReader(factory.newInstance(context)); + } } private final BytesRefBuilder bytesBuild = new BytesRefBuilder(); @@ -28,18 +45,14 @@ public static BlockLoader blockLoader(StringFieldScript.LeafFactory factory) { } @Override - public int docID() { + public int docId() { return docId; } @Override - public BlockLoader.BytesRefBuilder builder(BlockLoader.BuilderFactory factory, int expectedCount) { - return factory.bytesRefs(expectedCount); // Note that we don't pre-sort our output so we can't use bytesRefsFromDocValues - } - - @Override - public BlockLoader.Block readValues(BlockLoader.BuilderFactory factory, BlockLoader.Docs docs) { - try (BlockLoader.BytesRefBuilder builder = builder(factory, docs.count())) { + public BlockLoader.Block read(BlockLoader.BlockFactory factory, BlockLoader.Docs docs) throws IOException { + // Note that we don't pre-sort our output so we can't use bytesRefsFromDocValues + try (BlockLoader.BytesRefBuilder builder = factory.bytesRefs(docs.count())) { for (int i = 0; i < docs.count(); i++) { read(docs.get(i), builder); } @@ -48,7 +61,7 @@ public BlockLoader.Block readValues(BlockLoader.BuilderFactory factory, BlockLoa } @Override - public void readValuesFromSingleDoc(int docId, BlockLoader.Builder builder) { + public void read(int docId, BlockLoader.StoredFields storedFields, BlockLoader.Builder builder) throws IOException { this.docId = docId; read(docId, (BlockLoader.BytesRefBuilder) builder); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/KeywordScriptFieldType.java b/server/src/main/java/org/elasticsearch/index/mapper/KeywordScriptFieldType.java index 879a28d4c76c8..188f0ae508fcc 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/KeywordScriptFieldType.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/KeywordScriptFieldType.java @@ -112,7 +112,7 @@ public Object valueForDisplay(Object value) { @Override public BlockLoader blockLoader(BlockLoaderContext blContext) { - return KeywordScriptBlockDocValuesReader.blockLoader(leafFactory(blContext.lookup())); + return new KeywordScriptBlockDocValuesReader.KeywordScriptBlockLoader(leafFactory(blContext.lookup())); } @Override diff --git a/server/src/main/java/org/elasticsearch/index/mapper/LongScriptBlockDocValuesReader.java b/server/src/main/java/org/elasticsearch/index/mapper/LongScriptBlockDocValuesReader.java index 91c099cd2813b..73ad359147571 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/LongScriptBlockDocValuesReader.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/LongScriptBlockDocValuesReader.java @@ -8,14 +8,31 @@ package org.elasticsearch.index.mapper; +import org.apache.lucene.index.LeafReaderContext; import org.elasticsearch.script.LongFieldScript; +import java.io.IOException; + /** * {@link BlockDocValuesReader} implementation for {@code long} scripts. */ public class LongScriptBlockDocValuesReader extends BlockDocValuesReader { - public static BlockLoader blockLoader(LongFieldScript.LeafFactory factory) { - return context -> new LongScriptBlockDocValuesReader(factory.newInstance(context)); + static class LongScriptBlockLoader extends DocValuesBlockLoader { + private final LongFieldScript.LeafFactory factory; + + LongScriptBlockLoader(LongFieldScript.LeafFactory factory) { + this.factory = factory; + } + + @Override + public Builder builder(BlockFactory factory, int expectedCount) { + return factory.longs(expectedCount); + } + + @Override + public AllReader reader(LeafReaderContext context) throws IOException { + return new LongScriptBlockDocValuesReader(factory.newInstance(context)); + } } private final LongFieldScript script; @@ -26,18 +43,14 @@ public static BlockLoader blockLoader(LongFieldScript.LeafFactory factory) { } @Override - public int docID() { + public int docId() { return docId; } @Override - public BlockLoader.LongBuilder builder(BlockLoader.BuilderFactory factory, int expectedCount) { - return factory.longs(expectedCount); // Note that we don't pre-sort our output so we can't use longsFromDocValues - } - - @Override - public BlockLoader.Block readValues(BlockLoader.BuilderFactory factory, BlockLoader.Docs docs) { - try (BlockLoader.LongBuilder builder = builder(factory, docs.count())) { + public BlockLoader.Block read(BlockLoader.BlockFactory factory, BlockLoader.Docs docs) throws IOException { + // Note that we don't pre-sort our output so we can't use longsFromDocValues + try (BlockLoader.LongBuilder builder = factory.longs(docs.count())) { for (int i = 0; i < docs.count(); i++) { read(docs.get(i), builder); } @@ -46,7 +59,7 @@ public BlockLoader.Block readValues(BlockLoader.BuilderFactory factory, BlockLoa } @Override - public void readValuesFromSingleDoc(int docId, BlockLoader.Builder builder) { + public void read(int docId, BlockLoader.StoredFields storedFields, BlockLoader.Builder builder) throws IOException { this.docId = docId; read(docId, (BlockLoader.LongBuilder) builder); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/LongScriptFieldType.java b/server/src/main/java/org/elasticsearch/index/mapper/LongScriptFieldType.java index f89babe32d0a9..f099ee3625922 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/LongScriptFieldType.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/LongScriptFieldType.java @@ -107,7 +107,7 @@ public DocValueFormat docValueFormat(String format, ZoneId timeZone) { @Override public BlockLoader blockLoader(BlockLoaderContext blContext) { - return LongScriptBlockDocValuesReader.blockLoader(leafFactory(blContext.lookup())); + return new LongScriptBlockDocValuesReader.LongScriptBlockLoader(leafFactory(blContext.lookup())); } @Override diff --git a/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java index 84e9e84fb8ceb..091e3c61764b0 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java @@ -440,12 +440,12 @@ protected void writeValue(XContentBuilder b, long value) throws IOException { @Override BlockLoader blockLoaderFromDocValues(String fieldName) { - return BlockDocValuesReader.doubles(fieldName, l -> HalfFloatPoint.sortableShortToHalfFloat((short) l)); + return new BlockDocValuesReader.DoublesBlockLoader(fieldName, l -> HalfFloatPoint.sortableShortToHalfFloat((short) l)); } @Override BlockLoader blockLoaderFromSource(SourceValueFetcher sourceValueFetcher) { - return BlockSourceReader.doubles(sourceValueFetcher); + return new BlockSourceReader.DoublesBlockLoader(sourceValueFetcher); } }, FLOAT("float", NumericType.FLOAT) { @@ -602,12 +602,12 @@ protected void writeValue(XContentBuilder b, long value) throws IOException { @Override BlockLoader blockLoaderFromDocValues(String fieldName) { - return BlockDocValuesReader.doubles(fieldName, l -> NumericUtils.sortableIntToFloat((int) l)); + return new BlockDocValuesReader.DoublesBlockLoader(fieldName, l -> NumericUtils.sortableIntToFloat((int) l)); } @Override BlockLoader blockLoaderFromSource(SourceValueFetcher sourceValueFetcher) { - return BlockSourceReader.doubles(sourceValueFetcher); + return new BlockSourceReader.DoublesBlockLoader(sourceValueFetcher); } }, DOUBLE("double", NumericType.DOUBLE) { @@ -742,12 +742,12 @@ protected void writeValue(XContentBuilder b, long value) throws IOException { @Override BlockLoader blockLoaderFromDocValues(String fieldName) { - return BlockDocValuesReader.doubles(fieldName, NumericUtils::sortableLongToDouble); + return new BlockDocValuesReader.DoublesBlockLoader(fieldName, NumericUtils::sortableLongToDouble); } @Override BlockLoader blockLoaderFromSource(SourceValueFetcher sourceValueFetcher) { - return BlockSourceReader.doubles(sourceValueFetcher); + return new BlockSourceReader.DoublesBlockLoader(sourceValueFetcher); } }, BYTE("byte", NumericType.BYTE) { @@ -845,12 +845,12 @@ SourceLoader.SyntheticFieldLoader syntheticFieldLoader(String fieldName, String @Override BlockLoader blockLoaderFromDocValues(String fieldName) { - return BlockDocValuesReader.ints(fieldName); + return new BlockDocValuesReader.IntsBlockLoader(fieldName); } @Override BlockLoader blockLoaderFromSource(SourceValueFetcher sourceValueFetcher) { - return BlockSourceReader.ints(sourceValueFetcher); + return new BlockSourceReader.IntsBlockLoader(sourceValueFetcher); } }, SHORT("short", NumericType.SHORT) { @@ -944,12 +944,12 @@ SourceLoader.SyntheticFieldLoader syntheticFieldLoader(String fieldName, String @Override BlockLoader blockLoaderFromDocValues(String fieldName) { - return BlockDocValuesReader.ints(fieldName); + return new BlockDocValuesReader.IntsBlockLoader(fieldName); } @Override BlockLoader blockLoaderFromSource(SourceValueFetcher sourceValueFetcher) { - return BlockSourceReader.ints(sourceValueFetcher); + return new BlockSourceReader.IntsBlockLoader(sourceValueFetcher); } }, INTEGER("integer", NumericType.INT) { @@ -1111,12 +1111,12 @@ SourceLoader.SyntheticFieldLoader syntheticFieldLoader(String fieldName, String @Override BlockLoader blockLoaderFromDocValues(String fieldName) { - return BlockDocValuesReader.ints(fieldName); + return new BlockDocValuesReader.IntsBlockLoader(fieldName); } @Override BlockLoader blockLoaderFromSource(SourceValueFetcher sourceValueFetcher) { - return BlockSourceReader.ints(sourceValueFetcher); + return new BlockSourceReader.IntsBlockLoader(sourceValueFetcher); } }, LONG("long", NumericType.LONG) { @@ -1248,12 +1248,12 @@ SourceLoader.SyntheticFieldLoader syntheticFieldLoader(String fieldName, String @Override BlockLoader blockLoaderFromDocValues(String fieldName) { - return BlockDocValuesReader.longs(fieldName); + return new BlockDocValuesReader.LongsBlockLoader(fieldName); } @Override BlockLoader blockLoaderFromSource(SourceValueFetcher sourceValueFetcher) { - return BlockSourceReader.longs(sourceValueFetcher); + return new BlockSourceReader.LongsBlockLoader(sourceValueFetcher); } }; @@ -1656,7 +1656,7 @@ public Function pointReaderIfPossible() { public BlockLoader blockLoader(BlockLoaderContext blContext) { if (indexMode == IndexMode.TIME_SERIES && metricType == TimeSeriesParams.MetricType.COUNTER) { // Counters are not supported by ESQL so we load them in null - return BlockDocValuesReader.nulls(); + return BlockLoader.CONSTANT_NULLS; } if (hasDocValues()) { return type.blockLoaderFromDocValues(name()); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/ProvidedIdFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/ProvidedIdFieldMapper.java index f681d54ebbead..d8a4177ee3211 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/ProvidedIdFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/ProvidedIdFieldMapper.java @@ -119,7 +119,7 @@ public Query termsQuery(Collection values, SearchExecutionContext context) { @Override public BlockLoader blockLoader(BlockLoaderContext blContext) { - return BlockStoredFieldsReader.id(); + return new BlockStoredFieldsReader.IdBlockLoader(); } @Override diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java index 5a0d9c7c0cf79..420f92cfbf847 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java @@ -678,7 +678,7 @@ public TextFieldType( super(name, indexed, stored, false, tsi, meta); fielddata = false; this.isSyntheticSource = isSyntheticSource; - this.syntheticSourceDelegate = syntheticSourceDelegate; + this.syntheticSourceDelegate = syntheticSourceDelegate; // TODO rename to "exactDelegate" or something this.eagerGlobalOrdinals = eagerGlobalOrdinals; this.indexPhrases = indexPhrases; } @@ -939,11 +939,16 @@ public boolean isAggregatable() { @Override public BlockLoader blockLoader(BlockLoaderContext blContext) { if (syntheticSourceDelegate != null) { - return syntheticSourceDelegate.blockLoader(blContext); + return new BlockLoader.Delegating(syntheticSourceDelegate.blockLoader(blContext)) { + @Override + protected String delegatingTo() { + return syntheticSourceDelegate.name(); + } + }; } if (isSyntheticSource) { if (isStored()) { - return BlockStoredFieldsReader.bytesRefsFromStrings(name()); + return new BlockStoredFieldsReader.BytesFromStringsBlockLoader(name()); } /* * We *shouldn't fall to this exception. The mapping should be @@ -957,7 +962,7 @@ public BlockLoader blockLoader(BlockLoaderContext blContext) { + "] is not supported because synthetic _source is enabled and we don't have a way to load the fields" ); } - return BlockSourceReader.bytesRefs(SourceValueFetcher.toString(blContext.sourcePaths(name()))); + return new BlockSourceReader.BytesRefsBlockLoader(SourceValueFetcher.toString(blContext.sourcePaths(name()))); } @Override @@ -1034,6 +1039,10 @@ protected BytesRef storedToBytesRef(Object stored) { public boolean isSyntheticSource() { return isSyntheticSource; } + + KeywordFieldMapper.KeywordFieldType syntheticSourceDelegate() { + return syntheticSourceDelegate; + } } public static class ConstantScoreTextFieldType extends TextFieldType { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapper.java index 9d43ef398feac..9245e78602eb7 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapper.java @@ -89,7 +89,7 @@ public Query termsQuery(Collection values, SearchExecutionContext context) { @Override public BlockLoader blockLoader(BlockLoaderContext blContext) { - return BlockStoredFieldsReader.id(); + return new BlockStoredFieldsReader.IdBlockLoader(); } @Override diff --git a/server/src/main/java/org/elasticsearch/index/mapper/VersionFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/VersionFieldMapper.java index 54a44dd55caa4..8f69f6afe47db 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/VersionFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/VersionFieldMapper.java @@ -56,7 +56,7 @@ public ValueFetcher valueFetcher(SearchExecutionContext context, String format) @Override public BlockLoader blockLoader(BlockLoaderContext blContext) { - return BlockDocValuesReader.longs(name()); + return new BlockDocValuesReader.LongsBlockLoader(name()); } @Override diff --git a/server/src/main/java/org/elasticsearch/search/fetch/StoredFieldsSpec.java b/server/src/main/java/org/elasticsearch/search/fetch/StoredFieldsSpec.java index 48aea98887ff0..87cbf9b1d6b85 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/StoredFieldsSpec.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/StoredFieldsSpec.java @@ -38,6 +38,9 @@ public boolean noRequirements() { * Combine these stored field requirements with those from another StoredFieldsSpec */ public StoredFieldsSpec merge(StoredFieldsSpec other) { + if (this == other) { + return this; + } Set mergedFields = new HashSet<>(this.requiredStoredFields); mergedFields.addAll(other.requiredStoredFields); return new StoredFieldsSpec( diff --git a/server/src/test/java/org/elasticsearch/index/mapper/BooleanScriptFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/BooleanScriptFieldTypeTests.java index 8d5a47f08c663..d8f063ece35c0 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/BooleanScriptFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/BooleanScriptFieldTypeTests.java @@ -417,8 +417,8 @@ public void testBlockLoader() throws IOException { try (DirectoryReader reader = iw.getReader()) { BooleanScriptFieldType fieldType = build("xor_param", Map.of("param", false), OnScriptError.FAIL); List expected = List.of(false, true); - assertThat(blockLoaderReadValues(reader, fieldType), equalTo(expected)); - assertThat(blockLoaderReadValuesFromSingleDoc(reader, fieldType), equalTo(expected)); + assertThat(blockLoaderReadValuesFromColumnAtATimeReader(reader, fieldType), equalTo(expected)); + assertThat(blockLoaderReadValuesFromRowStrideReader(reader, fieldType), equalTo(expected)); } } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DateScriptFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DateScriptFieldTypeTests.java index d1652b9f57716..eb3daf472ea2e 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DateScriptFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DateScriptFieldTypeTests.java @@ -477,8 +477,11 @@ public void testBlockLoader() throws IOException { iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"timestamp\": [1595432181355]}")))); try (DirectoryReader reader = iw.getReader()) { DateScriptFieldType fieldType = build("add_days", Map.of("days", 1), OnScriptError.FAIL); - assertThat(blockLoaderReadValues(reader, fieldType), equalTo(List.of(1595518581354L, 1595518581355L))); - assertThat(blockLoaderReadValuesFromSingleDoc(reader, fieldType), equalTo(List.of(1595518581354L, 1595518581355L))); + assertThat( + blockLoaderReadValuesFromColumnAtATimeReader(reader, fieldType), + equalTo(List.of(1595518581354L, 1595518581355L)) + ); + assertThat(blockLoaderReadValuesFromRowStrideReader(reader, fieldType), equalTo(List.of(1595518581354L, 1595518581355L))); } } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DoubleScriptFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DoubleScriptFieldTypeTests.java index 0f05dad8098f4..d37e42e04edca 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DoubleScriptFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DoubleScriptFieldTypeTests.java @@ -236,8 +236,8 @@ public void testBlockLoader() throws IOException { iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": [2]}")))); try (DirectoryReader reader = iw.getReader()) { DoubleScriptFieldType fieldType = build("add_param", Map.of("param", 1), OnScriptError.FAIL); - assertThat(blockLoaderReadValues(reader, fieldType), equalTo(List.of(2d, 3d))); - assertThat(blockLoaderReadValuesFromSingleDoc(reader, fieldType), equalTo(List.of(2d, 3d))); + assertThat(blockLoaderReadValuesFromColumnAtATimeReader(reader, fieldType), equalTo(List.of(2d, 3d))); + assertThat(blockLoaderReadValuesFromRowStrideReader(reader, fieldType), equalTo(List.of(2d, 3d))); } } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IpScriptFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IpScriptFieldTypeTests.java index 56ca5f3dae89f..cd19bb50b842c 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/IpScriptFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/IpScriptFieldTypeTests.java @@ -256,8 +256,8 @@ public void testBlockLoader() throws IOException { new BytesRef(InetAddressPoint.encode(InetAddresses.forString("192.168.0.1"))), new BytesRef(InetAddressPoint.encode(InetAddresses.forString("192.168.1.1"))) ); - assertThat(blockLoaderReadValues(reader, fieldType), equalTo(expected)); - assertThat(blockLoaderReadValuesFromSingleDoc(reader, fieldType), equalTo(expected)); + assertThat(blockLoaderReadValuesFromColumnAtATimeReader(reader, fieldType), equalTo(expected)); + assertThat(blockLoaderReadValuesFromRowStrideReader(reader, fieldType), equalTo(expected)); } } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/KeywordScriptFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/KeywordScriptFieldTypeTests.java index 65f4c2e3ea6eb..ce705f2e9ae8b 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/KeywordScriptFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/KeywordScriptFieldTypeTests.java @@ -382,9 +382,12 @@ public void testBlockLoader() throws IOException { iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": [2]}")))); try (DirectoryReader reader = iw.getReader()) { KeywordScriptFieldType fieldType = build("append_param", Map.of("param", "-Suffix"), OnScriptError.FAIL); - assertThat(blockLoaderReadValues(reader, fieldType), equalTo(List.of(new BytesRef("1-Suffix"), new BytesRef("2-Suffix")))); assertThat( - blockLoaderReadValuesFromSingleDoc(reader, fieldType), + blockLoaderReadValuesFromColumnAtATimeReader(reader, fieldType), + equalTo(List.of(new BytesRef("1-Suffix"), new BytesRef("2-Suffix"))) + ); + assertThat( + blockLoaderReadValuesFromRowStrideReader(reader, fieldType), equalTo(List.of(new BytesRef("1-Suffix"), new BytesRef("2-Suffix"))) ); } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/LongScriptFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/LongScriptFieldTypeTests.java index 1688cab24af3e..fd20b6c71e984 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/LongScriptFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/LongScriptFieldTypeTests.java @@ -269,8 +269,8 @@ public void testBlockLoader() throws IOException { iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": [2]}")))); try (DirectoryReader reader = iw.getReader()) { LongScriptFieldType fieldType = build("add_param", Map.of("param", 1), OnScriptError.FAIL); - assertThat(blockLoaderReadValues(reader, fieldType), equalTo(List.of(2L, 3L))); - assertThat(blockLoaderReadValuesFromSingleDoc(reader, fieldType), equalTo(List.of(2L, 3L))); + assertThat(blockLoaderReadValuesFromColumnAtATimeReader(reader, fieldType), equalTo(List.of(2L, 3L))); + assertThat(blockLoaderReadValuesFromRowStrideReader(reader, fieldType), equalTo(List.of(2L, 3L))); } } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java index bbfeaaa8b9d69..b2a729d6868d2 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java @@ -1324,4 +1324,10 @@ public void testEmpty() throws Exception { assertFalse(dv.advanceExact(3)); }); } + + @Override + protected boolean supportsColumnAtATimeReader(MappedFieldType ft) { + TextFieldMapper.TextFieldType text = (TextFieldType) ft; + return text.syntheticSourceDelegate() != null && text.syntheticSourceDelegate().hasDocValues(); + } } diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/AbstractScriptFieldTypeTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/AbstractScriptFieldTypeTestCase.java index 56ad35bee83d5..7eb2511f58206 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/AbstractScriptFieldTypeTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/AbstractScriptFieldTypeTestCase.java @@ -383,11 +383,12 @@ public final void testCacheable() throws IOException { } } - protected final List blockLoaderReadValues(DirectoryReader reader, MappedFieldType fieldType) throws IOException { + protected final List blockLoaderReadValuesFromColumnAtATimeReader(DirectoryReader reader, MappedFieldType fieldType) + throws IOException { BlockLoader loader = fieldType.blockLoader(blContext()); List all = new ArrayList<>(); for (LeafReaderContext ctx : reader.leaves()) { - TestBlock block = (TestBlock) loader.reader(ctx).readValues(TestBlock.FACTORY, TestBlock.docs(ctx)); + TestBlock block = (TestBlock) loader.columnAtATimeReader(ctx).read(TestBlock.FACTORY, TestBlock.docs(ctx)); for (int i = 0; i < block.size(); i++) { all.add(block.get(i)); } @@ -395,15 +396,17 @@ protected final List blockLoaderReadValues(DirectoryReader reader, Mappe return all; } - protected final List blockLoaderReadValuesFromSingleDoc(DirectoryReader reader, MappedFieldType fieldType) throws IOException { + protected final List blockLoaderReadValuesFromRowStrideReader(DirectoryReader reader, MappedFieldType fieldType) + throws IOException { BlockLoader loader = fieldType.blockLoader(blContext()); List all = new ArrayList<>(); for (LeafReaderContext ctx : reader.leaves()) { - BlockDocValuesReader blockReader = loader.reader(ctx); - TestBlock block = (TestBlock) blockReader.builder(TestBlock.FACTORY, ctx.reader().numDocs()); + BlockLoader.RowStrideReader blockReader = loader.rowStrideReader(ctx); + BlockLoader.Builder builder = loader.builder(TestBlock.FACTORY, ctx.reader().numDocs()); for (int i = 0; i < ctx.reader().numDocs(); i++) { - blockReader.readValuesFromSingleDoc(i, block); + blockReader.read(i, null, builder); } + TestBlock block = (TestBlock) builder.build(); for (int i = 0; i < block.size(); i++) { all.add(block.get(i)); } diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java index e34072fbf1668..d68324ff902e2 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java @@ -31,7 +31,6 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.core.CheckedConsumer; -import org.elasticsearch.core.CheckedFunction; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.IndexVersions; @@ -1240,19 +1239,19 @@ public final void testSyntheticEmptyListNoDocValuesLoader() throws IOException { assertNoDocValueLoader(b -> b.startArray("field").endArray()); } - public final void testBlockLoaderReadValues() throws IOException { - testBlockLoader(blockReader -> (TestBlock) blockReader.readValues(TestBlock.FACTORY, TestBlock.docs(0))); + public final void testBlockLoaderFromColumnReader() throws IOException { + testBlockLoader(true); } - public final void testBlockLoaderReadValuesFromSingleDoc() throws IOException { - testBlockLoader(blockReader -> { - TestBlock block = (TestBlock) blockReader.builder(TestBlock.FACTORY, 1); - blockReader.readValuesFromSingleDoc(0, block); - return block; - }); + public final void testBlockLoaderFromRowStrideReader() throws IOException { + testBlockLoader(false); + } + + protected boolean supportsColumnAtATimeReader(MappedFieldType ft) { + return ft.hasDocValues(); } - private void testBlockLoader(CheckedFunction body) throws IOException { + private void testBlockLoader(boolean columnReader) throws IOException { SyntheticSourceExample example = syntheticSourceSupport(false).example(5); MapperService mapper = createMapperService(syntheticSourceMapping(b -> { b.startObject("field"); @@ -1289,7 +1288,25 @@ public Set sourcePaths(String name) { iw.addDocument(doc); iw.close(); try (DirectoryReader reader = DirectoryReader.open(directory)) { - TestBlock block = body.apply(loader.reader(reader.leaves().get(0))); + LeafReaderContext ctx = reader.leaves().get(0); + TestBlock block; + if (columnReader) { + if (supportsColumnAtATimeReader(mapper.fieldType("field"))) { + block = (TestBlock) loader.columnAtATimeReader(ctx).read(TestBlock.FACTORY, TestBlock.docs(0)); + } else { + assertNull(loader.columnAtATimeReader(ctx)); + return; + } + } else { + BlockLoaderStoredFieldsFromLeafLoader storedFieldsLoader = new BlockLoaderStoredFieldsFromLeafLoader( + StoredFieldLoader.fromSpec(loader.rowStrideStoredFieldSpec()).getLoader(ctx, null), + loader.rowStrideStoredFieldSpec().requiresSource() + ); + storedFieldsLoader.advanceTo(0); + BlockLoader.Builder builder = loader.builder(TestBlock.FACTORY, 1); + loader.rowStrideReader(ctx).read(0, storedFieldsLoader, builder); + block = (TestBlock) builder.build(); + } Object inBlock = block.get(0); if (inBlock != null) { if (inBlock instanceof List l) { @@ -1319,7 +1336,7 @@ public Set sourcePaths(String name) { } /** - * Matcher for {@link #testBlockLoaderReadValues} and {@link #testBlockLoaderReadValuesFromSingleDoc}. + * Matcher for {@link #testBlockLoaderFromColumnReader} and {@link #testBlockLoaderFromRowStrideReader}. */ protected Matcher blockItemMatcher(Object expected) { return equalTo(expected); diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/TestBlock.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/TestBlock.java index 298acb9519532..30dece5767b61 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/TestBlock.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/TestBlock.java @@ -11,7 +11,6 @@ import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.SortedDocValues; import org.apache.lucene.util.BytesRef; -import org.elasticsearch.core.Nullable; import java.io.IOException; import java.io.UncheckedIOException; @@ -21,74 +20,130 @@ import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; -public class TestBlock - implements - BlockLoader.BooleanBuilder, - BlockLoader.BytesRefBuilder, - BlockLoader.DoubleBuilder, - BlockLoader.IntBuilder, - BlockLoader.LongBuilder, - BlockLoader.SingletonOrdinalsBuilder, - BlockLoader.Block { - public static BlockLoader.BuilderFactory FACTORY = new BlockLoader.BuilderFactory() { +public class TestBlock implements BlockLoader.Block { + public static BlockLoader.BlockFactory FACTORY = new BlockLoader.BlockFactory() { @Override public BlockLoader.BooleanBuilder booleansFromDocValues(int expectedCount) { - return new TestBlock(null); + return booleans(expectedCount); } @Override public BlockLoader.BooleanBuilder booleans(int expectedCount) { - return new TestBlock(null); + class BooleansBuilder extends TestBlock.Builder implements BlockLoader.BooleanBuilder { + @Override + public BooleansBuilder appendBoolean(boolean value) { + add(value); + return this; + } + } + return new BooleansBuilder(); } @Override public BlockLoader.BytesRefBuilder bytesRefsFromDocValues(int expectedCount) { - return new TestBlock(null); + return bytesRefs(expectedCount); } @Override public BlockLoader.BytesRefBuilder bytesRefs(int expectedCount) { - return new TestBlock(null); + class BytesRefsBuilder extends TestBlock.Builder implements BlockLoader.BytesRefBuilder { + @Override + public BytesRefsBuilder appendBytesRef(BytesRef value) { + add(BytesRef.deepCopyOf(value)); + return this; + } + } + return new BytesRefsBuilder(); } @Override public BlockLoader.DoubleBuilder doublesFromDocValues(int expectedCount) { - return new TestBlock(null); + return doubles(expectedCount); } @Override public BlockLoader.DoubleBuilder doubles(int expectedCount) { - return new TestBlock(null); + class DoublesBuilder extends TestBlock.Builder implements BlockLoader.DoubleBuilder { + @Override + public DoublesBuilder appendDouble(double value) { + add(value); + return this; + } + } + return new DoublesBuilder(); } @Override public BlockLoader.IntBuilder intsFromDocValues(int expectedCount) { - return new TestBlock(null); + return ints(expectedCount); } @Override public BlockLoader.IntBuilder ints(int expectedCount) { - return new TestBlock(null); + class IntsBuilder extends TestBlock.Builder implements BlockLoader.IntBuilder { + @Override + public IntsBuilder appendInt(int value) { + add(value); + return this; + } + } + return new IntsBuilder(); } @Override public BlockLoader.LongBuilder longsFromDocValues(int expectedCount) { - return new TestBlock(null); + return longs(expectedCount); } @Override public BlockLoader.LongBuilder longs(int expectedCount) { - return new TestBlock(null); + class LongsBuilder extends TestBlock.Builder implements BlockLoader.LongBuilder { + @Override + public LongsBuilder appendLong(long value) { + add(value); + return this; + } + } + return new LongsBuilder(); } @Override public BlockLoader.Builder nulls(int expectedCount) { - return new TestBlock(null); + return longs(expectedCount); + } + + @Override + public BlockLoader.Block constantNulls(int size) { + BlockLoader.LongBuilder builder = longs(size); + for (int i = 0; i < size; i++) { + builder.appendNull(); + } + return builder.build(); + } + + @Override + public BlockLoader.Block constantBytes(BytesRef value, int size) { + BlockLoader.BytesRefBuilder builder = bytesRefs(size); + for (int i = 0; i < size; i++) { + builder.appendBytesRef(value); + } + return builder.build(); } @Override public BlockLoader.SingletonOrdinalsBuilder singletonOrdinalsBuilder(SortedDocValues ordinals, int count) { - return new TestBlock(ordinals); + class SingletonOrdsBuilder extends TestBlock.Builder implements BlockLoader.SingletonOrdinalsBuilder { + @Override + public SingletonOrdsBuilder appendOrd(int value) { + try { + add(ordinals.lookupOrd(value)); + return this; + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + } + return new SingletonOrdsBuilder(); } }; @@ -120,13 +175,10 @@ public int get(int i) { }; } - private final SortedDocValues sortedDocValues; - private final List values = new ArrayList<>(); - - private List currentPosition = null; + private final List values; - private TestBlock(@Nullable SortedDocValues sortedDocValues) { - this.sortedDocValues = sortedDocValues; + private TestBlock(List values) { + this.values = values; } public Object get(int i) { @@ -138,73 +190,49 @@ public int size() { } @Override - public TestBlock appendNull() { - assertNull(currentPosition); - values.add(null); - return this; - } - - @Override - public TestBlock beginPositionEntry() { - assertNull(currentPosition); - currentPosition = new ArrayList<>(); - values.add(currentPosition); - return this; - } - - @Override - public TestBlock endPositionEntry() { - assertNotNull(currentPosition); - currentPosition = null; - return this; - } - - @Override - public TestBlock appendBoolean(boolean value) { - return add(value); + public void close() { + // TODO assert that we close the test blocks } - @Override - public TestBlock appendBytesRef(BytesRef value) { - return add(BytesRef.deepCopyOf(value)); - } + private abstract static class Builder implements BlockLoader.Builder { + private final List values = new ArrayList<>(); - @Override - public TestBlock appendDouble(double value) { - return add(value); - } + private List currentPosition = null; - @Override - public TestBlock appendInt(int value) { - return add(value); - } + @Override + public Builder appendNull() { + assertNull(currentPosition); + values.add(null); + return this; + } - @Override - public TestBlock appendLong(long value) { - return add(value); - } + @Override + public Builder beginPositionEntry() { + assertNull(currentPosition); + currentPosition = new ArrayList<>(); + values.add(currentPosition); + return this; + } - @Override - public TestBlock appendOrd(int value) { - try { - return add(sortedDocValues.lookupOrd(value)); - } catch (IOException e) { - throw new UncheckedIOException(e); + @Override + public Builder endPositionEntry() { + assertNotNull(currentPosition); + currentPosition = null; + return this; } - } - @Override - public TestBlock build() { - return this; - } + protected void add(Object value) { + (currentPosition == null ? values : currentPosition).add(value); + } - private TestBlock add(Object value) { - (currentPosition == null ? values : currentPosition).add(value); - return this; - } + @Override + public TestBlock build() { + return new TestBlock(values); + } - @Override - public void close() { - // TODO assert that we close the test blocks + @Override + public void close() { + // TODO assert that we close the test block builders + } } } diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/BlockReaderFactories.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/BlockReaderFactories.java index a0d08bc798fbb..a730931208663 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/BlockReaderFactories.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/BlockReaderFactories.java @@ -7,17 +7,13 @@ package org.elasticsearch.compute.lucene; -import org.apache.lucene.index.IndexReader; -import org.apache.lucene.index.SortedSetDocValues; import org.elasticsearch.common.logging.HeaderWarning; -import org.elasticsearch.index.mapper.BlockDocValuesReader; import org.elasticsearch.index.mapper.BlockLoader; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.search.internal.SearchContext; import org.elasticsearch.search.lookup.SearchLookup; -import java.io.IOException; import java.util.ArrayList; import java.util.List; import java.util.Set; @@ -36,23 +32,19 @@ private BlockReaderFactories() {} * @param asUnsupportedSource should the field be loaded as "unsupported"? * These will always have {@code null} values */ - public static List factories( - List searchContexts, - String fieldName, - boolean asUnsupportedSource - ) { - List factories = new ArrayList<>(searchContexts.size()); + public static List loaders(List searchContexts, String fieldName, boolean asUnsupportedSource) { + List loaders = new ArrayList<>(searchContexts.size()); for (SearchContext searchContext : searchContexts) { SearchExecutionContext ctx = searchContext.getSearchExecutionContext(); if (asUnsupportedSource) { - factories.add(loaderToFactory(ctx.getIndexReader(), BlockDocValuesReader.nulls())); + loaders.add(BlockLoader.CONSTANT_NULLS); continue; } MappedFieldType fieldType = ctx.getFieldType(fieldName); if (fieldType == null) { // the field does not exist in this context - factories.add(loaderToFactory(ctx.getIndexReader(), BlockDocValuesReader.nulls())); + loaders.add(BlockLoader.CONSTANT_NULLS); continue; } BlockLoader loader = fieldType.blockLoader(new MappedFieldType.BlockLoaderContext() { @@ -73,36 +65,12 @@ public Set sourcePaths(String name) { }); if (loader == null) { HeaderWarning.addWarning("Field [{}] cannot be retrieved, it is unsupported or not indexed; returning null", fieldName); - factories.add(loaderToFactory(ctx.getIndexReader(), BlockDocValuesReader.nulls())); + loaders.add(BlockLoader.CONSTANT_NULLS); continue; } - factories.add(loaderToFactory(ctx.getIndexReader(), loader)); + loaders.add(loader); } - return factories; - } - - /** - * Converts a {@link BlockLoader}, something defined in core elasticsearch at - * the field level, into a {@link BlockDocValuesReader.Factory} which can be - * used inside ESQL. - */ - public static BlockDocValuesReader.Factory loaderToFactory(IndexReader reader, BlockLoader loader) { - return new BlockDocValuesReader.Factory() { - @Override - public BlockDocValuesReader build(int segment) throws IOException { - return loader.reader(reader.leaves().get(segment)); - } - - @Override - public boolean supportsOrdinals() { - return loader.supportsOrdinals(); - } - - @Override - public SortedSetDocValues ordinals(int segment) throws IOException { - return loader.ordinals(reader.leaves().get(segment)); - } - }; + return loaders; } } diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/ValuesSourceReaderOperator.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/ValuesSourceReaderOperator.java index 61c1bd9730e02..8d7a9df523c3d 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/ValuesSourceReaderOperator.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/ValuesSourceReaderOperator.java @@ -7,13 +7,17 @@ package org.elasticsearch.compute.lucene; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.SortedDocValues; +import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.compute.data.Block; import org.elasticsearch.compute.data.BlockFactory; +import org.elasticsearch.compute.data.BytesRefBlock; import org.elasticsearch.compute.data.DocBlock; import org.elasticsearch.compute.data.DocVector; import org.elasticsearch.compute.data.ElementType; @@ -23,75 +27,69 @@ import org.elasticsearch.compute.operator.AbstractPageMappingOperator; import org.elasticsearch.compute.operator.DriverContext; import org.elasticsearch.compute.operator.Operator; -import org.elasticsearch.index.mapper.BlockDocValuesReader; +import org.elasticsearch.core.Releasable; +import org.elasticsearch.core.Releasables; +import org.elasticsearch.index.fieldvisitor.StoredFieldLoader; import org.elasticsearch.index.mapper.BlockLoader; -import org.elasticsearch.search.aggregations.support.ValuesSource; +import org.elasticsearch.index.mapper.BlockLoaderStoredFieldsFromLeafLoader; +import org.elasticsearch.search.fetch.StoredFieldsSpec; import org.elasticsearch.xcontent.XContentBuilder; import java.io.IOException; import java.io.UncheckedIOException; +import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.TreeMap; +import java.util.stream.Collectors; /** * Operator that extracts doc_values from a Lucene index out of pages that have been produced by {@link LuceneSourceOperator} - * and outputs them to a new column. The operator leverages the {@link ValuesSource} infrastructure for extracting - * field values. This allows for a more uniform way of extracting data compared to deciding the correct doc_values - * loader for different field types. + * and outputs them to a new column. */ public class ValuesSourceReaderOperator extends AbstractPageMappingOperator { /** - * Creates a new extractor that uses ValuesSources load data - * @param sources the value source, type and index readers to use for extraction + * Creates a factory for {@link ValuesSourceReaderOperator}. + * @param fields fields to load * @param docChannel the channel containing the shard, leaf/segment and doc id - * @param field the lucene field being loaded */ - public record ValuesSourceReaderOperatorFactory(List sources, int docChannel, String field) - implements - OperatorFactory { + public record Factory(List fields, List readers, int docChannel) implements OperatorFactory { @Override public Operator get(DriverContext driverContext) { - return new ValuesSourceReaderOperator(driverContext.blockFactory(), sources, docChannel, field); + return new ValuesSourceReaderOperator(driverContext.blockFactory(), fields, readers, docChannel); } @Override public String describe() { - return "ValuesSourceReaderOperator[field = " + field + "]"; + return "ValuesSourceReaderOperator[field = " + fields.stream().map(f -> f.name).collect(Collectors.joining(", ")) + "]"; } } - /** - * A list, one entry per shard, of factories for {@link BlockDocValuesReader}s - * which perform the actual reading. - */ - private final List factories; + private final List fields; + private final List readers; private final int docChannel; - private final String field; private final ComputeBlockLoaderFactory blockFactory; - private BlockDocValuesReader lastReader; - private int lastShard = -1; - private int lastSegment = -1; - private final Map readersBuilt = new TreeMap<>(); + /** + * Configuration for a field to load. + * + * {@code blockLoaders} is a list, one entry per shard, of + * {@link BlockLoader}s which load the actual blocks. + */ + public record FieldInfo(String name, List blockLoaders) {} + /** * Creates a new extractor - * @param factories builds {@link BlockDocValuesReader} + * @param fields fields to load * @param docChannel the channel containing the shard, leaf/segment and doc id - * @param field the lucene field being loaded */ - public ValuesSourceReaderOperator( - BlockFactory blockFactory, - List factories, - int docChannel, - String field - ) { - this.factories = factories; + public ValuesSourceReaderOperator(BlockFactory blockFactory, List fields, List readers, int docChannel) { + this.fields = fields.stream().map(f -> new FieldWork(f)).toList(); + this.readers = readers; this.docChannel = docChannel; - this.field = field; this.blockFactory = new ComputeBlockLoaderFactory(blockFactory); } @@ -99,21 +97,31 @@ public ValuesSourceReaderOperator( protected Page process(Page page) { DocVector docVector = page.getBlock(docChannel).asVector(); + Block[] blocks = new Block[fields.size()]; + boolean success = false; try { if (docVector.singleSegmentNonDecreasing()) { - return page.appendBlock(loadFromSingleLeaf(docVector)); + loadFromSingleLeaf(blocks, docVector); + } else { + loadFromManyLeaves(blocks, docVector); } - return page.appendBlock(loadFromManyLeaves(docVector)); + success = true; } catch (IOException e) { throw new UncheckedIOException(e); + } finally { + if (success == false) { + Releasables.closeExpectNoException(blocks); + } } + return page.appendBlocks(blocks); } - private Block loadFromSingleLeaf(DocVector docVector) throws IOException { - setupReader(docVector.shards().getInt(0), docVector.segments().getInt(0), docVector.docs().getInt(0)); - return ((Block) lastReader.readValues(blockFactory, new BlockLoader.Docs() { - private final IntVector docs = docVector.docs(); - + private void loadFromSingleLeaf(Block[] blocks, DocVector docVector) throws IOException { + int shard = docVector.shards().getInt(0); + int segment = docVector.segments().getInt(0); + int firstDoc = docVector.docs().getInt(0); + IntVector docs = docVector.docs(); + BlockLoader.Docs loaderDocs = new BlockLoader.Docs() { @Override public int count() { return docs.getPositionCount(); @@ -123,44 +131,209 @@ public int count() { public int get(int i) { return docs.getInt(i); } - })); + }; + StoredFieldsSpec storedFieldsSpec = StoredFieldsSpec.NO_REQUIREMENTS; + List rowStrideReaders = new ArrayList<>(fields.size()); + try { + for (int b = 0; b < fields.size(); b++) { + FieldWork field = fields.get(b); + BlockLoader.ColumnAtATimeReader columnAtATime = field.columnAtATime.reader(shard, segment, firstDoc); + if (columnAtATime != null) { + blocks[b] = (Block) columnAtATime.read(blockFactory, loaderDocs); + } else { + BlockLoader.RowStrideReader rowStride = field.rowStride.reader(shard, segment, firstDoc); + rowStrideReaders.add( + new RowStrideReaderWork( + rowStride, + (Block.Builder) field.info.blockLoaders.get(shard).builder(blockFactory, docs.getPositionCount()), + b + ) + ); + storedFieldsSpec = storedFieldsSpec.merge(field.info.blockLoaders.get(shard).rowStrideStoredFieldSpec()); + } + } + + if (rowStrideReaders.isEmpty()) { + return; + } + if (storedFieldsSpec.equals(StoredFieldsSpec.NO_REQUIREMENTS)) { + throw new IllegalStateException( + "found row stride readers [" + rowStrideReaders + "] without stored fields [" + storedFieldsSpec + "]" + ); + } + BlockLoaderStoredFieldsFromLeafLoader storedFields = new BlockLoaderStoredFieldsFromLeafLoader( + // TODO enable the optimization by passing non-null to docs if correct + StoredFieldLoader.fromSpec(storedFieldsSpec).getLoader(ctx(shard, segment), null), + storedFieldsSpec.requiresSource() + ); + trackStoredFields(storedFieldsSpec); // TODO when optimization is enabled add it to tracking + for (int p = 0; p < docs.getPositionCount(); p++) { + int doc = docs.getInt(p); + if (storedFields != null) { + storedFields.advanceTo(doc); + } + for (int r = 0; r < rowStrideReaders.size(); r++) { + RowStrideReaderWork work = rowStrideReaders.get(r); + work.reader.read(doc, storedFields, work.builder); + } + } + for (int r = 0; r < rowStrideReaders.size(); r++) { + RowStrideReaderWork work = rowStrideReaders.get(r); + blocks[work.offset] = work.builder.build(); + } + } finally { + Releasables.close(rowStrideReaders); + } } - private Block loadFromManyLeaves(DocVector docVector) throws IOException { + private void loadFromManyLeaves(Block[] blocks, DocVector docVector) throws IOException { + IntVector shards = docVector.shards(); + IntVector segments = docVector.segments(); + IntVector docs = docVector.docs(); + Block.Builder[] builders = new Block.Builder[blocks.length]; int[] forwards = docVector.shardSegmentDocMapForwards(); - int doc = docVector.docs().getInt(forwards[0]); - setupReader(docVector.shards().getInt(forwards[0]), docVector.segments().getInt(forwards[0]), doc); - try (BlockLoader.Builder builder = lastReader.builder(blockFactory, forwards.length)) { - lastReader.readValuesFromSingleDoc(doc, builder); - for (int i = 1; i < forwards.length; i++) { - int shard = docVector.shards().getInt(forwards[i]); - int segment = docVector.segments().getInt(forwards[i]); - doc = docVector.docs().getInt(forwards[i]); - if (segment != lastSegment || shard != lastShard) { - setupReader(shard, segment, doc); + try { + for (int b = 0; b < fields.size(); b++) { + FieldWork field = fields.get(b); + builders[b] = builderFromFirstNonNull(field, docs.getPositionCount()); + } + int lastShard = -1; + int lastSegment = -1; + BlockLoaderStoredFieldsFromLeafLoader storedFields = null; + for (int i = 0; i < forwards.length; i++) { + int p = forwards[i]; + int shard = shards.getInt(p); + int segment = segments.getInt(p); + int doc = docs.getInt(p); + if (shard != lastShard || segment != lastSegment) { + lastShard = shard; + lastSegment = segment; + StoredFieldsSpec storedFieldsSpec = storedFieldsSpecForShard(shard); + storedFields = new BlockLoaderStoredFieldsFromLeafLoader( + StoredFieldLoader.fromSpec(storedFieldsSpec).getLoader(ctx(shard, segment), null), + storedFieldsSpec.requiresSource() + ); + if (false == storedFieldsSpec.equals(StoredFieldsSpec.NO_REQUIREMENTS)) { + trackStoredFields(storedFieldsSpec); + } + } + storedFields.advanceTo(doc); + for (int r = 0; r < blocks.length; r++) { + fields.get(r).rowStride.reader(shard, segment, doc).read(doc, storedFields, builders[r]); } - lastReader.readValuesFromSingleDoc(doc, builder); } - try (Block orig = ((Block.Builder) builder).build()) { - return orig.filter(docVector.shardSegmentDocMapBackwards()); + for (int r = 0; r < blocks.length; r++) { + try (Block orig = builders[r].build()) { + blocks[r] = orig.filter(docVector.shardSegmentDocMapBackwards()); + } } + } finally { + Releasables.closeExpectNoException(builders); } } - private void setupReader(int shard, int segment, int doc) throws IOException { - if (lastSegment == segment && lastShard == shard && BlockDocValuesReader.canReuse(lastReader, doc)) { - return; + private void trackStoredFields(StoredFieldsSpec spec) { + readersBuilt.merge( + "stored_fields[" + "requires_source:" + spec.requiresSource() + ", fields:" + spec.requiredStoredFields().size() + "]", + 1, + (prev, one) -> prev + one + ); + } + + /** + * Returns a builder from the first non - {@link BlockLoader#CONSTANT_NULLS} loader + * in the list. If they are all the null loader then returns a null builder. + */ + private Block.Builder builderFromFirstNonNull(FieldWork field, int positionCount) { + for (BlockLoader loader : field.info.blockLoaders) { + if (loader != BlockLoader.CONSTANT_NULLS) { + return (Block.Builder) loader.builder(blockFactory, positionCount); + } } + // All null, just let the first one build the null block loader. + return (Block.Builder) field.info.blockLoaders.get(0).builder(blockFactory, positionCount); + } - lastReader = factories.get(shard).build(segment); - lastShard = shard; - lastSegment = segment; - readersBuilt.compute(lastReader.toString(), (k, v) -> v == null ? 1 : v + 1); + private StoredFieldsSpec storedFieldsSpecForShard(int shard) { + StoredFieldsSpec storedFieldsSpec = StoredFieldsSpec.NO_REQUIREMENTS; + for (int b = 0; b < fields.size(); b++) { + FieldWork field = fields.get(b); + storedFieldsSpec = storedFieldsSpec.merge(field.info.blockLoaders.get(shard).rowStrideStoredFieldSpec()); + } + return storedFieldsSpec; + } + + private class FieldWork { + final FieldInfo info; + final GuardedReader columnAtATime = new GuardedReader<>() { + @Override + BlockLoader.ColumnAtATimeReader build(BlockLoader loader, LeafReaderContext ctx) throws IOException { + return loader.columnAtATimeReader(ctx); + } + + @Override + String type() { + return "column_at_a_time"; + } + }; + + final GuardedReader rowStride = new GuardedReader<>() { + @Override + BlockLoader.RowStrideReader build(BlockLoader loader, LeafReaderContext ctx) throws IOException { + return loader.rowStrideReader(ctx); + } + + @Override + String type() { + return "row_stride"; + } + }; + + FieldWork(FieldInfo info) { + this.info = info; + } + + private abstract class GuardedReader { + private int lastShard = -1; + private int lastSegment = -1; + V lastReader; + + V reader(int shard, int segment, int startingDocId) throws IOException { + if (lastShard == shard && lastSegment == segment) { + if (lastReader == null) { + return null; + } + if (lastReader.canReuse(startingDocId)) { + return lastReader; + } + } + lastShard = shard; + lastSegment = segment; + lastReader = build(info.blockLoaders.get(shard), ctx(shard, segment)); + readersBuilt.merge(info.name + ":" + type() + ":" + lastReader, 1, (prev, one) -> prev + one); + return lastReader; + } + + abstract V build(BlockLoader loader, LeafReaderContext ctx) throws IOException; + + abstract String type(); + } + } + + private record RowStrideReaderWork(BlockLoader.RowStrideReader reader, Block.Builder builder, int offset) implements Releasable { + @Override + public void close() { + builder.close(); + } + } + + private LeafReaderContext ctx(int shard, int segment) { + return readers.get(shard).leaves().get(segment); } @Override public String toString() { - return "ValuesSourceReaderOperator[field = " + field + "]"; + return "ValuesSourceReaderOperator[field = " + fields.stream().map(f -> f.info.name).collect(Collectors.joining(", ")) + "]"; } @Override @@ -233,7 +406,7 @@ public String toString() { } } - private static class ComputeBlockLoaderFactory implements BlockLoader.BuilderFactory { + private static class ComputeBlockLoaderFactory implements BlockLoader.BlockFactory { private final BlockFactory factory; private ComputeBlockLoaderFactory(BlockFactory factory) { @@ -295,9 +468,21 @@ public BlockLoader.Builder nulls(int expectedCount) { return ElementType.NULL.newBlockBuilder(expectedCount, factory); } + @Override + public Block constantNulls(int size) { + return factory.newConstantNullBlock(size); + } + + @Override + public BytesRefBlock constantBytes(BytesRef value, int size) { + return factory.newConstantBytesRefBlockWith(value, size); + } + @Override public BlockLoader.SingletonOrdinalsBuilder singletonOrdinalsBuilder(SortedDocValues ordinals, int count) { return new SingletonOrdinalsBuilder(factory, ordinals, count); } } + + // TODO tests that mix source loaded fields and doc values in the same block } diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/OrdinalsGroupingOperator.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/OrdinalsGroupingOperator.java index 07494f97cfd6d..2e1cbf9a1135d 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/OrdinalsGroupingOperator.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/OrdinalsGroupingOperator.java @@ -7,6 +7,7 @@ package org.elasticsearch.compute.operator; +import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.SortedSetDocValues; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.BytesRefBuilder; @@ -32,7 +33,7 @@ import org.elasticsearch.compute.operator.HashAggregationOperator.GroupSpec; import org.elasticsearch.core.Releasable; import org.elasticsearch.core.Releasables; -import org.elasticsearch.index.mapper.BlockDocValuesReader; +import org.elasticsearch.index.mapper.BlockLoader; import java.io.IOException; import java.io.UncheckedIOException; @@ -52,7 +53,8 @@ */ public class OrdinalsGroupingOperator implements Operator { public record OrdinalsGroupingOperatorFactory( - List readerFactories, + List blockLoaders, + List readers, ElementType groupingElementType, int docChannel, String groupingField, @@ -64,7 +66,8 @@ public record OrdinalsGroupingOperatorFactory( @Override public Operator get(DriverContext driverContext) { return new OrdinalsGroupingOperator( - readerFactories, + blockLoaders, + readers, groupingElementType, docChannel, groupingField, @@ -81,7 +84,8 @@ public String describe() { } } - private final List readerFactories; + private final List blockLoaders; + private final List readers; private final int docChannel; private final String groupingField; @@ -99,7 +103,8 @@ public String describe() { private ValuesAggregator valuesAggregator; public OrdinalsGroupingOperator( - List readerFactories, + List blockLoaders, + List readers, ElementType groupingElementType, int docChannel, String groupingField, @@ -109,7 +114,8 @@ public OrdinalsGroupingOperator( DriverContext driverContext ) { Objects.requireNonNull(aggregatorFactories); - this.readerFactories = readerFactories; + this.blockLoaders = blockLoaders; + this.readers = readers; this.groupingElementType = groupingElementType; this.docChannel = docChannel; this.groupingField = groupingField; @@ -131,10 +137,10 @@ public void addInput(Page page) { requireNonNull(page, "page is null"); DocVector docVector = page.getBlock(docChannel).asVector(); final int shardIndex = docVector.shards().getInt(0); - final var readerFactory = readerFactories.get(shardIndex); + final var blockLoader = blockLoaders.get(shardIndex); boolean pagePassed = false; try { - if (docVector.singleSegmentNonDecreasing() && readerFactory.supportsOrdinals()) { + if (docVector.singleSegmentNonDecreasing() && blockLoader.supportsOrdinals()) { final IntVector segmentIndexVector = docVector.segments(); assert segmentIndexVector.isConstant(); final OrdinalSegmentAggregator ordinalAggregator = this.ordinalAggregators.computeIfAbsent( @@ -144,7 +150,7 @@ public void addInput(Page page) { return new OrdinalSegmentAggregator( driverContext.blockFactory(), this::createGroupingAggregators, - () -> readerFactory.ordinals(k.segmentIndex), + () -> blockLoader.ordinals(readers.get(k.shardIndex).leaves().get(k.segmentIndex)), bigArrays ); } catch (IOException e) { @@ -158,7 +164,8 @@ public void addInput(Page page) { if (valuesAggregator == null) { int channelIndex = page.getBlockCount(); // extractor will append a new block at the end valuesAggregator = new ValuesAggregator( - readerFactories, + blockLoaders, + readers, groupingElementType, docChannel, groupingField, @@ -458,7 +465,8 @@ private static class ValuesAggregator implements Releasable { private final HashAggregationOperator aggregator; ValuesAggregator( - List factories, + List blockLoaders, + List readers, ElementType groupingElementType, int docChannel, String groupingField, @@ -467,7 +475,12 @@ private static class ValuesAggregator implements Releasable { int maxPageSize, DriverContext driverContext ) { - this.extractor = new ValuesSourceReaderOperator(BlockFactory.getNonBreakingInstance(), factories, docChannel, groupingField); + this.extractor = new ValuesSourceReaderOperator( + BlockFactory.getNonBreakingInstance(), + List.of(new ValuesSourceReaderOperator.FieldInfo(groupingField, blockLoaders)), + readers, + docChannel + ); this.aggregator = new HashAggregationOperator( aggregatorFactories, () -> BlockHash.build(List.of(new GroupSpec(channelIndex, groupingElementType)), driverContext, maxPageSize, false), diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/OperatorTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/OperatorTests.java index b45f597553e1b..d9730d3f602c7 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/OperatorTests.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/OperatorTests.java @@ -45,7 +45,6 @@ import org.elasticsearch.compute.data.IntVector; import org.elasticsearch.compute.data.LongBlock; import org.elasticsearch.compute.data.Page; -import org.elasticsearch.compute.lucene.BlockReaderFactories; import org.elasticsearch.compute.lucene.DataPartitioning; import org.elasticsearch.compute.lucene.LuceneOperator; import org.elasticsearch.compute.lucene.LuceneSourceOperator; @@ -230,9 +229,8 @@ public String toString() { } }, new OrdinalsGroupingOperator( - List.of( - BlockReaderFactories.loaderToFactory(reader, new KeywordFieldMapper.KeywordFieldType("g").blockLoader(null)) - ), + List.of(new KeywordFieldMapper.KeywordFieldType("g").blockLoader(null)), + List.of(reader), ElementType.BYTES_REF, 0, gField, diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/ValuesSourceReaderOperatorTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/ValuesSourceReaderOperatorTests.java index 269a478560bac..76810dbf2e3bc 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/ValuesSourceReaderOperatorTests.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/ValuesSourceReaderOperatorTests.java @@ -9,9 +9,12 @@ import org.apache.lucene.document.Document; import org.apache.lucene.document.DoubleDocValuesField; +import org.apache.lucene.document.FieldType; import org.apache.lucene.document.NumericDocValuesField; import org.apache.lucene.document.SortedDocValuesField; import org.apache.lucene.document.SortedNumericDocValuesField; +import org.apache.lucene.document.StoredField; +import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexableField; import org.apache.lucene.index.NoMergePolicy; @@ -23,6 +26,8 @@ import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.NumericUtils; import org.elasticsearch.common.Randomness; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.lucene.Lucene; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.util.BigArrays; import org.elasticsearch.compute.data.Block; @@ -46,20 +51,41 @@ import org.elasticsearch.compute.operator.PageConsumerOperator; import org.elasticsearch.compute.operator.SourceOperator; import org.elasticsearch.core.IOUtils; +import org.elasticsearch.index.IndexMode; +import org.elasticsearch.index.IndexVersion; +import org.elasticsearch.index.mapper.BlockLoader; import org.elasticsearch.index.mapper.BooleanFieldMapper; +import org.elasticsearch.index.mapper.IdFieldMapper; import org.elasticsearch.index.mapper.KeywordFieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.NumberFieldMapper; +import org.elasticsearch.index.mapper.ProvidedIdFieldMapper; +import org.elasticsearch.index.mapper.SourceFieldMapper; +import org.elasticsearch.index.mapper.TextFieldMapper; +import org.elasticsearch.index.mapper.TextSearchInfo; +import org.elasticsearch.index.mapper.TsidExtractingIdFieldMapper; +import org.elasticsearch.search.lookup.SearchLookup; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.json.JsonXContent; +import org.hamcrest.Matcher; import org.junit.After; import java.io.IOException; import java.util.ArrayList; +import java.util.Collections; import java.util.List; +import java.util.Map; +import java.util.Set; import java.util.stream.IntStream; import static org.elasticsearch.compute.lucene.LuceneSourceOperatorTests.mockSearchContext; +import static org.elasticsearch.test.MapMatcher.assertMap; +import static org.elasticsearch.test.MapMatcher.matchesMap; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasSize; +import static org.hamcrest.Matchers.lessThanOrEqualTo; +import static org.hamcrest.Matchers.not; +import static org.hamcrest.Matchers.nullValue; /** * Tests for {@link ValuesSourceReaderOperator}. Turns off {@link HandleLimitFS} @@ -86,19 +112,48 @@ public void closeIndex() throws IOException { @Override protected Operator.OperatorFactory simple(BigArrays bigArrays) { - return factory(reader, new NumberFieldMapper.NumberFieldType("long", NumberFieldMapper.NumberType.LONG)); + if (reader == null) { + // Init a reader if one hasn't been built, so things don't blow up + try { + initIndex(100); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + return factory(reader, docValuesNumberField("long", NumberFieldMapper.NumberType.LONG)); } static Operator.OperatorFactory factory(IndexReader reader, MappedFieldType ft) { - return new ValuesSourceReaderOperator.ValuesSourceReaderOperatorFactory( - List.of(BlockReaderFactories.loaderToFactory(reader, ft.blockLoader(null))), - 0, - ft.name() + return factory(reader, ft.name(), ft.blockLoader(null)); + } + + static Operator.OperatorFactory factory(IndexReader reader, String name, BlockLoader loader) { + return new ValuesSourceReaderOperator.Factory( + List.of(new ValuesSourceReaderOperator.FieldInfo(name, List.of(loader))), + List.of(reader), + 0 ); } @Override protected SourceOperator simpleInput(BlockFactory blockFactory, int size) { + try { + initIndex(size); + } catch (IOException e) { + throw new RuntimeException(e); + } + var luceneFactory = new LuceneSourceOperator.Factory( + List.of(mockSearchContext(reader)), + ctx -> new MatchAllDocsQuery(), + randomFrom(DataPartitioning.values()), + randomIntBetween(1, 10), + randomPageSize(), + LuceneOperator.NO_LIMIT + ); + return luceneFactory.get(driverContext()); + } + + private void initIndex(int size) throws IOException { // The test wants more than one segment. We shoot for about 10. int commitEvery = Math.max(1, size / 10); try ( @@ -110,40 +165,68 @@ protected SourceOperator simpleInput(BlockFactory blockFactory, int size) { ) { for (int d = 0; d < size; d++) { List doc = new ArrayList<>(); + doc.add(IdFieldMapper.standardIdField("id")); doc.add(new SortedNumericDocValuesField("key", d)); + doc.add(new SortedNumericDocValuesField("int", d)); + doc.add(new SortedNumericDocValuesField("short", (short) d)); + doc.add(new SortedNumericDocValuesField("byte", (byte) d)); doc.add(new SortedNumericDocValuesField("long", d)); doc.add( new KeywordFieldMapper.KeywordField("kwd", new BytesRef(Integer.toString(d)), KeywordFieldMapper.Defaults.FIELD_TYPE) ); + doc.add(new StoredField("stored_kwd", new BytesRef(Integer.toString(d)))); + doc.add(new StoredField("stored_text", Integer.toString(d))); doc.add(new SortedNumericDocValuesField("bool", d % 2 == 0 ? 1 : 0)); doc.add(new SortedNumericDocValuesField("double", NumericUtils.doubleToSortableLong(d / 123_456d))); for (int v = 0; v <= d % 3; v++) { - doc.add( - new KeywordFieldMapper.KeywordField("mv_kwd", new BytesRef(PREFIX[v] + d), KeywordFieldMapper.Defaults.FIELD_TYPE) - ); doc.add(new SortedNumericDocValuesField("mv_bool", v % 2 == 0 ? 1 : 0)); - doc.add(new SortedNumericDocValuesField("mv_key", 1_000 * d + v)); + doc.add(new SortedNumericDocValuesField("mv_int", 1_000 * d + v)); + doc.add(new SortedNumericDocValuesField("mv_short", (short) (2_000 * d + v))); + doc.add(new SortedNumericDocValuesField("mv_byte", (byte) (3_000 * d + v))); doc.add(new SortedNumericDocValuesField("mv_long", -1_000 * d + v)); doc.add(new SortedNumericDocValuesField("mv_double", NumericUtils.doubleToSortableLong(d / 123_456d + v))); + doc.add( + new KeywordFieldMapper.KeywordField("mv_kwd", new BytesRef(PREFIX[v] + d), KeywordFieldMapper.Defaults.FIELD_TYPE) + ); + doc.add(new StoredField("mv_stored_kwd", new BytesRef(PREFIX[v] + d))); + doc.add(new StoredField("mv_stored_text", PREFIX[v] + d)); + } + XContentBuilder source = JsonXContent.contentBuilder(); + source.startObject(); + source.field("source_kwd", Integer.toString(d)); + source.startArray("mv_source_kwd"); + for (int v = 0; v <= d % 3; v++) { + source.value(PREFIX[v] + d); + } + source.endArray(); + source.field("source_text", Integer.toString(d)); + source.startArray("mv_source_text"); + for (int v = 0; v <= d % 3; v++) { + source.value(PREFIX[v] + d); } + source.endArray(); + source.field("source_long", (long) d); + source.startArray("mv_source_long"); + for (int v = 0; v <= d % 3; v++) { + source.value((long) (-1_000 * d + v)); + } + source.endArray(); + source.field("source_int", d); + source.startArray("mv_source_int"); + for (int v = 0; v <= d % 3; v++) { + source.value(1_000 * d + v); + } + source.endArray(); + + source.endObject(); + doc.add(new StoredField(SourceFieldMapper.NAME, BytesReference.bytes(source).toBytesRef())); writer.addDocument(doc); if (d % commitEvery == 0) { writer.commit(); } } reader = writer.getReader(); - } catch (IOException e) { - throw new RuntimeException(e); } - var luceneFactory = new LuceneSourceOperator.Factory( - List.of(mockSearchContext(reader)), - ctx -> new MatchAllDocsQuery(), - randomFrom(DataPartitioning.values()), - randomIntBetween(1, 10), - randomPageSize(), - LuceneOperator.NO_LIMIT - ); - return luceneFactory.get(driverContext()); } @Override @@ -184,7 +267,8 @@ public void testLoadAll() { DriverContext driverContext = driverContext(); loadSimpleAndAssert( driverContext, - CannedSourceOperator.collectPages(simpleInput(driverContext.blockFactory(), between(100, 5000))) + CannedSourceOperator.collectPages(simpleInput(driverContext.blockFactory(), between(100, 5000))), + Block.MvOrdering.DEDUPLICATED_AND_SORTED_ASCENDING ); } @@ -196,13 +280,18 @@ public void testLoadAllInOnePage() { CannedSourceOperator.mergePages( CannedSourceOperator.collectPages(simpleInput(driverContext.blockFactory(), between(100, 5000))) ) - ) + ), + Block.MvOrdering.UNORDERED ); } public void testEmpty() { DriverContext driverContext = driverContext(); - loadSimpleAndAssert(driverContext, CannedSourceOperator.collectPages(simpleInput(driverContext.blockFactory(), 0))); + loadSimpleAndAssert( + driverContext, + CannedSourceOperator.collectPages(simpleInput(driverContext.blockFactory(), 0)), + Block.MvOrdering.UNORDERED + ); } public void testLoadAllInOnePageShuffled() { @@ -219,99 +308,681 @@ public void testLoadAllInOnePageShuffled() { shuffledBlocks[b] = source.getBlock(b).filter(shuffleArray); } source = new Page(shuffledBlocks); - loadSimpleAndAssert(driverContext, List.of(source)); - } - - private void loadSimpleAndAssert(DriverContext driverContext, List input) { - List operators = List.of( - factory(reader, new NumberFieldMapper.NumberFieldType("key", NumberFieldMapper.NumberType.INTEGER)).get(driverContext), - factory(reader, new NumberFieldMapper.NumberFieldType("long", NumberFieldMapper.NumberType.LONG)).get(driverContext), - factory(reader, new KeywordFieldMapper.KeywordFieldType("kwd")).get(driverContext), - factory(reader, new KeywordFieldMapper.KeywordFieldType("mv_kwd")).get(driverContext), - factory(reader, new BooleanFieldMapper.BooleanFieldType("bool")).get(driverContext), - factory(reader, new BooleanFieldMapper.BooleanFieldType("mv_bool")).get(driverContext), - factory(reader, new NumberFieldMapper.NumberFieldType("mv_key", NumberFieldMapper.NumberType.INTEGER)).get(driverContext), - factory(reader, new NumberFieldMapper.NumberFieldType("mv_long", NumberFieldMapper.NumberType.LONG)).get(driverContext), - factory(reader, new NumberFieldMapper.NumberFieldType("double", NumberFieldMapper.NumberType.DOUBLE)).get(driverContext), - factory(reader, new NumberFieldMapper.NumberFieldType("mv_double", NumberFieldMapper.NumberType.DOUBLE)).get(driverContext) + loadSimpleAndAssert(driverContext, List.of(source), Block.MvOrdering.UNORDERED); + } + + private static ValuesSourceReaderOperator.FieldInfo fieldInfo(MappedFieldType ft) { + return new ValuesSourceReaderOperator.FieldInfo(ft.name(), List.of(ft.blockLoader(new MappedFieldType.BlockLoaderContext() { + @Override + public String indexName() { + return "test_index"; + } + + @Override + public SearchLookup lookup() { + throw new UnsupportedOperationException(); + } + + @Override + public Set sourcePaths(String name) { + return Set.of(name); + } + }))); + } + + private void loadSimpleAndAssert(DriverContext driverContext, List input, Block.MvOrdering docValuesMvOrdering) { + List cases = infoAndChecksForEachType(docValuesMvOrdering); + + List operators = new ArrayList<>(); + operators.add( + new ValuesSourceReaderOperator.Factory( + List.of(fieldInfo(docValuesNumberField("key", NumberFieldMapper.NumberType.INTEGER))), + List.of(reader), + 0 + ).get(driverContext) ); + List tests = new ArrayList<>(); + while (cases.isEmpty() == false) { + List b = randomNonEmptySubsetOf(cases); + cases.removeAll(b); + tests.addAll(b); + operators.add( + new ValuesSourceReaderOperator.Factory(b.stream().map(i -> i.info).toList(), List.of(reader), 0).get(driverContext) + ); + } List results = drive(operators, input.iterator(), driverContext); assertThat(results, hasSize(input.size())); - for (Page p : results) { - assertThat(p.getBlockCount(), equalTo(11)); - IntVector keys = p.getBlock(1).asVector(); - LongVector longs = p.getBlock(2).asVector(); - BytesRefVector keywords = p.getBlock(3).asVector(); - BytesRefBlock mvKeywords = p.getBlock(4); - BooleanVector bools = p.getBlock(5).asVector(); - BooleanBlock mvBools = p.getBlock(6); - IntBlock mvInts = p.getBlock(7); - LongBlock mvLongs = p.getBlock(8); - DoubleVector doubles = p.getBlock(9).asVector(); - DoubleBlock mvDoubles = p.getBlock(10); - - for (int i = 0; i < p.getPositionCount(); i++) { - int key = keys.getInt(i); - assertThat(longs.getLong(i), equalTo((long) key)); - assertThat(keywords.getBytesRef(i, new BytesRef()).utf8ToString(), equalTo(Integer.toString(key))); - - assertThat(mvKeywords.getValueCount(i), equalTo(key % 3 + 1)); - int offset = mvKeywords.getFirstValueIndex(i); - for (int v = 0; v <= key % 3; v++) { - assertThat(mvKeywords.getBytesRef(offset + v, new BytesRef()).utf8ToString(), equalTo(PREFIX[v] + key)); - } - if (key % 3 > 0) { - assertThat(mvKeywords.mvOrdering(), equalTo(Block.MvOrdering.DEDUPLICATED_AND_SORTED_ASCENDING)); + for (Page page : results) { + assertThat(page.getBlockCount(), equalTo(tests.size() + 2 /* one for doc and one for keys */)); + IntVector keys = page.getBlock(1).asVector(); + for (int p = 0; p < page.getPositionCount(); p++) { + int key = keys.getInt(p); + for (int i = 0; i < tests.size(); i++) { + try { + tests.get(i).checkResults.check(page.getBlock(2 + i), p, key); + } catch (AssertionError e) { + throw new AssertionError("error checking " + tests.get(i).info.name() + "[" + p + "]: " + e.getMessage(), e); + } } + } + } + for (Operator op : operators) { + assertThat(((ValuesSourceReaderOperator) op).status().pagesProcessed(), equalTo(input.size())); + } + assertDriverContext(driverContext); + } - assertThat(bools.getBoolean(i), equalTo(key % 2 == 0)); - assertThat(mvBools.getValueCount(i), equalTo(key % 3 + 1)); - offset = mvBools.getFirstValueIndex(i); - for (int v = 0; v <= key % 3; v++) { - assertThat(mvBools.getBoolean(offset + v), equalTo(BOOLEANS[key % 3][v])); - } - if (key % 3 > 0) { - assertThat(mvBools.mvOrdering(), equalTo(Block.MvOrdering.DEDUPLICATED_AND_SORTED_ASCENDING)); - } + interface CheckResults { + void check(Block block, int position, int key); + } - assertThat(mvInts.getValueCount(i), equalTo(key % 3 + 1)); - offset = mvInts.getFirstValueIndex(i); - for (int v = 0; v <= key % 3; v++) { - assertThat(mvInts.getInt(offset + v), equalTo(1_000 * key + v)); - } - if (key % 3 > 0) { - assertThat(mvInts.mvOrdering(), equalTo(Block.MvOrdering.DEDUPLICATED_AND_SORTED_ASCENDING)); - } + interface CheckReaders { + void check(boolean forcedRowByRow, int pageCount, int segmentCount, Map readersBuilt); + } - assertThat(mvLongs.getValueCount(i), equalTo(key % 3 + 1)); - offset = mvLongs.getFirstValueIndex(i); - for (int v = 0; v <= key % 3; v++) { - assertThat(mvLongs.getLong(offset + v), equalTo(-1_000L * key + v)); - } - if (key % 3 > 0) { - assertThat(mvLongs.mvOrdering(), equalTo(Block.MvOrdering.DEDUPLICATED_AND_SORTED_ASCENDING)); - } + record FieldCase(ValuesSourceReaderOperator.FieldInfo info, CheckResults checkResults, CheckReaders checkReaders) { + FieldCase(MappedFieldType ft, CheckResults checkResults, CheckReaders checkReaders) { + this(fieldInfo(ft), checkResults, checkReaders); + } + } + + /** + * Asserts that {@link ValuesSourceReaderOperator#status} claims that only + * the expected readers are built after loading singleton pages. + */ + // @Repeat(iterations = 100) + public void testLoadAllStatus() { + DriverContext driverContext = driverContext(); + testLoadAllStatus(false); + } + + /** + * Asserts that {@link ValuesSourceReaderOperator#status} claims that only + * the expected readers are built after loading non-singleton pages. + */ + // @Repeat(iterations = 100) + public void testLoadAllStatusAllInOnePage() { + testLoadAllStatus(true); + } + + private void testLoadAllStatus(boolean allInOnePage) { + DriverContext driverContext = driverContext(); + List input = CannedSourceOperator.collectPages(simpleInput(driverContext.blockFactory(), between(100, 5000))); + List cases = infoAndChecksForEachType(Block.MvOrdering.DEDUPLICATED_AND_SORTED_ASCENDING); + // Build one operator for each field, so we get a unique map to assert on + List operators = cases.stream() + .map(i -> new ValuesSourceReaderOperator.Factory(List.of(i.info), List.of(reader), 0).get(driverContext)) + .toList(); + if (allInOnePage) { + input = List.of(CannedSourceOperator.mergePages(input)); + } + drive(operators, input.iterator(), driverContext); + for (int i = 0; i < cases.size(); i++) { + ValuesSourceReaderOperator.Status status = (ValuesSourceReaderOperator.Status) operators.get(i).status(); + assertThat(status.pagesProcessed(), equalTo(input.size())); + FieldCase fc = cases.get(i); + fc.checkReaders.check(allInOnePage, input.size(), reader.leaves().size(), status.readersBuilt()); + } + } + + private List infoAndChecksForEachType(Block.MvOrdering docValuesMvOrdering) { + Checks checks = new Checks(docValuesMvOrdering); + List r = new ArrayList<>(); + r.add( + new FieldCase(docValuesNumberField("long", NumberFieldMapper.NumberType.LONG), checks::longs, StatusChecks::longsFromDocValues) + ); + r.add( + new FieldCase( + docValuesNumberField("mv_long", NumberFieldMapper.NumberType.LONG), + checks::mvLongsFromDocValues, + StatusChecks::mvLongsFromDocValues + ) + ); + r.add( + new FieldCase(sourceNumberField("source_long", NumberFieldMapper.NumberType.LONG), checks::longs, StatusChecks::longsFromSource) + ); + r.add( + new FieldCase( + sourceNumberField("mv_source_long", NumberFieldMapper.NumberType.LONG), + checks::mvLongsUnordered, + StatusChecks::mvLongsFromSource + ) + ); + r.add( + new FieldCase(docValuesNumberField("int", NumberFieldMapper.NumberType.INTEGER), checks::ints, StatusChecks::intsFromDocValues) + ); + r.add( + new FieldCase( + docValuesNumberField("mv_int", NumberFieldMapper.NumberType.INTEGER), + checks::mvIntsFromDocValues, + StatusChecks::mvIntsFromDocValues + ) + ); + r.add( + new FieldCase(sourceNumberField("source_int", NumberFieldMapper.NumberType.INTEGER), checks::ints, StatusChecks::intsFromSource) + ); + r.add( + new FieldCase( + sourceNumberField("mv_source_int", NumberFieldMapper.NumberType.INTEGER), + checks::mvIntsUnordered, + StatusChecks::mvIntsFromSource + ) + ); + r.add( + new FieldCase( + docValuesNumberField("short", NumberFieldMapper.NumberType.SHORT), + checks::shorts, + StatusChecks::shortsFromDocValues + ) + ); + r.add( + new FieldCase( + docValuesNumberField("mv_short", NumberFieldMapper.NumberType.SHORT), + checks::mvShorts, + StatusChecks::mvShortsFromDocValues + ) + ); + r.add( + new FieldCase(docValuesNumberField("byte", NumberFieldMapper.NumberType.BYTE), checks::bytes, StatusChecks::bytesFromDocValues) + ); + r.add( + new FieldCase( + docValuesNumberField("mv_byte", NumberFieldMapper.NumberType.BYTE), + checks::mvBytes, + StatusChecks::mvBytesFromDocValues + ) + ); + r.add( + new FieldCase( + docValuesNumberField("double", NumberFieldMapper.NumberType.DOUBLE), + checks::doubles, + StatusChecks::doublesFromDocValues + ) + ); + r.add( + new FieldCase( + docValuesNumberField("mv_double", NumberFieldMapper.NumberType.DOUBLE), + checks::mvDoubles, + StatusChecks::mvDoublesFromDocValues + ) + ); + r.add(new FieldCase(new BooleanFieldMapper.BooleanFieldType("bool"), checks::bools, StatusChecks::boolFromDocValues)); + r.add(new FieldCase(new BooleanFieldMapper.BooleanFieldType("mv_bool"), checks::mvBools, StatusChecks::mvBoolFromDocValues)); + r.add(new FieldCase(new KeywordFieldMapper.KeywordFieldType("kwd"), checks::strings, StatusChecks::keywordsFromDocValues)); + r.add( + new FieldCase( + new KeywordFieldMapper.KeywordFieldType("mv_kwd"), + checks::mvStringsFromDocValues, + StatusChecks::mvKeywordsFromDocValues + ) + ); + r.add(new FieldCase(storedKeywordField("stored_kwd"), checks::strings, StatusChecks::keywordsFromStored)); + r.add(new FieldCase(storedKeywordField("mv_stored_kwd"), checks::mvStringsUnordered, StatusChecks::mvKeywordsFromStored)); + r.add(new FieldCase(sourceKeywordField("source_kwd"), checks::strings, StatusChecks::keywordsFromSource)); + r.add(new FieldCase(sourceKeywordField("mv_source_kwd"), checks::mvStringsUnordered, StatusChecks::mvKeywordsFromSource)); + r.add(new FieldCase(new TextFieldMapper.TextFieldType("source_text", false), checks::strings, StatusChecks::textFromSource)); + r.add( + new FieldCase( + new TextFieldMapper.TextFieldType("mv_source_text", false), + checks::mvStringsUnordered, + StatusChecks::mvTextFromSource + ) + ); + r.add(new FieldCase(storedTextField("stored_text"), checks::strings, StatusChecks::textFromStored)); + r.add(new FieldCase(storedTextField("mv_stored_text"), checks::mvStringsUnordered, StatusChecks::mvTextFromStored)); + r.add( + new FieldCase( + textFieldWithDelegate("text_with_delegate", new KeywordFieldMapper.KeywordFieldType("kwd")), + checks::strings, + StatusChecks::textWithDelegate + ) + ); + r.add( + new FieldCase( + textFieldWithDelegate("mv_text_with_delegate", new KeywordFieldMapper.KeywordFieldType("mv_kwd")), + checks::mvStringsFromDocValues, + StatusChecks::mvTextWithDelegate + ) + ); + r.add(new FieldCase(new ProvidedIdFieldMapper(() -> false).fieldType(), checks::ids, StatusChecks::id)); + r.add(new FieldCase(TsidExtractingIdFieldMapper.INSTANCE.fieldType(), checks::ids, StatusChecks::id)); + r.add( + new FieldCase( + new ValuesSourceReaderOperator.FieldInfo("constant_bytes", List.of(BlockLoader.constantBytes(new BytesRef("foo")))), + checks::constantBytes, + StatusChecks::constantBytes + ) + ); + r.add( + new FieldCase( + new ValuesSourceReaderOperator.FieldInfo("null", List.of(BlockLoader.CONSTANT_NULLS)), + checks::constantNulls, + StatusChecks::constantNulls + ) + ); + Collections.shuffle(r, random()); + return r; + } + + record Checks(Block.MvOrdering docValuesMvOrdering) { + void longs(Block block, int position, int key) { + LongVector longs = ((LongBlock) block).asVector(); + assertThat(longs.getLong(position), equalTo((long) key)); + } + + void ints(Block block, int position, int key) { + IntVector ints = ((IntBlock) block).asVector(); + assertThat(ints.getInt(position), equalTo(key)); + } + + void shorts(Block block, int position, int key) { + IntVector ints = ((IntBlock) block).asVector(); + assertThat(ints.getInt(position), equalTo((int) (short) key)); + } + + void bytes(Block block, int position, int key) { + IntVector ints = ((IntBlock) block).asVector(); + assertThat(ints.getInt(position), equalTo((int) (byte) key)); + } + + void doubles(Block block, int position, int key) { + DoubleVector doubles = ((DoubleBlock) block).asVector(); + assertThat(doubles.getDouble(position), equalTo(key / 123_456d)); + } + + void strings(Block block, int position, int key) { + BytesRefVector keywords = ((BytesRefBlock) block).asVector(); + assertThat(keywords.getBytesRef(position, new BytesRef()).utf8ToString(), equalTo(Integer.toString(key))); + } + + void bools(Block block, int position, int key) { + BooleanVector bools = ((BooleanBlock) block).asVector(); + assertThat(bools.getBoolean(position), equalTo(key % 2 == 0)); + } + + void ids(Block block, int position, int key) { + BytesRefVector ids = ((BytesRefBlock) block).asVector(); + assertThat(ids.getBytesRef(position, new BytesRef()).utf8ToString(), equalTo("id")); + } + + void constantBytes(Block block, int position, int key) { + BytesRefVector keywords = ((BytesRefBlock) block).asVector(); + assertThat(keywords.getBytesRef(position, new BytesRef()).utf8ToString(), equalTo("foo")); + } + + void constantNulls(Block block, int position, int key) { + assertTrue(block.areAllValuesNull()); + assertTrue(block.isNull(position)); + } + + void mvLongsFromDocValues(Block block, int position, int key) { + mvLongs(block, position, key, docValuesMvOrdering); + } + + void mvLongsUnordered(Block block, int position, int key) { + mvLongs(block, position, key, Block.MvOrdering.UNORDERED); + } + + private void mvLongs(Block block, int position, int key, Block.MvOrdering expectedMv) { + LongBlock longs = (LongBlock) block; + assertThat(longs.getValueCount(position), equalTo(key % 3 + 1)); + int offset = longs.getFirstValueIndex(position); + for (int v = 0; v <= key % 3; v++) { + assertThat(longs.getLong(offset + v), equalTo(-1_000L * key + v)); + } + if (key % 3 > 0) { + assertThat(longs.mvOrdering(), equalTo(expectedMv)); + } + } + + void mvIntsFromDocValues(Block block, int position, int key) { + mvInts(block, position, key, docValuesMvOrdering); + } + + void mvIntsUnordered(Block block, int position, int key) { + mvInts(block, position, key, Block.MvOrdering.UNORDERED); + } + + private void mvInts(Block block, int position, int key, Block.MvOrdering expectedMv) { + IntBlock ints = (IntBlock) block; + assertThat(ints.getValueCount(position), equalTo(key % 3 + 1)); + int offset = ints.getFirstValueIndex(position); + for (int v = 0; v <= key % 3; v++) { + assertThat(ints.getInt(offset + v), equalTo(1_000 * key + v)); + } + if (key % 3 > 0) { + assertThat(ints.mvOrdering(), equalTo(expectedMv)); + } + } + + void mvShorts(Block block, int position, int key) { + IntBlock ints = (IntBlock) block; + assertThat(ints.getValueCount(position), equalTo(key % 3 + 1)); + int offset = ints.getFirstValueIndex(position); + for (int v = 0; v <= key % 3; v++) { + assertThat(ints.getInt(offset + v), equalTo((int) (short) (2_000 * key + v))); + } + if (key % 3 > 0) { + assertThat(ints.mvOrdering(), equalTo(docValuesMvOrdering)); + } + } + + void mvBytes(Block block, int position, int key) { + IntBlock ints = (IntBlock) block; + assertThat(ints.getValueCount(position), equalTo(key % 3 + 1)); + int offset = ints.getFirstValueIndex(position); + for (int v = 0; v <= key % 3; v++) { + assertThat(ints.getInt(offset + v), equalTo((int) (byte) (3_000 * key + v))); + } + if (key % 3 > 0) { + assertThat(ints.mvOrdering(), equalTo(docValuesMvOrdering)); + } + } + + void mvDoubles(Block block, int position, int key) { + DoubleBlock doubles = (DoubleBlock) block; + int offset = doubles.getFirstValueIndex(position); + for (int v = 0; v <= key % 3; v++) { + assertThat(doubles.getDouble(offset + v), equalTo(key / 123_456d + v)); + } + if (key % 3 > 0) { + assertThat(doubles.mvOrdering(), equalTo(docValuesMvOrdering)); + } + } + + void mvStringsFromDocValues(Block block, int position, int key) { + mvStrings(block, position, key, docValuesMvOrdering); + } + + void mvStringsUnordered(Block block, int position, int key) { + mvStrings(block, position, key, Block.MvOrdering.UNORDERED); + } + + void mvStrings(Block block, int position, int key, Block.MvOrdering expectedMv) { + BytesRefBlock text = (BytesRefBlock) block; + assertThat(text.getValueCount(position), equalTo(key % 3 + 1)); + int offset = text.getFirstValueIndex(position); + for (int v = 0; v <= key % 3; v++) { + assertThat(text.getBytesRef(offset + v, new BytesRef()).utf8ToString(), equalTo(PREFIX[v] + key)); + } + if (key % 3 > 0) { + assertThat(text.mvOrdering(), equalTo(expectedMv)); + } + } + + void mvBools(Block block, int position, int key) { + BooleanBlock bools = (BooleanBlock) block; + assertThat(bools.getValueCount(position), equalTo(key % 3 + 1)); + int offset = bools.getFirstValueIndex(position); + for (int v = 0; v <= key % 3; v++) { + assertThat(bools.getBoolean(offset + v), equalTo(BOOLEANS[key % 3][v])); + } + if (key % 3 > 0) { + assertThat(bools.mvOrdering(), equalTo(docValuesMvOrdering)); + } + } + } + + class StatusChecks { + static void longsFromDocValues(boolean forcedRowByRow, int pageCount, int segmentCount, Map readers) { + docValues("long", "Longs", forcedRowByRow, pageCount, segmentCount, readers); + } + + static void longsFromSource(boolean forcedRowByRow, int pageCount, int segmentCount, Map readers) { + source("source_long", "Longs", forcedRowByRow, pageCount, segmentCount, readers); + } + + static void intsFromDocValues(boolean forcedRowByRow, int pageCount, int segmentCount, Map readers) { + docValues("int", "Ints", forcedRowByRow, pageCount, segmentCount, readers); + } + + static void intsFromSource(boolean forcedRowByRow, int pageCount, int segmentCount, Map readers) { + source("source_int", "Ints", forcedRowByRow, pageCount, segmentCount, readers); + } + + static void shortsFromDocValues(boolean forcedRowByRow, int pageCount, int segmentCount, Map readers) { + docValues("short", "Ints", forcedRowByRow, pageCount, segmentCount, readers); + } + + static void bytesFromDocValues(boolean forcedRowByRow, int pageCount, int segmentCount, Map readers) { + docValues("byte", "Ints", forcedRowByRow, pageCount, segmentCount, readers); + } + + static void doublesFromDocValues(boolean forcedRowByRow, int pageCount, int segmentCount, Map readers) { + docValues("double", "Doubles", forcedRowByRow, pageCount, segmentCount, readers); + } + + static void boolFromDocValues(boolean forcedRowByRow, int pageCount, int segmentCount, Map readers) { + docValues("bool", "Booleans", forcedRowByRow, pageCount, segmentCount, readers); + } + + static void keywordsFromDocValues(boolean forcedRowByRow, int pageCount, int segmentCount, Map readers) { + docValues("kwd", "Ordinals", forcedRowByRow, pageCount, segmentCount, readers); + } + + static void keywordsFromStored(boolean forcedRowByRow, int pageCount, int segmentCount, Map readers) { + stored("stored_kwd", "Bytes", forcedRowByRow, pageCount, segmentCount, readers); + } + + static void keywordsFromSource(boolean forcedRowByRow, int pageCount, int segmentCount, Map readers) { + source("source_kwd", "Bytes", forcedRowByRow, pageCount, segmentCount, readers); + } + + static void textFromSource(boolean forcedRowByRow, int pageCount, int segmentCount, Map readers) { + source("source_text", "Bytes", forcedRowByRow, pageCount, segmentCount, readers); + } + + static void textFromStored(boolean forcedRowByRow, int pageCount, int segmentCount, Map readers) { + stored("stored_text", "Bytes", forcedRowByRow, pageCount, segmentCount, readers); + } + + static void mvLongsFromDocValues(boolean forcedRowByRow, int pageCount, int segmentCount, Map readers) { + mvDocValues("mv_long", "Longs", forcedRowByRow, pageCount, segmentCount, readers); + } + + static void mvLongsFromSource(boolean forcedRowByRow, int pageCount, int segmentCount, Map readers) { + source("mv_source_long", "Longs", forcedRowByRow, pageCount, segmentCount, readers); + } + + static void mvIntsFromDocValues(boolean forcedRowByRow, int pageCount, int segmentCount, Map readers) { + mvDocValues("mv_int", "Ints", forcedRowByRow, pageCount, segmentCount, readers); + } + + static void mvIntsFromSource(boolean forcedRowByRow, int pageCount, int segmentCount, Map readers) { + source("mv_source_int", "Ints", forcedRowByRow, pageCount, segmentCount, readers); + } + + static void mvShortsFromDocValues(boolean forcedRowByRow, int pageCount, int segmentCount, Map readers) { + mvDocValues("mv_short", "Ints", forcedRowByRow, pageCount, segmentCount, readers); + } + + static void mvBytesFromDocValues(boolean forcedRowByRow, int pageCount, int segmentCount, Map readers) { + mvDocValues("mv_byte", "Ints", forcedRowByRow, pageCount, segmentCount, readers); + } + + static void mvDoublesFromDocValues(boolean forcedRowByRow, int pageCount, int segmentCount, Map readers) { + mvDocValues("mv_double", "Doubles", forcedRowByRow, pageCount, segmentCount, readers); + } + + static void mvBoolFromDocValues(boolean forcedRowByRow, int pageCount, int segmentCount, Map readers) { + mvDocValues("mv_bool", "Booleans", forcedRowByRow, pageCount, segmentCount, readers); + } + + static void mvKeywordsFromDocValues(boolean forcedRowByRow, int pageCount, int segmentCount, Map readers) { + mvDocValues("mv_kwd", "Ordinals", forcedRowByRow, pageCount, segmentCount, readers); + } + + static void mvKeywordsFromStored(boolean forcedRowByRow, int pageCount, int segmentCount, Map readers) { + stored("mv_stored_kwd", "Bytes", forcedRowByRow, pageCount, segmentCount, readers); + } + + static void mvKeywordsFromSource(boolean forcedRowByRow, int pageCount, int segmentCount, Map readers) { + source("mv_source_kwd", "Bytes", forcedRowByRow, pageCount, segmentCount, readers); + } + + static void mvTextFromStored(boolean forcedRowByRow, int pageCount, int segmentCount, Map readers) { + stored("mv_stored_text", "Bytes", forcedRowByRow, pageCount, segmentCount, readers); + } + + static void mvTextFromSource(boolean forcedRowByRow, int pageCount, int segmentCount, Map readers) { + source("mv_source_text", "Bytes", forcedRowByRow, pageCount, segmentCount, readers); + } + + static void textWithDelegate(boolean forcedRowByRow, int pageCount, int segmentCount, Map readers) { + if (forcedRowByRow) { + assertMap( + readers, + matchesMap().entry( + "text_with_delegate:row_stride:Delegating[to=kwd, impl=BlockDocValuesReader.SingletonOrdinals]", + segmentCount + ) + ); + } else { + assertMap( + readers, + matchesMap().entry( + "text_with_delegate:column_at_a_time:Delegating[to=kwd, impl=BlockDocValuesReader.SingletonOrdinals]", + lessThanOrEqualTo(pageCount) + ) + ); + } + } + + static void mvTextWithDelegate(boolean forcedRowByRow, int pageCount, int segmentCount, Map readers) { + if (forcedRowByRow) { + assertMap( + readers, + matchesMap().entry( + "mv_text_with_delegate:row_stride:Delegating[to=mv_kwd, impl=BlockDocValuesReader.SingletonOrdinals]", + lessThanOrEqualTo(segmentCount) + ) + .entry( + "mv_text_with_delegate:row_stride:Delegating[to=mv_kwd, impl=BlockDocValuesReader.Ordinals]", + lessThanOrEqualTo(segmentCount) + ) + ); + } else { + assertMap( + readers, + matchesMap().entry( + "mv_text_with_delegate:column_at_a_time:Delegating[to=mv_kwd, impl=BlockDocValuesReader.SingletonOrdinals]", + lessThanOrEqualTo(pageCount) + ) + .entry( + "mv_text_with_delegate:column_at_a_time:Delegating[to=mv_kwd, impl=BlockDocValuesReader.Ordinals]", + lessThanOrEqualTo(pageCount) + ) + ); + } + } + + private static void docValues( + String name, + String type, + boolean forcedRowByRow, + int pageCount, + int segmentCount, + Map readers + ) { + if (forcedRowByRow) { + assertMap( + readers, + matchesMap().entry(name + ":row_stride:BlockDocValuesReader.Singleton" + type, lessThanOrEqualTo(segmentCount)) + ); + } else { + assertMap( + readers, + matchesMap().entry(name + ":column_at_a_time:BlockDocValuesReader.Singleton" + type, lessThanOrEqualTo(pageCount)) + ); + } + } - assertThat(doubles.getDouble(i), equalTo(key / 123_456d)); - offset = mvDoubles.getFirstValueIndex(i); - for (int v = 0; v <= key % 3; v++) { - assertThat(mvDoubles.getDouble(offset + v), equalTo(key / 123_456d + v)); + private static void mvDocValues( + String name, + String type, + boolean forcedRowByRow, + int pageCount, + int segmentCount, + Map readers + ) { + if (forcedRowByRow) { + Integer singletons = (Integer) readers.remove(name + ":row_stride:BlockDocValuesReader.Singleton" + type); + if (singletons != null) { + segmentCount -= singletons; } - if (key % 3 > 0) { - assertThat(mvDoubles.mvOrdering(), equalTo(Block.MvOrdering.DEDUPLICATED_AND_SORTED_ASCENDING)); + assertMap(readers, matchesMap().entry(name + ":row_stride:BlockDocValuesReader." + type, segmentCount)); + } else { + Integer singletons = (Integer) readers.remove(name + ":column_at_a_time:BlockDocValuesReader.Singleton" + type); + if (singletons != null) { + pageCount -= singletons; } + assertMap( + readers, + matchesMap().entry(name + ":column_at_a_time:BlockDocValuesReader." + type, lessThanOrEqualTo(pageCount)) + ); } } - for (Operator op : operators) { - assertThat(((ValuesSourceReaderOperator) op).status().pagesProcessed(), equalTo(input.size())); + + static void id(boolean forcedRowByRow, int pageCount, int segmentCount, Map readers) { + stored("_id", "Id", forcedRowByRow, pageCount, segmentCount, readers); + } + + private static void source(String name, String type, boolean forcedRowByRow, int pageCount, int segmentCount, Map readers) { + Matcher count; + if (forcedRowByRow) { + count = equalTo(segmentCount); + } else { + count = lessThanOrEqualTo(pageCount); + Integer columnAttempts = (Integer) readers.remove(name + ":column_at_a_time:null"); + assertThat(columnAttempts, not(nullValue())); + } + assertMap( + readers, + matchesMap().entry(name + ":row_stride:BlockSourceReader." + type, count) + .entry("stored_fields[requires_source:true, fields:0]", count) + ); + } + + private static void stored(String name, String type, boolean forcedRowByRow, int pageCount, int segmentCount, Map readers) { + Matcher count; + if (forcedRowByRow) { + count = equalTo(segmentCount); + } else { + count = lessThanOrEqualTo(pageCount); + Integer columnAttempts = (Integer) readers.remove(name + ":column_at_a_time:null"); + assertThat(columnAttempts, not(nullValue())); + } + assertMap( + readers, + matchesMap().entry(name + ":row_stride:BlockStoredFieldsReader." + type, count) + .entry("stored_fields[requires_source:false, fields:1]", count) + ); + } + + static void constantBytes(boolean forcedRowByRow, int pageCount, int segmentCount, Map readers) { + if (forcedRowByRow) { + assertMap(readers, matchesMap().entry("constant_bytes:row_stride:constant[[66 6f 6f]]", segmentCount)); + } else { + assertMap( + readers, + matchesMap().entry("constant_bytes:column_at_a_time:constant[[66 6f 6f]]", lessThanOrEqualTo(pageCount)) + ); + } + } + + static void constantNulls(boolean forcedRowByRow, int pageCount, int segmentCount, Map readers) { + if (forcedRowByRow) { + assertMap(readers, matchesMap().entry("null:row_stride:constant_nulls", segmentCount)); + } else { + assertMap(readers, matchesMap().entry("null:column_at_a_time:constant_nulls", lessThanOrEqualTo(pageCount))); + } } - assertDriverContext(driverContext); } - public void testValuesSourceReaderOperatorWithNulls() throws IOException { - MappedFieldType intFt = new NumberFieldMapper.NumberFieldType("i", NumberFieldMapper.NumberType.INTEGER); - MappedFieldType longFt = new NumberFieldMapper.NumberFieldType("j", NumberFieldMapper.NumberType.LONG); - MappedFieldType doubleFt = new NumberFieldMapper.NumberFieldType("d", NumberFieldMapper.NumberType.DOUBLE); + public void testWithNulls() throws IOException { + MappedFieldType intFt = docValuesNumberField("i", NumberFieldMapper.NumberType.INTEGER); + MappedFieldType longFt = docValuesNumberField("j", NumberFieldMapper.NumberType.LONG); + MappedFieldType doubleFt = docValuesNumberField("d", NumberFieldMapper.NumberType.DOUBLE); MappedFieldType kwFt = new KeywordFieldMapper.KeywordFieldType("kw"); NumericDocValuesField intField = new NumericDocValuesField(intFt.name(), 0); @@ -384,4 +1055,85 @@ public void testValuesSourceReaderOperatorWithNulls() throws IOException { } assertDriverContext(driverContext); } + + private NumberFieldMapper.NumberFieldType docValuesNumberField(String name, NumberFieldMapper.NumberType type) { + return new NumberFieldMapper.NumberFieldType(name, type); + } + + private NumberFieldMapper.NumberFieldType sourceNumberField(String name, NumberFieldMapper.NumberType type) { + return new NumberFieldMapper.NumberFieldType( + name, + type, + randomBoolean(), + false, + false, + randomBoolean(), + null, + Map.of(), + null, + false, + null, + randomFrom(IndexMode.values()) + ); + } + + private KeywordFieldMapper.KeywordFieldType storedKeywordField(String name) { + FieldType ft = new FieldType(KeywordFieldMapper.Defaults.FIELD_TYPE); + ft.setDocValuesType(DocValuesType.NONE); + ft.setStored(true); + ft.freeze(); + return new KeywordFieldMapper.KeywordFieldType( + name, + ft, + Lucene.KEYWORD_ANALYZER, + Lucene.KEYWORD_ANALYZER, + Lucene.KEYWORD_ANALYZER, + new KeywordFieldMapper.Builder(name, IndexVersion.current()).docValues(false), + true // TODO randomize - load from stored keyword fields if stored even in synthetic source + ); + } + + private KeywordFieldMapper.KeywordFieldType sourceKeywordField(String name) { + FieldType ft = new FieldType(KeywordFieldMapper.Defaults.FIELD_TYPE); + ft.setDocValuesType(DocValuesType.NONE); + ft.setStored(false); + ft.freeze(); + return new KeywordFieldMapper.KeywordFieldType( + name, + ft, + Lucene.KEYWORD_ANALYZER, + Lucene.KEYWORD_ANALYZER, + Lucene.KEYWORD_ANALYZER, + new KeywordFieldMapper.Builder(name, IndexVersion.current()).docValues(false), + false + ); + } + + private TextFieldMapper.TextFieldType storedTextField(String name) { + return new TextFieldMapper.TextFieldType( + name, + false, + true, + new TextSearchInfo(TextFieldMapper.Defaults.FIELD_TYPE, null, Lucene.STANDARD_ANALYZER, Lucene.STANDARD_ANALYZER), + true, // TODO randomize - if the field is stored we should load from the stored field even if there is source + null, + Map.of(), + false, + false + ); + } + + private TextFieldMapper.TextFieldType textFieldWithDelegate(String name, KeywordFieldMapper.KeywordFieldType delegate) { + return new TextFieldMapper.TextFieldType( + name, + false, + false, + new TextSearchInfo(TextFieldMapper.Defaults.FIELD_TYPE, null, Lucene.STANDARD_ANALYZER, Lucene.STANDARD_ANALYZER), + randomBoolean(), + delegate, + Map.of(), + false, + false + ); + } } diff --git a/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/20_aggs.yml b/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/20_aggs.yml index 1087bd5ce06eb..e94cb6ccd8e3c 100644 --- a/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/20_aggs.yml +++ b/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/20_aggs.yml @@ -22,91 +22,93 @@ setup: type: long color: type: keyword + text: + type: text - do: bulk: index: "test" refresh: true body: - { "index": { } } - - { "data": 1, "count": 40, "data_d": 1, "count_d": 40, "time": 1674835275187, "color": "red" } + - { "data": 1, "count": 40, "data_d": 1, "count_d": 40, "time": 1674835275187, "color": "red", "text": "rr red" } - { "index": { } } - - { "data": 2, "count": 42, "data_d": 2, "count_d": 42, "time": 1674835275188, "color": "blue" } + - { "data": 2, "count": 42, "data_d": 2, "count_d": 42, "time": 1674835275188, "color": "blue", "text": "bb blue" } - { "index": { } } - - { "data": 1, "count": 44, "data_d": 1, "count_d": 44, "time": 1674835275189, "color": "green" } + - { "data": 1, "count": 44, "data_d": 1, "count_d": 44, "time": 1674835275189, "color": "green", "text": "gg green" } - { "index": { } } - - { "data": 2, "count": 46, "data_d": 2, "count_d": 46, "time": 1674835275190, "color": "red" } + - { "data": 2, "count": 46, "data_d": 2, "count_d": 46, "time": 1674835275190, "color": "red", "text": "rr red" } - { "index": { } } - - { "data": 1, "count": 40, "data_d": 1, "count_d": 40, "time": 1674835275191, "color": "red" } + - { "data": 1, "count": 40, "data_d": 1, "count_d": 40, "time": 1674835275191, "color": "red", "text": "rr red" } - { "index": { } } - - { "data": 2, "count": 42, "data_d": 2, "count_d": 42, "time": 1674835275192, "color": "blue" } + - { "data": 2, "count": 42, "data_d": 2, "count_d": 42, "time": 1674835275192, "color": "blue", "text": "bb blue" } - { "index": { } } - - { "data": 1, "count": 44, "data_d": 1, "count_d": 44, "time": 1674835275193, "color": "green" } + - { "data": 1, "count": 44, "data_d": 1, "count_d": 44, "time": 1674835275193, "color": "green", "text": "gg green" } - { "index": { } } - - { "data": 2, "count": 46, "data_d": 2, "count_d": 46, "time": 1674835275194, "color": "red" } + - { "data": 2, "count": 46, "data_d": 2, "count_d": 46, "time": 1674835275194, "color": "red", "text": "rr red" } - { "index": { } } - - { "data": 1, "count": 40, "data_d": 1, "count_d": 40, "time": 1674835275195, "color": "red" } + - { "data": 1, "count": 40, "data_d": 1, "count_d": 40, "time": 1674835275195, "color": "red", "text": "rr red" } - { "index": { } } - - { "data": 2, "count": 42, "data_d": 2, "count_d": 42, "time": 1674835275196, "color": "blue" } + - { "data": 2, "count": 42, "data_d": 2, "count_d": 42, "time": 1674835275196, "color": "blue", "text": "bb blue" } - { "index": { } } - - { "data": 1, "count": 44, "data_d": 1, "count_d": 44, "time": 1674835275197, "color": "green" } + - { "data": 1, "count": 44, "data_d": 1, "count_d": 44, "time": 1674835275197, "color": "green", "text": "gg green" } - { "index": { } } - - { "data": 2, "count": 46, "data_d": 2, "count_d": 46, "time": 1674835275198, "color": "red" } + - { "data": 2, "count": 46, "data_d": 2, "count_d": 46, "time": 1674835275198, "color": "red", "text": "rr red" } - { "index": { } } - - { "data": 1, "count": 40, "data_d": 1, "count_d": 40, "time": 1674835275199, "color": "red" } + - { "data": 1, "count": 40, "data_d": 1, "count_d": 40, "time": 1674835275199, "color": "red", "text": "rr red" } - { "index": { } } - - { "data": 2, "count": 42, "data_d": 2, "count_d": 42, "time": 1674835275200, "color": "blue" } + - { "data": 2, "count": 42, "data_d": 2, "count_d": 42, "time": 1674835275200, "color": "blue", "text": "bb blue" } - { "index": { } } - - { "data": 1, "count": 44, "data_d": 1, "count_d": 44, "time": 1674835275201, "color": "green" } + - { "data": 1, "count": 44, "data_d": 1, "count_d": 44, "time": 1674835275201, "color": "green", "text": "gg green" } - { "index": { } } - - { "data": 2, "count": 46, "data_d": 2, "count_d": 46, "time": 1674835275202, "color": "red" } + - { "data": 2, "count": 46, "data_d": 2, "count_d": 46, "time": 1674835275202, "color": "red", "text": "rr red" } - { "index": { } } - - { "data": 1, "count": 40, "data_d": 1, "count_d": 40, "time": 1674835275203, "color": "red" } + - { "data": 1, "count": 40, "data_d": 1, "count_d": 40, "time": 1674835275203, "color": "red", "text": "rr red" } - { "index": { } } - - { "data": 2, "count": 42, "data_d": 2, "count_d": 42, "time": 1674835275204, "color": "blue" } + - { "data": 2, "count": 42, "data_d": 2, "count_d": 42, "time": 1674835275204, "color": "blue", "text": "bb blue" } - { "index": { } } - - { "data": 1, "count": 44, "data_d": 1, "count_d": 44, "time": 1674835275205, "color": "green" } + - { "data": 1, "count": 44, "data_d": 1, "count_d": 44, "time": 1674835275205, "color": "green", "text": "gg green" } - { "index": { } } - - { "data": 2, "count": 46, "data_d": 2, "count_d": 46, "time": 1674835275206, "color": "red" } + - { "data": 2, "count": 46, "data_d": 2, "count_d": 46, "time": 1674835275206, "color": "red", "text": "rr red" } - { "index": { } } - - { "data": 1, "count": 40, "data_d": 1, "count_d": 40, "time": 1674835275207, "color": "red" } + - { "data": 1, "count": 40, "data_d": 1, "count_d": 40, "time": 1674835275207, "color": "red", "text": "rr red" } - { "index": { } } - - { "data": 2, "count": 42, "data_d": 2, "count_d": 42, "time": 1674835275208, "color": "blue" } + - { "data": 2, "count": 42, "data_d": 2, "count_d": 42, "time": 1674835275208, "color": "blue", "text": "bb blue" } - { "index": { } } - - { "data": 1, "count": 44, "data_d": 1, "count_d": 44, "time": 1674835275209, "color": "green" } + - { "data": 1, "count": 44, "data_d": 1, "count_d": 44, "time": 1674835275209, "color": "green", "text": "gg green" } - { "index": { } } - - { "data": 2, "count": 46, "data_d": 2, "count_d": 46, "time": 1674835275210, "color": "red" } + - { "data": 2, "count": 46, "data_d": 2, "count_d": 46, "time": 1674835275210, "color": "red", "text": "rr red" } - { "index": { } } - - { "data": 1, "count": 40, "data_d": 1, "count_d": 40, "time": 1674835275211, "color": "red" } + - { "data": 1, "count": 40, "data_d": 1, "count_d": 40, "time": 1674835275211, "color": "red", "text": "rr red" } - { "index": { } } - - { "data": 2, "count": 42, "data_d": 2, "count_d": 42, "time": 1674835275212, "color": "blue" } + - { "data": 2, "count": 42, "data_d": 2, "count_d": 42, "time": 1674835275212, "color": "blue", "text": "bb blue" } - { "index": { } } - - { "data": 1, "count": 44, "data_d": 1, "count_d": 44, "time": 1674835275213, "color": "green" } + - { "data": 1, "count": 44, "data_d": 1, "count_d": 44, "time": 1674835275213, "color": "green", "text": "gg green" } - { "index": { } } - - { "data": 2, "count": 46, "data_d": 2, "count_d": 46, "time": 1674835275214, "color": "red" } + - { "data": 2, "count": 46, "data_d": 2, "count_d": 46, "time": 1674835275214, "color": "red", "text": "rr red" } - { "index": { } } - - { "data": 1, "count": 40, "data_d": 1, "count_d": 40, "time": 1674835275215, "color": "red" } + - { "data": 1, "count": 40, "data_d": 1, "count_d": 40, "time": 1674835275215, "color": "red", "text": "rr red" } - { "index": { } } - - { "data": 2, "count": 42, "data_d": 2, "count_d": 42, "time": 1674835275216, "color": "blue" } + - { "data": 2, "count": 42, "data_d": 2, "count_d": 42, "time": 1674835275216, "color": "blue", "text": "bb blue" } - { "index": { } } - - { "data": 1, "count": 44, "data_d": 1, "count_d": 44, "time": 1674835275217, "color": "green" } + - { "data": 1, "count": 44, "data_d": 1, "count_d": 44, "time": 1674835275217, "color": "green", "text": "gg green" } - { "index": { } } - - { "data": 2, "count": 46, "data_d": 2, "count_d": 46, "time": 1674835275218, "color": "red" } + - { "data": 2, "count": 46, "data_d": 2, "count_d": 46, "time": 1674835275218, "color": "red", "text": "rr red" } - { "index": { } } - - { "data": 1, "count": 40, "data_d": 1, "count_d": 40, "time": 1674835275219, "color": "red" } + - { "data": 1, "count": 40, "data_d": 1, "count_d": 40, "time": 1674835275219, "color": "red", "text": "rr red" } - { "index": { } } - - { "data": 2, "count": 42, "data_d": 2, "count_d": 42, "time": 1674835275220, "color": "blue" } + - { "data": 2, "count": 42, "data_d": 2, "count_d": 42, "time": 1674835275220, "color": "blue", "text": "bb blue" } - { "index": { } } - - { "data": 1, "count": 44, "data_d": 1, "count_d": 44, "time": 1674835275221, "color": "green" } + - { "data": 1, "count": 44, "data_d": 1, "count_d": 44, "time": 1674835275221, "color": "green", "text": "gg green" } - { "index": { } } - - { "data": 2, "count": 46, "data_d": 2, "count_d": 46, "time": 1674835275222, "color": "red" } + - { "data": 2, "count": 46, "data_d": 2, "count_d": 46, "time": 1674835275222, "color": "red", "text": "rr red" } - { "index": { } } - - { "data": 1, "count": 40, "data_d": 1, "count_d": 40, "time": 1674835275223, "color": "red" } + - { "data": 1, "count": 40, "data_d": 1, "count_d": 40, "time": 1674835275223, "color": "red", "text": "rr red" } - { "index": { } } - - { "data": 2, "count": 42, "data_d": 2, "count_d": 42, "time": 1674835275224, "color": "blue" } + - { "data": 2, "count": 42, "data_d": 2, "count_d": 42, "time": 1674835275224, "color": "blue", "text": "bb blue" } - { "index": { } } - - { "data": 1, "count": 44, "data_d": 1, "count_d": 44, "time": 1674835275225, "color": "green" } + - { "data": 1, "count": 44, "data_d": 1, "count_d": 44, "time": 1674835275225, "color": "green", "text": "gg green" } - { "index": { } } - - { "data": 2, "count": 46, "data_d": 2, "count_d": 46, "time": 1674835275226, "color": "red" } + - { "data": 2, "count": 46, "data_d": 2, "count_d": 46, "time": 1674835275226, "color": "red", "text": "rr red" } --- "Test From": @@ -127,8 +129,10 @@ setup: - match: {columns.3.type: "long"} - match: {columns.4.name: "data_d"} - match: {columns.4.type: "double"} - - match: {columns.5.name: "time"} - - match: {columns.5.type: "long"} + - match: {columns.5.name: "text"} + - match: {columns.5.type: "text"} + - match: {columns.6.name: "time"} + - match: {columns.6.type: "long"} - length: {values: 40} --- @@ -429,11 +433,11 @@ setup: body: query: 'from test | eval nullsum = count_d + null | sort nullsum | limit 1' - - length: {columns: 7} + - length: {columns: 8} - length: {values: 1} - - match: {columns.6.name: "nullsum"} - - match: {columns.6.type: "double"} - - match: {values.0.6: null} + - match: {columns.7.name: "nullsum"} + - match: {columns.7.type: "double"} + - match: {values.0.7: null} --- "Test Eval Row With Null": @@ -501,3 +505,19 @@ setup: - match: {values.0.2: null} - match: {values.0.3: null} +--- +grouping on text: + - do: + warnings: + - "No limit defined, adding default limit of [500]" + esql.query: + body: + query: 'FROM test | STATS med=median(count) BY text | SORT med' + columnar: true + + - match: {columns.0.name: "med"} + - match: {columns.0.type: "double"} + - match: {columns.1.name: "text"} + - match: {columns.1.type: "text"} + - match: {values.0: [42.0, 43.0, 44.0]} + - match: {values.1: ["bb blue", "rr red", "gg green"]} diff --git a/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/50_index_patterns.yml b/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/50_index_patterns.yml index 5fceeee2f6e57..2098b9ee60d1e 100644 --- a/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/50_index_patterns.yml +++ b/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/50_index_patterns.yml @@ -28,9 +28,9 @@ disjoint_mappings: index: test1 refresh: true body: - - { "index": {} } - - { "message1": "foo1"} - - { "index": {} } + - { "index": { } } + - { "message1": "foo1" } + - { "index": { } } - { "message1": "foo2" } - do: @@ -38,9 +38,9 @@ disjoint_mappings: index: test2 refresh: true body: - - { "index": {} } + - { "index": { } } - { "message2": 1 } - - { "index": {} } + - { "index": { } } - { "message2": 2 } - do: @@ -315,9 +315,9 @@ same_name_different_type: index: test1 refresh: true body: - - { "index": {} } - - { "message": "foo1"} - - { "index": {} } + - { "index": { } } + - { "message": "foo1" } + - { "index": { } } - { "message": "foo2" } - do: @@ -325,9 +325,9 @@ same_name_different_type: index: test2 refresh: true body: - - { "index": {} } + - { "index": { } } - { "message": 1 } - - { "index": {} } + - { "index": { } } - { "message": 2 } - do: @@ -367,9 +367,9 @@ same_name_different_type_same_family: index: test1 refresh: true body: - - { "index": {} } - - { "message": "foo1"} - - { "index": {} } + - { "index": { } } + - { "message": "foo1" } + - { "index": { } } - { "message": "foo2" } - do: @@ -377,9 +377,9 @@ same_name_different_type_same_family: index: test2 refresh: true body: - - { "index": {} } + - { "index": { } } - { "message": "foo3" } - - { "index": {} } + - { "index": { } } - { "message": "foo4" } - do: diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlActionTaskIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlActionTaskIT.java index edaf9d91e9771..402ae2722d7ca 100644 --- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlActionTaskIT.java +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlActionTaskIT.java @@ -177,7 +177,10 @@ public void testTaskContents() throws Exception { } if (o.operator().equals("ValuesSourceReaderOperator[field = pause_me]")) { ValuesSourceReaderOperator.Status oStatus = (ValuesSourceReaderOperator.Status) o.status(); - assertMap(oStatus.readersBuilt(), matchesMap().entry("ScriptLongs", greaterThanOrEqualTo(1))); + assertMap( + oStatus.readersBuilt(), + matchesMap().entry("pause_me:column_at_a_time:ScriptLongs", greaterThanOrEqualTo(1)) + ); assertThat(oStatus.pagesProcessed(), greaterThanOrEqualTo(1)); valuesSourceReaders++; continue; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/EnrichLookupService.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/EnrichLookupService.java index 384563cb815a4..bad7dd00d6c18 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/EnrichLookupService.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/EnrichLookupService.java @@ -257,21 +257,28 @@ private void doLookup( }; List intermediateOperators = new ArrayList<>(extractFields.size() + 2); final ElementType[] mergingTypes = new ElementType[extractFields.size()]; - // extract-field operators + + // load the fields + List fields = new ArrayList<>(extractFields.size()); for (int i = 0; i < extractFields.size(); i++) { NamedExpression extractField = extractFields.get(i); final ElementType elementType = LocalExecutionPlanner.toElementType(extractField.dataType()); mergingTypes[i] = elementType; - var sources = BlockReaderFactories.factories( + var loaders = BlockReaderFactories.loaders( List.of(searchContext), extractField instanceof Alias a ? ((NamedExpression) a.child()).name() : extractField.name(), EsqlDataTypes.isUnsupported(extractField.dataType()) ); - intermediateOperators.add(new ValuesSourceReaderOperator(blockFactory, sources, 0, extractField.name())); + fields.add(new ValuesSourceReaderOperator.FieldInfo(extractField.name(), loaders)); } + intermediateOperators.add( + new ValuesSourceReaderOperator(blockFactory, fields, List.of(searchContext.searcher().getIndexReader()), 0) + ); + // drop docs block intermediateOperators.add(droppingBlockOperator(extractFields.size() + 2, 0)); boolean singleLeaf = searchContext.searcher().getLeafContexts().size() == 1; + // merging field-values by position final int[] mergingChannels = IntStream.range(0, extractFields.size()).map(i -> i + 1).toArray(); intermediateOperators.add( diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsPhysicalOperationProviders.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsPhysicalOperationProviders.java index f73ab716cb534..1dddee5ed54ea 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsPhysicalOperationProviders.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsPhysicalOperationProviders.java @@ -7,6 +7,7 @@ package org.elasticsearch.xpack.esql.planner; +import org.apache.lucene.index.IndexReader; import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.Query; @@ -19,7 +20,7 @@ import org.elasticsearch.compute.lucene.ValuesSourceReaderOperator; import org.elasticsearch.compute.operator.Operator; import org.elasticsearch.compute.operator.OrdinalsGroupingOperator; -import org.elasticsearch.index.mapper.BlockDocValuesReader; +import org.elasticsearch.index.mapper.BlockLoader; import org.elasticsearch.index.mapper.NestedLookup; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryBuilders; @@ -75,16 +76,17 @@ public final PhysicalOperation fieldExtractPhysicalOperation(FieldExtractExec fi DataType dataType = attr.dataType(); String fieldName = attr.name(); - List factories = BlockReaderFactories.factories( - searchContexts, - fieldName, - EsqlDataTypes.isUnsupported(dataType) - ); + List loaders = BlockReaderFactories.loaders(searchContexts, fieldName, EsqlDataTypes.isUnsupported(dataType)); + List readers = searchContexts.stream().map(s -> s.searcher().getIndexReader()).toList(); int docChannel = previousLayout.get(sourceAttr.id()).channel(); op = op.with( - new ValuesSourceReaderOperator.ValuesSourceReaderOperatorFactory(factories, docChannel, fieldName), + new ValuesSourceReaderOperator.Factory( + List.of(new ValuesSourceReaderOperator.FieldInfo(fieldName, loaders)), + readers, + docChannel + ), layout.build() ); } @@ -173,7 +175,8 @@ public final Operator.OperatorFactory ordinalGroupingOperatorFactory( // The grouping-by values are ready, let's group on them directly. // Costin: why are they ready and not already exposed in the layout? return new OrdinalsGroupingOperator.OrdinalsGroupingOperatorFactory( - BlockReaderFactories.factories(searchContexts, attrSource.name(), EsqlDataTypes.isUnsupported(attrSource.dataType())), + BlockReaderFactories.loaders(searchContexts, attrSource.name(), EsqlDataTypes.isUnsupported(attrSource.dataType())), + searchContexts.stream().map(s -> s.searcher().getIndexReader()).toList(), groupElementType, docChannel, attrSource.name(), diff --git a/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapper.java b/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapper.java index 6c8462c9e4948..ebe25ea1da1d9 100644 --- a/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapper.java +++ b/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapper.java @@ -30,7 +30,6 @@ import org.elasticsearch.index.fielddata.FieldDataContext; import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.fielddata.plain.ConstantIndexFieldData; -import org.elasticsearch.index.mapper.BlockDocValuesReader; import org.elasticsearch.index.mapper.BlockLoader; import org.elasticsearch.index.mapper.ConstantFieldType; import org.elasticsearch.index.mapper.DocumentParserContext; @@ -137,45 +136,10 @@ public String familyTypeName() { @Override public BlockLoader blockLoader(BlockLoaderContext blContext) { - // TODO build a constant block directly if (value == null) { - return BlockDocValuesReader.nulls(); + return BlockLoader.CONSTANT_NULLS; } - BytesRef bytes = new BytesRef(value); - return context -> new BlockDocValuesReader() { - private int docId; - - @Override - public int docID() { - return docId; - } - - @Override - public BlockLoader.BytesRefBuilder builder(BlockLoader.BuilderFactory factory, int expectedCount) { - return factory.bytesRefs(expectedCount); - } - - @Override - public BlockLoader.Block readValues(BlockLoader.BuilderFactory factory, BlockLoader.Docs docs) { - try (BlockLoader.BytesRefBuilder builder = builder(factory, docs.count())) { - for (int i = 0; i < docs.count(); i++) { - builder.appendBytesRef(bytes); - } - return builder.build(); - } - } - - @Override - public void readValuesFromSingleDoc(int docId, BlockLoader.Builder builder) { - this.docId = docId; - ((BlockLoader.BytesRefBuilder) builder).appendBytesRef(bytes); - } - - @Override - public String toString() { - return "ConstantKeyword"; - } - }; + return BlockLoader.constantBytes(new BytesRef(value)); } @Override diff --git a/x-pack/plugin/mapper-constant-keyword/src/test/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapperTests.java b/x-pack/plugin/mapper-constant-keyword/src/test/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapperTests.java index aaa28e28b72c9..87db404a40142 100644 --- a/x-pack/plugin/mapper-constant-keyword/src/test/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapperTests.java +++ b/x-pack/plugin/mapper-constant-keyword/src/test/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapperTests.java @@ -16,8 +16,6 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.core.CheckedFunction; -import org.elasticsearch.index.mapper.BlockDocValuesReader; import org.elasticsearch.index.mapper.BlockLoader; import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.index.mapper.DocumentParsingException; @@ -229,24 +227,7 @@ protected boolean allowsNullValues() { * for newly created indices that haven't received any documents that * contain the field. */ - public void testNullValueBlockLoaderReadValues() throws IOException { - testNullBlockLoader(blockReader -> (TestBlock) blockReader.readValues(TestBlock.FACTORY, TestBlock.docs(0))); - } - - /** - * Test loading blocks when there is no defined value. This is allowed - * for newly created indices that haven't received any documents that - * contain the field. - */ - public void testNullValueBlockLoaderReadValuesFromSingleDoc() throws IOException { - testNullBlockLoader(blockReader -> { - TestBlock block = (TestBlock) blockReader.builder(TestBlock.FACTORY, 1); - blockReader.readValuesFromSingleDoc(0, block); - return block; - }); - } - - private void testNullBlockLoader(CheckedFunction body) throws IOException { + public void testNullValueBlockLoader() throws IOException { MapperService mapper = createMapperService(syntheticSourceMapping(b -> { b.startObject("field"); b.field("type", "constant_keyword"); @@ -274,7 +255,18 @@ public Set sourcePaths(String name) { iw.addDocument(doc); iw.close(); try (DirectoryReader reader = DirectoryReader.open(directory)) { - TestBlock block = body.apply(loader.reader(reader.leaves().get(0))); + TestBlock block = (TestBlock) loader.columnAtATimeReader(reader.leaves().get(0)) + .read(TestBlock.FACTORY, new BlockLoader.Docs() { + @Override + public int count() { + return 1; + } + + @Override + public int get(int i) { + return 0; + } + }); assertThat(block.get(0), nullValue()); } } diff --git a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java index 90c055f3e77bb..97ffd50d5b8c3 100644 --- a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java +++ b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java @@ -319,12 +319,12 @@ public Query rangeQuery( public BlockLoader blockLoader(BlockLoaderContext blContext) { if (indexMode == IndexMode.TIME_SERIES && metricType == TimeSeriesParams.MetricType.COUNTER) { // Counters are not supported by ESQL so we load them in null - return BlockDocValuesReader.nulls(); + return BlockLoader.CONSTANT_NULLS; } if (hasDocValues()) { - return BlockDocValuesReader.longs(name()); + return new BlockDocValuesReader.LongsBlockLoader(name()); } - return BlockSourceReader.longs(new SourceValueFetcher(blContext.sourcePaths(name()), nullValueFormatted) { + return new BlockSourceReader.LongsBlockLoader(new SourceValueFetcher(blContext.sourcePaths(name()), nullValueFormatted) { @Override protected Object parseSourceValue(Object value) { if (value.equals("")) { diff --git a/x-pack/plugin/mapper-version/src/main/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapper.java b/x-pack/plugin/mapper-version/src/main/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapper.java index f4fb83fd9a91c..1ed63bb17e201 100644 --- a/x-pack/plugin/mapper-version/src/main/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapper.java +++ b/x-pack/plugin/mapper-version/src/main/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapper.java @@ -295,7 +295,7 @@ protected BytesRef indexedValueForSearch(Object value) { @Override public BlockLoader blockLoader(BlockLoaderContext blContext) { failIfNoDocValues(); - return BlockDocValuesReader.bytesRefsFromOrds(name()); + return new BlockDocValuesReader.BytesRefsFromOrdsBlockLoader(name()); } @Override diff --git a/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java b/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java index 480704b89ca60..1954e291b1a7f 100644 --- a/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java +++ b/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java @@ -855,9 +855,7 @@ public Query termsQuery(Collection values, SearchExecutionContext context) { @Override public BlockLoader blockLoader(BlockLoaderContext blContext) { if (hasDocValues()) { - // TODO it'd almost certainly be faster to drop directly to doc values like we do with keyword but this'll do for now - IndexFieldData fd = new StringBinaryIndexFieldData(name(), CoreValuesSourceType.KEYWORD, null); - return BlockDocValuesReader.bytesRefsFromDocValues(context -> fd.load(context).getBytesValues()); + return new BlockDocValuesReader.BytesRefsFromBinaryBlockLoader(name()); } return null; } From 951acf2ca25d0f32fb486a6c757e9a102a8d2f9a Mon Sep 17 00:00:00 2001 From: David Turner Date: Mon, 20 Nov 2023 14:51:13 +0000 Subject: [PATCH 21/28] Migrate JoinValidationService to TransportVersion (#102372) We introduced a new join validation protocol in version 8.3, effectively a different transport protocol. However today we are still checking the node's release version when deciding which validation protocol to use. This commit migrates to using the `TransportVersion` of the relevant connection. --- .../coordination/JoinValidationService.java | 114 ++++++++++-------- 1 file changed, 64 insertions(+), 50 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/JoinValidationService.java b/server/src/main/java/org/elasticsearch/cluster/coordination/JoinValidationService.java index d9911ad12df84..dc18a7950394a 100644 --- a/server/src/main/java/org/elasticsearch/cluster/coordination/JoinValidationService.java +++ b/server/src/main/java/org/elasticsearch/cluster/coordination/JoinValidationService.java @@ -12,7 +12,7 @@ import org.apache.logging.log4j.Logger; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.TransportVersion; -import org.elasticsearch.Version; +import org.elasticsearch.TransportVersions; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.ActionRunnable; import org.elasticsearch.cluster.ClusterState; @@ -30,6 +30,7 @@ import org.elasticsearch.core.Nullable; import org.elasticsearch.core.RefCounted; import org.elasticsearch.core.TimeValue; +import org.elasticsearch.core.UpdateForV9; import org.elasticsearch.env.Environment; import org.elasticsearch.node.NodeClosedException; import org.elasticsearch.threadpool.ThreadPool; @@ -148,10 +149,23 @@ public JoinValidationService( } public void validateJoin(DiscoveryNode discoveryNode, ActionListener listener) { - if (discoveryNode.getVersion().onOrAfter(Version.V_8_3_0)) { + // This node isn't in the cluster yet so ClusterState#getMinTransportVersion() doesn't apply, we must obtain a specific connection + // so we can check its transport version to decide how to proceed. + + final Transport.Connection connection; + try { + connection = transportService.getConnection(discoveryNode); + assert connection != null; + } catch (Exception e) { + assert e instanceof NodeNotConnectedException : e; + listener.onFailure(e); + return; + } + + if (connection.getTransportVersion().onOrAfter(TransportVersions.V_8_3_0)) { if (executeRefs.tryIncRef()) { try { - execute(new JoinValidation(discoveryNode, listener)); + execute(new JoinValidation(discoveryNode, connection, listener)); } finally { executeRefs.decRef(); } @@ -159,39 +173,44 @@ public void validateJoin(DiscoveryNode discoveryNode, ActionListener liste listener.onFailure(new NodeClosedException(transportService.getLocalNode())); } } else { - final var responseHandler = TransportResponseHandler.empty(responseExecutor, listener.delegateResponse((l, e) -> { - logger.warn(() -> "failed to validate incoming join request from node [" + discoveryNode + "]", e); - listener.onFailure( - new IllegalStateException( - String.format( - Locale.ROOT, - "failure when sending a join validation request from [%s] to [%s]", - transportService.getLocalNode().descriptionWithoutAttributes(), - discoveryNode.descriptionWithoutAttributes() - ), - e - ) - ); - })); - final var clusterState = clusterStateSupplier.get(); - if (clusterState != null) { - assert clusterState.nodes().isLocalNodeElectedMaster(); - transportService.sendRequest( - discoveryNode, - JOIN_VALIDATE_ACTION_NAME, - new ValidateJoinRequest(clusterState), - REQUEST_OPTIONS, - responseHandler - ); - } else { - transportService.sendRequest( - discoveryNode, - JoinHelper.JOIN_PING_ACTION_NAME, - TransportRequest.Empty.INSTANCE, - REQUEST_OPTIONS, - responseHandler - ); - } + legacyValidateJoin(discoveryNode, listener, connection); + } + } + + @UpdateForV9 + private void legacyValidateJoin(DiscoveryNode discoveryNode, ActionListener listener, Transport.Connection connection) { + final var responseHandler = TransportResponseHandler.empty(responseExecutor, listener.delegateResponse((l, e) -> { + logger.warn(() -> "failed to validate incoming join request from node [" + discoveryNode + "]", e); + listener.onFailure( + new IllegalStateException( + String.format( + Locale.ROOT, + "failure when sending a join validation request from [%s] to [%s]", + transportService.getLocalNode().descriptionWithoutAttributes(), + discoveryNode.descriptionWithoutAttributes() + ), + e + ) + ); + })); + final var clusterState = clusterStateSupplier.get(); + if (clusterState != null) { + assert clusterState.nodes().isLocalNodeElectedMaster(); + transportService.sendRequest( + connection, + JOIN_VALIDATE_ACTION_NAME, + new ValidateJoinRequest(clusterState), + REQUEST_OPTIONS, + responseHandler + ); + } else { + transportService.sendRequest( + connection, + JoinHelper.JOIN_PING_ACTION_NAME, + TransportRequest.Empty.INSTANCE, + REQUEST_OPTIONS, + responseHandler + ); } } @@ -312,27 +331,22 @@ public String toString() { private class JoinValidation extends ActionRunnable { private final DiscoveryNode discoveryNode; + private final Transport.Connection connection; - JoinValidation(DiscoveryNode discoveryNode, ActionListener listener) { + JoinValidation(DiscoveryNode discoveryNode, Transport.Connection connection, ActionListener listener) { super(listener); this.discoveryNode = discoveryNode; + this.connection = connection; } @Override - protected void doRun() throws Exception { - assert discoveryNode.getVersion().onOrAfter(Version.V_8_3_0) : discoveryNode.getVersion(); + protected void doRun() { + assert connection.getTransportVersion().onOrAfter(TransportVersions.V_8_3_0) : discoveryNode.getVersion(); // NB these things never run concurrently to each other, or to the cache cleaner (see IMPLEMENTATION NOTES above) so it is safe // to do these (non-atomic) things to the (unsynchronized) statesByVersion map. - Transport.Connection connection; - try { - connection = transportService.getConnection(discoveryNode); - } catch (NodeNotConnectedException e) { - listener.onFailure(e); - return; - } - var version = connection.getTransportVersion(); - var cachedBytes = statesByVersion.get(version); - var bytes = maybeSerializeClusterState(cachedBytes, discoveryNode, version); + var transportVersion = connection.getTransportVersion(); + var cachedBytes = statesByVersion.get(transportVersion); + var bytes = maybeSerializeClusterState(cachedBytes, discoveryNode, transportVersion); if (bytes == null) { // Normally if we're not the master then the Coordinator sends a ping message just to validate connectivity instead of // getting here. But if we were the master when the Coordinator checked then we might not be the master any more, so we @@ -354,7 +368,7 @@ protected void doRun() throws Exception { transportService.sendRequest( connection, JOIN_VALIDATE_ACTION_NAME, - new BytesTransportRequest(bytes, version), + new BytesTransportRequest(bytes, transportVersion), REQUEST_OPTIONS, new CleanableResponseHandler<>( listener.map(ignored -> null), From 893e0bed9f853e1b5f4fc9ded1b20b6dc1b4edc8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Przemys=C5=82aw=20Witek?= Date: Mon, 20 Nov 2023 16:00:57 +0100 Subject: [PATCH 22/28] [Transform] Skip shards that don't match the source query during checkpointing (#102138) --- docs/changelog/102138.yaml | 5 + .../org/elasticsearch/TransportVersions.java | 1 + .../transform/action/GetCheckpointAction.java | 30 +- .../GetCheckpointActionRequestTests.java | 21 +- .../checkpoint/TransformCCSCanMatchIT.java | 415 ++++++++++++++++++ .../checkpoint/TransformGetCheckpointIT.java | 40 ++ .../TransformGetCheckpointTests.java | 12 +- .../action/TransportGetCheckpointAction.java | 285 +++++++----- .../TransportGetTransformStatsAction.java | 3 +- .../checkpoint/DefaultCheckpointProvider.java | 15 +- .../TransportGetCheckpointActionTests.java | 134 ++++++ 11 files changed, 841 insertions(+), 120 deletions(-) create mode 100644 docs/changelog/102138.yaml create mode 100644 x-pack/plugin/transform/src/internalClusterTest/java/org/elasticsearch/xpack/transform/checkpoint/TransformCCSCanMatchIT.java create mode 100644 x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/action/TransportGetCheckpointActionTests.java diff --git a/docs/changelog/102138.yaml b/docs/changelog/102138.yaml new file mode 100644 index 0000000000000..3819e3201150e --- /dev/null +++ b/docs/changelog/102138.yaml @@ -0,0 +1,5 @@ +pr: 102138 +summary: Skip shards that don't match the source query during checkpointing +area: Transform +type: enhancement +issues: [] diff --git a/server/src/main/java/org/elasticsearch/TransportVersions.java b/server/src/main/java/org/elasticsearch/TransportVersions.java index baae500b70d55..5ad1d43c0d4f8 100644 --- a/server/src/main/java/org/elasticsearch/TransportVersions.java +++ b/server/src/main/java/org/elasticsearch/TransportVersions.java @@ -172,6 +172,7 @@ static TransportVersion def(int id) { public static final TransportVersion DATA_STREAM_FAILURE_STORE_ADDED = def(8_541_00_0); public static final TransportVersion ML_INFERENCE_OPENAI_ADDED = def(8_542_00_0); public static final TransportVersion SHUTDOWN_MIGRATION_STATUS_INCLUDE_COUNTS = def(8_543_00_0); + public static final TransportVersion TRANSFORM_GET_CHECKPOINT_QUERY_AND_CLUSTER_ADDED = def(8_544_00_0); /* * STOP! READ THIS FIRST! No, really, diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/action/GetCheckpointAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/action/GetCheckpointAction.java index 7ac27d79d3cb8..e492a98748af2 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/action/GetCheckpointAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/action/GetCheckpointAction.java @@ -18,6 +18,7 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.core.TimeValue; +import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.tasks.CancellableTask; import org.elasticsearch.tasks.TaskId; @@ -48,12 +49,21 @@ public static class Request extends ActionRequest implements IndicesRequest.Repl private String[] indices; private final IndicesOptions indicesOptions; + private final QueryBuilder query; + private final String cluster; private final TimeValue timeout; public Request(StreamInput in) throws IOException { super(in); indices = in.readStringArray(); indicesOptions = IndicesOptions.readIndicesOptions(in); + if (in.getTransportVersion().onOrAfter(TransportVersions.TRANSFORM_GET_CHECKPOINT_QUERY_AND_CLUSTER_ADDED)) { + query = in.readOptionalNamedWriteable(QueryBuilder.class); + cluster = in.readOptionalString(); + } else { + query = null; + cluster = null; + } if (in.getTransportVersion().onOrAfter(TransportVersions.TRANSFORM_GET_CHECKPOINT_TIMEOUT_ADDED)) { timeout = in.readOptionalTimeValue(); } else { @@ -61,9 +71,11 @@ public Request(StreamInput in) throws IOException { } } - public Request(String[] indices, IndicesOptions indicesOptions, TimeValue timeout) { + public Request(String[] indices, IndicesOptions indicesOptions, QueryBuilder query, String cluster, TimeValue timeout) { this.indices = indices != null ? indices : Strings.EMPTY_ARRAY; this.indicesOptions = indicesOptions; + this.query = query; + this.cluster = cluster; this.timeout = timeout; } @@ -82,6 +94,14 @@ public IndicesOptions indicesOptions() { return indicesOptions; } + public QueryBuilder getQuery() { + return query; + } + + public String getCluster() { + return cluster; + } + public TimeValue getTimeout() { return timeout; } @@ -98,12 +118,14 @@ public boolean equals(Object obj) { return Arrays.equals(indices, that.indices) && Objects.equals(indicesOptions, that.indicesOptions) + && Objects.equals(query, that.query) + && Objects.equals(cluster, that.cluster) && Objects.equals(timeout, that.timeout); } @Override public int hashCode() { - return Objects.hash(Arrays.hashCode(indices), indicesOptions, timeout); + return Objects.hash(Arrays.hashCode(indices), indicesOptions, query, cluster, timeout); } @Override @@ -111,6 +133,10 @@ public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); out.writeStringArray(indices); indicesOptions.writeIndicesOptions(out); + if (out.getTransportVersion().onOrAfter(TransportVersions.TRANSFORM_GET_CHECKPOINT_QUERY_AND_CLUSTER_ADDED)) { + out.writeOptionalNamedWriteable(query); + out.writeOptionalString(cluster); + } if (out.getTransportVersion().onOrAfter(TransportVersions.TRANSFORM_GET_CHECKPOINT_TIMEOUT_ADDED)) { out.writeOptionalTimeValue(timeout); } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/action/GetCheckpointActionRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/action/GetCheckpointActionRequestTests.java index 43ec0a0f1b4f5..e96a7741b4f52 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/action/GetCheckpointActionRequestTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/action/GetCheckpointActionRequestTests.java @@ -11,9 +11,10 @@ import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.common.io.stream.Writeable.Reader; import org.elasticsearch.core.TimeValue; +import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.tasks.CancellableTask; import org.elasticsearch.tasks.TaskId; -import org.elasticsearch.test.AbstractWireSerializingTestCase; import org.elasticsearch.xpack.core.transform.action.GetCheckpointAction.Request; import java.util.ArrayList; @@ -26,7 +27,7 @@ import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; -public class GetCheckpointActionRequestTests extends AbstractWireSerializingTestCase { +public class GetCheckpointActionRequestTests extends AbstractWireSerializingTransformTestCase { @Override protected Request createTestInstance() { @@ -42,9 +43,11 @@ protected Reader instanceReader() { protected Request mutateInstance(Request instance) { List indices = instance.indices() != null ? new ArrayList<>(Arrays.asList(instance.indices())) : new ArrayList<>(); IndicesOptions indicesOptions = instance.indicesOptions(); + QueryBuilder query = instance.getQuery(); + String cluster = instance.getCluster(); TimeValue timeout = instance.getTimeout(); - switch (between(0, 2)) { + switch (between(0, 4)) { case 0: indices.add(randomAlphaOfLengthBetween(1, 20)); break; @@ -58,13 +61,19 @@ protected Request mutateInstance(Request instance) { ); break; case 2: + query = query != null ? null : QueryBuilders.matchAllQuery(); + break; + case 3: + cluster = cluster != null ? null : randomAlphaOfLengthBetween(1, 10); + break; + case 4: timeout = timeout != null ? null : TimeValue.timeValueSeconds(randomIntBetween(1, 300)); break; default: throw new AssertionError("Illegal randomization branch"); } - return new Request(indices.toArray(new String[0]), indicesOptions, timeout); + return new Request(indices.toArray(new String[0]), indicesOptions, query, cluster, timeout); } public void testCreateTask() { @@ -74,7 +83,7 @@ public void testCreateTask() { } public void testCreateTaskWithNullIndices() { - Request request = new Request(null, null, null); + Request request = new Request(null, null, null, null, null); CancellableTask task = request.createTask(123, "type", "action", new TaskId("dummy-node:456"), Map.of()); assertThat(task.getDescription(), is(equalTo("get_checkpoint[0]"))); } @@ -89,6 +98,8 @@ private static Request randomRequest(Integer numIndices) { Boolean.toString(randomBoolean()), SearchRequest.DEFAULT_INDICES_OPTIONS ), + randomBoolean() ? QueryBuilders.matchAllQuery() : null, + randomBoolean() ? randomAlphaOfLengthBetween(1, 10) : null, randomBoolean() ? TimeValue.timeValueSeconds(randomIntBetween(1, 300)) : null ); } diff --git a/x-pack/plugin/transform/src/internalClusterTest/java/org/elasticsearch/xpack/transform/checkpoint/TransformCCSCanMatchIT.java b/x-pack/plugin/transform/src/internalClusterTest/java/org/elasticsearch/xpack/transform/checkpoint/TransformCCSCanMatchIT.java new file mode 100644 index 0000000000000..d05acc7a7b368 --- /dev/null +++ b/x-pack/plugin/transform/src/internalClusterTest/java/org/elasticsearch/xpack/transform/checkpoint/TransformCCSCanMatchIT.java @@ -0,0 +1,415 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.transform.checkpoint; + +import org.apache.lucene.document.LongPoint; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.PointValues; +import org.apache.lucene.util.SetOnce; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.action.support.IndicesOptions; +import org.elasticsearch.action.support.master.AcknowledgedResponse; +import org.elasticsearch.client.internal.Client; +import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.CollectionUtils; +import org.elasticsearch.common.util.concurrent.EsExecutors; +import org.elasticsearch.core.TimeValue; +import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.index.engine.EngineConfig; +import org.elasticsearch.index.engine.EngineFactory; +import org.elasticsearch.index.engine.InternalEngine; +import org.elasticsearch.index.engine.InternalEngineFactory; +import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.index.shard.IndexLongFieldRange; +import org.elasticsearch.index.shard.ShardLongFieldRange; +import org.elasticsearch.node.NodeRoleSettings; +import org.elasticsearch.plugins.EnginePlugin; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.search.SearchModule; +import org.elasticsearch.search.aggregations.BaseAggregationBuilder; +import org.elasticsearch.search.builder.SearchSourceBuilder; +import org.elasticsearch.test.AbstractMultiClustersTestCase; +import org.elasticsearch.test.hamcrest.ElasticsearchAssertions; +import org.elasticsearch.xcontent.NamedXContentRegistry; +import org.elasticsearch.xcontent.ParseField; +import org.elasticsearch.xcontent.XContentParser; +import org.elasticsearch.xcontent.json.JsonXContent; +import org.elasticsearch.xpack.core.ClientHelper; +import org.elasticsearch.xpack.core.XPackSettings; +import org.elasticsearch.xpack.core.transform.MockDeprecatedAggregationBuilder; +import org.elasticsearch.xpack.core.transform.MockDeprecatedQueryBuilder; +import org.elasticsearch.xpack.core.transform.TransformNamedXContentProvider; +import org.elasticsearch.xpack.core.transform.action.GetCheckpointAction; +import org.elasticsearch.xpack.core.transform.action.GetTransformStatsAction; +import org.elasticsearch.xpack.core.transform.action.PutTransformAction; +import org.elasticsearch.xpack.core.transform.action.StartTransformAction; +import org.elasticsearch.xpack.core.transform.transforms.DestConfig; +import org.elasticsearch.xpack.core.transform.transforms.QueryConfig; +import org.elasticsearch.xpack.core.transform.transforms.SourceConfig; +import org.elasticsearch.xpack.core.transform.transforms.TransformConfig; +import org.elasticsearch.xpack.core.transform.transforms.TransformStats; +import org.elasticsearch.xpack.core.transform.transforms.latest.LatestConfig; +import org.elasticsearch.xpack.transform.LocalStateTransform; +import org.junit.Before; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.util.Arrays; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; + +import static java.util.Collections.emptyList; +import static org.elasticsearch.xpack.core.ClientHelper.TRANSFORM_ORIGIN; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasSize; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.nullValue; + +public class TransformCCSCanMatchIT extends AbstractMultiClustersTestCase { + + private static final String REMOTE_CLUSTER = "cluster_a"; + private static final TimeValue TIMEOUT = TimeValue.timeValueMinutes(1); + + private NamedXContentRegistry namedXContentRegistry; + private long timestamp; + private int oldLocalNumShards; + private int localOldDocs; + private int oldRemoteNumShards; + private int remoteOldDocs; + private int newLocalNumShards; + private int localNewDocs; + private int newRemoteNumShards; + private int remoteNewDocs; + + @Before + public void setUpNamedXContentRegistryAndIndices() throws Exception { + SearchModule searchModule = new SearchModule(Settings.EMPTY, emptyList()); + + List namedXContents = searchModule.getNamedXContents(); + namedXContents.add( + new NamedXContentRegistry.Entry( + QueryBuilder.class, + new ParseField(MockDeprecatedQueryBuilder.NAME), + (p, c) -> MockDeprecatedQueryBuilder.fromXContent(p) + ) + ); + namedXContents.add( + new NamedXContentRegistry.Entry( + BaseAggregationBuilder.class, + new ParseField(MockDeprecatedAggregationBuilder.NAME), + (p, c) -> MockDeprecatedAggregationBuilder.fromXContent(p) + ) + ); + + namedXContents.addAll(new TransformNamedXContentProvider().getNamedXContentParsers()); + + namedXContentRegistry = new NamedXContentRegistry(namedXContents); + + timestamp = randomLongBetween(10_000_000, 50_000_000); + + oldLocalNumShards = randomIntBetween(1, 5); + localOldDocs = createIndexAndIndexDocs(LOCAL_CLUSTER, "local_old_index", oldLocalNumShards, timestamp - 10_000, true); + oldRemoteNumShards = randomIntBetween(1, 5); + remoteOldDocs = createIndexAndIndexDocs(REMOTE_CLUSTER, "remote_old_index", oldRemoteNumShards, timestamp - 10_000, true); + + newLocalNumShards = randomIntBetween(1, 5); + localNewDocs = createIndexAndIndexDocs(LOCAL_CLUSTER, "local_new_index", newLocalNumShards, timestamp, randomBoolean()); + newRemoteNumShards = randomIntBetween(1, 5); + remoteNewDocs = createIndexAndIndexDocs(REMOTE_CLUSTER, "remote_new_index", newRemoteNumShards, timestamp, randomBoolean()); + } + + private int createIndexAndIndexDocs(String cluster, String index, int numberOfShards, long timestamp, boolean exposeTimestamp) + throws Exception { + Client client = client(cluster); + ElasticsearchAssertions.assertAcked( + client.admin() + .indices() + .prepareCreate(index) + .setSettings( + Settings.builder() + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, numberOfShards) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) + ) + .setMapping("@timestamp", "type=date", "position", "type=long") + ); + int numDocs = between(100, 500); + for (int i = 0; i < numDocs; i++) { + client.prepareIndex(index).setSource("position", i, "@timestamp", timestamp + i).get(); + } + if (exposeTimestamp) { + client.admin().indices().prepareClose(index).get(); + client.admin() + .indices() + .prepareUpdateSettings(index) + .setSettings(Settings.builder().put(IndexMetadata.INDEX_BLOCKS_WRITE_SETTING.getKey(), true).build()) + .get(); + client.admin().indices().prepareOpen(index).get(); + assertBusy(() -> { + IndexLongFieldRange timestampRange = cluster(cluster).clusterService().state().metadata().index(index).getTimestampRange(); + assertTrue(Strings.toString(timestampRange), timestampRange.containsAllShardRanges()); + }); + } else { + client.admin().indices().prepareRefresh(index).get(); + } + return numDocs; + } + + public void testSearchAction_MatchAllQuery() { + testSearchAction(QueryBuilders.matchAllQuery(), true, localOldDocs + localNewDocs + remoteOldDocs + remoteNewDocs, 0); + testSearchAction(QueryBuilders.matchAllQuery(), false, localOldDocs + localNewDocs + remoteOldDocs + remoteNewDocs, 0); + } + + public void testSearchAction_RangeQuery() { + testSearchAction( + QueryBuilders.rangeQuery("@timestamp").from(timestamp), // This query only matches new documents + true, + localNewDocs + remoteNewDocs, + oldLocalNumShards + oldRemoteNumShards + ); + testSearchAction( + QueryBuilders.rangeQuery("@timestamp").from(timestamp), // This query only matches new documents + false, + localNewDocs + remoteNewDocs, + oldLocalNumShards + oldRemoteNumShards + ); + } + + public void testSearchAction_RangeQueryThatMatchesNoShards() { + testSearchAction( + QueryBuilders.rangeQuery("@timestamp").from(100_000_000), // This query matches no documents + true, + 0, + // All but 2 shards are skipped. TBH I don't know why this 2 shards are not skipped + oldLocalNumShards + newLocalNumShards + oldRemoteNumShards + newRemoteNumShards - 2 + ); + testSearchAction( + QueryBuilders.rangeQuery("@timestamp").from(100_000_000), // This query matches no documents + false, + 0, + // All but 1 shards are skipped. TBH I don't know why this 1 shard is not skipped + oldLocalNumShards + newLocalNumShards + oldRemoteNumShards + newRemoteNumShards - 1 + ); + } + + private void testSearchAction(QueryBuilder query, boolean ccsMinimizeRoundtrips, long expectedHitCount, int expectedSkippedShards) { + SearchSourceBuilder source = new SearchSourceBuilder().query(query); + SearchRequest request = new SearchRequest("local_*", "*:remote_*"); + request.source(source).setCcsMinimizeRoundtrips(ccsMinimizeRoundtrips); + SearchResponse response = client().search(request).actionGet(); + ElasticsearchAssertions.assertHitCount(response, expectedHitCount); + int expectedTotalShards = oldLocalNumShards + newLocalNumShards + oldRemoteNumShards + newRemoteNumShards; + assertThat("Response was: " + response, response.getTotalShards(), is(equalTo(expectedTotalShards))); + assertThat("Response was: " + response, response.getSuccessfulShards(), is(equalTo(expectedTotalShards))); + assertThat("Response was: " + response, response.getFailedShards(), is(equalTo(0))); + assertThat("Response was: " + response, response.getSkippedShards(), is(equalTo(expectedSkippedShards))); + } + + public void testGetCheckpointAction_MatchAllQuery() throws InterruptedException { + testGetCheckpointAction( + client(), + null, + new String[] { "local_*" }, + QueryBuilders.matchAllQuery(), + Set.of("local_old_index", "local_new_index") + ); + testGetCheckpointAction( + client().getRemoteClusterClient(REMOTE_CLUSTER, EsExecutors.DIRECT_EXECUTOR_SERVICE), + REMOTE_CLUSTER, + new String[] { "remote_*" }, + QueryBuilders.matchAllQuery(), + Set.of("remote_old_index", "remote_new_index") + ); + } + + public void testGetCheckpointAction_RangeQuery() throws InterruptedException { + testGetCheckpointAction( + client(), + null, + new String[] { "local_*" }, + QueryBuilders.rangeQuery("@timestamp").from(timestamp), + Set.of("local_new_index") + ); + testGetCheckpointAction( + client().getRemoteClusterClient(REMOTE_CLUSTER, EsExecutors.DIRECT_EXECUTOR_SERVICE), + REMOTE_CLUSTER, + new String[] { "remote_*" }, + QueryBuilders.rangeQuery("@timestamp").from(timestamp), + Set.of("remote_new_index") + ); + } + + public void testGetCheckpointAction_RangeQueryThatMatchesNoShards() throws InterruptedException { + testGetCheckpointAction( + client(), + null, + new String[] { "local_*" }, + QueryBuilders.rangeQuery("@timestamp").from(100_000_000), + Set.of() + ); + testGetCheckpointAction( + client().getRemoteClusterClient(REMOTE_CLUSTER, EsExecutors.DIRECT_EXECUTOR_SERVICE), + REMOTE_CLUSTER, + new String[] { "remote_*" }, + QueryBuilders.rangeQuery("@timestamp").from(100_000_000), + Set.of() + ); + } + + private void testGetCheckpointAction(Client client, String cluster, String[] indices, QueryBuilder query, Set expectedIndices) + throws InterruptedException { + final GetCheckpointAction.Request request = new GetCheckpointAction.Request( + indices, + IndicesOptions.LENIENT_EXPAND_OPEN, + query, + cluster, + TIMEOUT + ); + + CountDownLatch latch = new CountDownLatch(1); + SetOnce finalResponse = new SetOnce<>(); + SetOnce finalException = new SetOnce<>(); + ClientHelper.executeAsyncWithOrigin( + client, + TRANSFORM_ORIGIN, + GetCheckpointAction.INSTANCE, + request, + ActionListener.wrap(response -> { + finalResponse.set(response); + latch.countDown(); + }, e -> { + finalException.set(e); + latch.countDown(); + }) + ); + latch.await(10, TimeUnit.SECONDS); + + assertThat(finalException.get(), is(nullValue())); + assertThat("Response was: " + finalResponse.get(), finalResponse.get().getCheckpoints().keySet(), is(equalTo(expectedIndices))); + } + + public void testTransformLifecycle_MatchAllQuery() throws Exception { + testTransformLifecycle(QueryBuilders.matchAllQuery(), localOldDocs + localNewDocs + remoteOldDocs + remoteNewDocs); + } + + public void testTransformLifecycle_RangeQuery() throws Exception { + testTransformLifecycle(QueryBuilders.rangeQuery("@timestamp").from(timestamp), localNewDocs + remoteNewDocs); + } + + public void testTransformLifecycle_RangeQueryThatMatchesNoShards() throws Exception { + testTransformLifecycle(QueryBuilders.rangeQuery("@timestamp").from(100_000_000), 0); + } + + private void testTransformLifecycle(QueryBuilder query, long expectedHitCount) throws Exception { + String transformId = "test-transform-lifecycle"; + { + QueryConfig queryConfig; + try (XContentParser parser = createParser(JsonXContent.jsonXContent, query.toString())) { + queryConfig = QueryConfig.fromXContent(parser, true); + assertNotNull(queryConfig.getQuery()); + } + TransformConfig transformConfig = TransformConfig.builder() + .setId(transformId) + .setSource(new SourceConfig(new String[] { "local_*", "*:remote_*" }, queryConfig, Map.of())) + .setDest(new DestConfig(transformId + "-dest", null, null)) + .setLatestConfig(new LatestConfig(List.of("position"), "@timestamp")) + .build(); + PutTransformAction.Request request = new PutTransformAction.Request(transformConfig, false, TIMEOUT); + AcknowledgedResponse response = client().execute(PutTransformAction.INSTANCE, request).actionGet(); + assertTrue(response.isAcknowledged()); + } + { + StartTransformAction.Request request = new StartTransformAction.Request(transformId, null, TIMEOUT); + StartTransformAction.Response response = client().execute(StartTransformAction.INSTANCE, request).actionGet(); + assertTrue(response.isAcknowledged()); + } + assertBusy(() -> { + GetTransformStatsAction.Request request = new GetTransformStatsAction.Request(transformId, TIMEOUT); + GetTransformStatsAction.Response response = client().execute(GetTransformStatsAction.INSTANCE, request).actionGet(); + assertThat("Stats were: " + response.getTransformsStats(), response.getTransformsStats(), hasSize(1)); + assertThat(response.getTransformsStats().get(0).getState(), is(equalTo(TransformStats.State.STOPPED))); + assertThat(response.getTransformsStats().get(0).getIndexerStats().getNumDocuments(), is(equalTo(expectedHitCount))); + assertThat(response.getTransformsStats().get(0).getIndexerStats().getNumDeletedDocuments(), is(equalTo(0L))); + assertThat(response.getTransformsStats().get(0).getIndexerStats().getSearchFailures(), is(equalTo(0L))); + assertThat(response.getTransformsStats().get(0).getIndexerStats().getIndexFailures(), is(equalTo(0L))); + }); + } + + @Override + protected NamedXContentRegistry xContentRegistry() { + return namedXContentRegistry; + } + + @Override + protected Collection remoteClusterAlias() { + return List.of(REMOTE_CLUSTER); + } + + @Override + protected Collection> nodePlugins(String clusterAlias) { + return CollectionUtils.appendToCopy( + CollectionUtils.appendToCopy(super.nodePlugins(clusterAlias), LocalStateTransform.class), + ExposingTimestampEnginePlugin.class + ); + } + + @Override + protected Settings nodeSettings() { + return Settings.builder() + .put(NodeRoleSettings.NODE_ROLES_SETTING.getKey(), "master, data, ingest, transform, remote_cluster_client") + .put(XPackSettings.SECURITY_ENABLED.getKey(), false) + .build(); + } + + private static class EngineWithExposingTimestamp extends InternalEngine { + EngineWithExposingTimestamp(EngineConfig engineConfig) { + super(engineConfig); + assert IndexMetadata.INDEX_BLOCKS_WRITE_SETTING.get(config().getIndexSettings().getSettings()) : "require read-only index"; + } + + @Override + public ShardLongFieldRange getRawFieldRange(String field) { + try (Searcher searcher = acquireSearcher("test")) { + final DirectoryReader directoryReader = searcher.getDirectoryReader(); + + final byte[] minPackedValue = PointValues.getMinPackedValue(directoryReader, field); + final byte[] maxPackedValue = PointValues.getMaxPackedValue(directoryReader, field); + if (minPackedValue == null || maxPackedValue == null) { + assert minPackedValue == null && maxPackedValue == null + : Arrays.toString(minPackedValue) + "-" + Arrays.toString(maxPackedValue); + return ShardLongFieldRange.EMPTY; + } + + return ShardLongFieldRange.of(LongPoint.decodeDimension(minPackedValue, 0), LongPoint.decodeDimension(maxPackedValue, 0)); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + } + + public static class ExposingTimestampEnginePlugin extends Plugin implements EnginePlugin { + + @Override + public Optional getEngineFactory(IndexSettings indexSettings) { + if (IndexMetadata.INDEX_BLOCKS_WRITE_SETTING.get(indexSettings.getSettings())) { + return Optional.of(EngineWithExposingTimestamp::new); + } else { + return Optional.of(new InternalEngineFactory()); + } + } + } +} diff --git a/x-pack/plugin/transform/src/internalClusterTest/java/org/elasticsearch/xpack/transform/checkpoint/TransformGetCheckpointIT.java b/x-pack/plugin/transform/src/internalClusterTest/java/org/elasticsearch/xpack/transform/checkpoint/TransformGetCheckpointIT.java index bb159856b965d..82a3ea85bfe6a 100644 --- a/x-pack/plugin/transform/src/internalClusterTest/java/org/elasticsearch/xpack/transform/checkpoint/TransformGetCheckpointIT.java +++ b/x-pack/plugin/transform/src/internalClusterTest/java/org/elasticsearch/xpack/transform/checkpoint/TransformGetCheckpointIT.java @@ -15,6 +15,7 @@ import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.common.Strings; import org.elasticsearch.core.TimeValue; +import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.xcontent.XContentType; import org.elasticsearch.xpack.core.transform.action.GetCheckpointAction; import org.elasticsearch.xpack.transform.TransformSingleNodeTestCase; @@ -25,6 +26,7 @@ import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; +import static org.hamcrest.Matchers.anEmptyMap; import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.startsWith; @@ -46,6 +48,8 @@ public void testGetCheckpoint() throws Exception { final GetCheckpointAction.Request request = new GetCheckpointAction.Request( new String[] { indexNamePrefix + "*" }, IndicesOptions.LENIENT_EXPAND_OPEN, + null, + null, TimeValue.timeValueSeconds(5) ); @@ -99,6 +103,40 @@ public void testGetCheckpoint() throws Exception { ); } + public void testGetCheckpointWithQueryThatFiltersOutEverything() throws Exception { + final String indexNamePrefix = "test_index-"; + final int indices = randomIntBetween(1, 5); + final int shards = randomIntBetween(1, 5); + final int docsToCreatePerShard = randomIntBetween(0, 10); + + for (int i = 0; i < indices; ++i) { + indicesAdmin().prepareCreate(indexNamePrefix + i) + .setSettings(indexSettings(shards, 1)) + .setMapping("field", "type=long", "@timestamp", "type=date") + .get(); + for (int j = 0; j < shards; ++j) { + for (int d = 0; d < docsToCreatePerShard; ++d) { + client().prepareIndex(indexNamePrefix + i) + .setSource(Strings.format("{ \"field\":%d, \"@timestamp\": %d }", j, 10_000_000 + d + i + j), XContentType.JSON) + .get(); + } + } + } + indicesAdmin().refresh(new RefreshRequest(indexNamePrefix + "*")); + + final GetCheckpointAction.Request request = new GetCheckpointAction.Request( + new String[] { indexNamePrefix + "*" }, + IndicesOptions.LENIENT_EXPAND_OPEN, + // This query does not match any documents + QueryBuilders.rangeQuery("@timestamp").gte(20_000_000), + null, + TimeValue.timeValueSeconds(5) + ); + + final GetCheckpointAction.Response response = client().execute(GetCheckpointAction.INSTANCE, request).get(); + assertThat("Response was: " + response.getCheckpoints(), response.getCheckpoints(), is(anEmptyMap())); + } + public void testGetCheckpointTimeoutExceeded() throws Exception { final String indexNamePrefix = "test_index-"; final int indices = 100; @@ -111,6 +149,8 @@ public void testGetCheckpointTimeoutExceeded() throws Exception { final GetCheckpointAction.Request request = new GetCheckpointAction.Request( new String[] { indexNamePrefix + "*" }, IndicesOptions.LENIENT_EXPAND_OPEN, + null, + null, TimeValue.ZERO ); diff --git a/x-pack/plugin/transform/src/internalClusterTest/java/org/elasticsearch/xpack/transform/checkpoint/TransformGetCheckpointTests.java b/x-pack/plugin/transform/src/internalClusterTest/java/org/elasticsearch/xpack/transform/checkpoint/TransformGetCheckpointTests.java index 1411576e61d58..300a075c9f1b2 100644 --- a/x-pack/plugin/transform/src/internalClusterTest/java/org/elasticsearch/xpack/transform/checkpoint/TransformGetCheckpointTests.java +++ b/x-pack/plugin/transform/src/internalClusterTest/java/org/elasticsearch/xpack/transform/checkpoint/TransformGetCheckpointTests.java @@ -15,6 +15,7 @@ import org.elasticsearch.action.support.ActionTestUtils; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.action.support.replication.ClusterStateCreationUtils; +import org.elasticsearch.client.internal.Client; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; @@ -71,6 +72,7 @@ public class TransformGetCheckpointTests extends ESSingleNodeTestCase { private IndicesService indicesService; private ThreadPool threadPool; private IndexNameExpressionResolver indexNameExpressionResolver; + private Client client; private MockTransport mockTransport; private Task transformTask; private final String indexNamePattern = "test_index-"; @@ -133,6 +135,8 @@ protected void onSendRequest(long requestId, String action, TransportRequest req .putCompatibilityVersions("node01", TransportVersions.V_8_5_0, Map.of()) .build(); + client = mock(Client.class); + transformTask = new Task( 1L, "persistent", @@ -157,6 +161,8 @@ public void testEmptyCheckpoint() throws InterruptedException { GetCheckpointAction.Request request = new GetCheckpointAction.Request( Strings.EMPTY_ARRAY, IndicesOptions.LENIENT_EXPAND_OPEN, + null, + null, TimeValue.timeValueSeconds(5) ); assertCheckpointAction(request, response -> { @@ -170,6 +176,8 @@ public void testSingleIndexRequest() throws InterruptedException { GetCheckpointAction.Request request = new GetCheckpointAction.Request( new String[] { indexNamePattern + "0" }, IndicesOptions.LENIENT_EXPAND_OPEN, + null, + null, TimeValue.timeValueSeconds(5) ); @@ -189,6 +197,8 @@ public void testMultiIndexRequest() throws InterruptedException { GetCheckpointAction.Request request = new GetCheckpointAction.Request( testIndices, IndicesOptions.LENIENT_EXPAND_OPEN, + null, + null, TimeValue.timeValueSeconds(5) ); assertCheckpointAction(request, response -> { @@ -208,7 +218,7 @@ public void testMultiIndexRequest() throws InterruptedException { class TestTransportGetCheckpointAction extends TransportGetCheckpointAction { TestTransportGetCheckpointAction() { - super(transportService, new ActionFilters(emptySet()), indicesService, clusterService, indexNameExpressionResolver); + super(transportService, new ActionFilters(emptySet()), indicesService, clusterService, indexNameExpressionResolver, client); } @Override diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportGetCheckpointAction.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportGetCheckpointAction.java index 5acc2d4541559..675d71a5d1db9 100644 --- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportGetCheckpointAction.java +++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportGetCheckpointAction.java @@ -14,9 +14,15 @@ import org.elasticsearch.action.NoShardAvailableActionException; import org.elasticsearch.action.OriginalIndices; import org.elasticsearch.action.UnavailableShardsException; +import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.action.search.SearchShardsAction; +import org.elasticsearch.action.search.SearchShardsGroup; +import org.elasticsearch.action.search.SearchShardsRequest; +import org.elasticsearch.action.search.SearchShardsResponse; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.GroupedActionListener; import org.elasticsearch.action.support.HandledTransportAction; +import org.elasticsearch.client.internal.Client; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.node.DiscoveryNode; @@ -31,10 +37,12 @@ import org.elasticsearch.indices.IndicesService; import org.elasticsearch.tasks.CancellableTask; import org.elasticsearch.tasks.Task; +import org.elasticsearch.tasks.TaskId; import org.elasticsearch.transport.ActionNotFoundTransportException; import org.elasticsearch.transport.TransportRequestOptions; import org.elasticsearch.transport.TransportResponseHandler; import org.elasticsearch.transport.TransportService; +import org.elasticsearch.xpack.core.ClientHelper; import org.elasticsearch.xpack.core.transform.action.GetCheckpointAction; import org.elasticsearch.xpack.core.transform.action.GetCheckpointAction.Request; import org.elasticsearch.xpack.core.transform.action.GetCheckpointAction.Response; @@ -57,6 +65,7 @@ public class TransportGetCheckpointAction extends HandledTransportAction listener) { - final ClusterState state = clusterService.state(); - resolveIndicesAndGetCheckpoint(task, request, listener, state); + final ClusterState clusterState = clusterService.state(); + resolveIndicesAndGetCheckpoint(task, request, listener, clusterState); } - protected void resolveIndicesAndGetCheckpoint(Task task, Request request, ActionListener listener, final ClusterState state) { + protected void resolveIndicesAndGetCheckpoint( + Task task, + Request request, + ActionListener listener, + final ClusterState clusterState + ) { + final String nodeId = clusterState.nodes().getLocalNode().getId(); + final TaskId parentTaskId = new TaskId(nodeId, task.getId()); + // note: when security is turned on, the indices are already resolved // TODO: do a quick check and only resolve if necessary?? - String[] concreteIndices = this.indexNameExpressionResolver.concreteIndexNames(state, request); - - Map> nodesAndShards = resolveIndicesToPrimaryShards(state, concreteIndices); - - if (nodesAndShards.size() == 0) { + String[] concreteIndices = this.indexNameExpressionResolver.concreteIndexNames(clusterState, request); + Map> nodesAndShards = resolveIndicesToPrimaryShards(clusterState, concreteIndices); + if (nodesAndShards.isEmpty()) { listener.onResponse(new Response(Collections.emptyMap())); return; } - new AsyncGetCheckpointsFromNodesAction(state, task, nodesAndShards, new OriginalIndices(request), request.getTimeout(), listener) - .start(); + if (request.getQuery() == null) { // If there is no query, then there is no point in filtering + getCheckpointsFromNodes(clusterState, task, nodesAndShards, new OriginalIndices(request), request.getTimeout(), listener); + return; + } + + SearchShardsRequest searchShardsRequest = new SearchShardsRequest( + request.indices(), + SearchRequest.DEFAULT_INDICES_OPTIONS, + request.getQuery(), + null, + null, + false, + request.getCluster() + ); + searchShardsRequest.setParentTask(parentTaskId); + ClientHelper.executeAsyncWithOrigin( + client, + ClientHelper.TRANSFORM_ORIGIN, + SearchShardsAction.INSTANCE, + searchShardsRequest, + ActionListener.wrap(searchShardsResponse -> { + Map> filteredNodesAndShards = filterOutSkippedShards(nodesAndShards, searchShardsResponse); + getCheckpointsFromNodes( + clusterState, + task, + filteredNodesAndShards, + new OriginalIndices(request), + request.getTimeout(), + listener + ); + }, e -> { + // search_shards API failed so we just log the error here and continue just like there was no query + logger.atWarn().withThrowable(e).log("search_shards API failed for cluster [{}]", request.getCluster()); + logger.atTrace() + .withThrowable(e) + .log("search_shards API failed for cluster [{}], request was [{}]", request.getCluster(), searchShardsRequest); + getCheckpointsFromNodes(clusterState, task, nodesAndShards, new OriginalIndices(request), request.getTimeout(), listener); + }) + ); } - private static Map> resolveIndicesToPrimaryShards(ClusterState state, String[] concreteIndices) { + private static Map> resolveIndicesToPrimaryShards(ClusterState clusterState, String[] concreteIndices) { if (concreteIndices.length == 0) { return Collections.emptyMap(); } - final DiscoveryNodes nodes = state.nodes(); + final DiscoveryNodes nodes = clusterState.nodes(); Map> nodesAndShards = new HashMap<>(); - ShardsIterator shardsIt = state.routingTable().allShards(concreteIndices); + ShardsIterator shardsIt = clusterState.routingTable().allShards(concreteIndices); for (ShardRouting shard : shardsIt) { // only take primary shards, which should be exactly 1, this isn't strictly necessary // and we should consider taking any shard copy, but then we need another way to de-dup @@ -112,7 +166,7 @@ private static Map> resolveIndicesToPrimaryShards(ClusterSt } if (shard.assignedToNode() && nodes.get(shard.currentNodeId()) != null) { // special case: The minimum TransportVersion in the cluster is on an old version - if (state.getMinTransportVersion().before(TransportVersions.V_8_2_0)) { + if (clusterState.getMinTransportVersion().before(TransportVersions.V_8_2_0)) { throw new ActionNotFoundTransportException(GetCheckpointNodeAction.NAME); } @@ -125,111 +179,128 @@ private static Map> resolveIndicesToPrimaryShards(ClusterSt return nodesAndShards; } - protected class AsyncGetCheckpointsFromNodesAction { - private final Task task; - private final ActionListener listener; - private final Map> nodesAndShards; - private final OriginalIndices originalIndices; - private final TimeValue timeout; - private final DiscoveryNodes nodes; - private final String localNodeId; - - protected AsyncGetCheckpointsFromNodesAction( - ClusterState clusterState, - Task task, - Map> nodesAndShards, - OriginalIndices originalIndices, - TimeValue timeout, - ActionListener listener - ) { - this.task = task; - this.listener = listener; - this.nodesAndShards = nodesAndShards; - this.originalIndices = originalIndices; - this.timeout = timeout; - this.nodes = clusterState.nodes(); - this.localNodeId = clusterService.localNode().getId(); + static Map> filterOutSkippedShards( + Map> nodesAndShards, + SearchShardsResponse searchShardsResponse + ) { + Map> filteredNodesAndShards = new HashMap<>(nodesAndShards.size()); + // Create a deep copy of the given nodes and shards map. + for (Map.Entry> nodeAndShardsEntry : nodesAndShards.entrySet()) { + String node = nodeAndShardsEntry.getKey(); + Set shards = nodeAndShardsEntry.getValue(); + filteredNodesAndShards.put(node, new HashSet<>(shards)); } - - public void start() { - GroupedActionListener groupedListener = new GroupedActionListener<>( - nodesAndShards.size(), - ActionListener.wrap(responses -> listener.onResponse(mergeNodeResponses(responses)), listener::onFailure) - ); - - for (Entry> oneNodeAndItsShards : nodesAndShards.entrySet()) { - if (task instanceof CancellableTask) { - // There is no point continuing this work if the task has been cancelled. - if (((CancellableTask) task).notifyIfCancelled(listener)) { - return; + // Remove (node, shard) pairs for all the skipped shards. + for (SearchShardsGroup shardGroup : searchShardsResponse.getGroups()) { + if (shardGroup.skipped()) { + for (String allocatedNode : shardGroup.allocatedNodes()) { + Set shards = filteredNodesAndShards.get(allocatedNode); + if (shards != null) { + shards.remove(shardGroup.shardId()); + if (shards.isEmpty()) { + // Remove node if no shards were left. + filteredNodesAndShards.remove(allocatedNode); + } } } - if (localNodeId.equals(oneNodeAndItsShards.getKey())) { - TransportGetCheckpointNodeAction.getGlobalCheckpoints( - indicesService, - task, - oneNodeAndItsShards.getValue(), - timeout, - Clock.systemUTC(), - groupedListener - ); - continue; - } + } + } + return filteredNodesAndShards; + } - GetCheckpointNodeAction.Request nodeCheckpointsRequest = new GetCheckpointNodeAction.Request( - oneNodeAndItsShards.getValue(), - originalIndices, - timeout - ); - DiscoveryNode node = nodes.get(oneNodeAndItsShards.getKey()); - - // paranoia: this should not be possible using the same cluster state - if (node == null) { - listener.onFailure( - new UnavailableShardsException( - oneNodeAndItsShards.getValue().iterator().next(), - "Node not found for [{}] shards", - oneNodeAndItsShards.getValue().size() - ) - ); + private void getCheckpointsFromNodes( + ClusterState clusterState, + Task task, + Map> nodesAndShards, + OriginalIndices originalIndices, + TimeValue timeout, + ActionListener listener + ) { + if (nodesAndShards.isEmpty()) { + listener.onResponse(new Response(Map.of())); + return; + } + + final String localNodeId = clusterService.localNode().getId(); + + GroupedActionListener groupedListener = new GroupedActionListener<>( + nodesAndShards.size(), + ActionListener.wrap(responses -> listener.onResponse(mergeNodeResponses(responses)), listener::onFailure) + ); + + for (Entry> oneNodeAndItsShards : nodesAndShards.entrySet()) { + if (task instanceof CancellableTask) { + // There is no point continuing this work if the task has been cancelled. + if (((CancellableTask) task).notifyIfCancelled(listener)) { return; } - - logger.trace("get checkpoints from node {}", node); - transportService.sendChildRequest( - node, - GetCheckpointNodeAction.NAME, - nodeCheckpointsRequest, + } + if (localNodeId.equals(oneNodeAndItsShards.getKey())) { + TransportGetCheckpointNodeAction.getGlobalCheckpoints( + indicesService, task, - TransportRequestOptions.EMPTY, - new ActionListenerResponseHandler<>( - groupedListener, - GetCheckpointNodeAction.Response::new, - TransportResponseHandler.TRANSPORT_WORKER + oneNodeAndItsShards.getValue(), + timeout, + Clock.systemUTC(), + groupedListener + ); + continue; + } + + DiscoveryNodes nodes = clusterState.nodes(); + DiscoveryNode node = nodes.get(oneNodeAndItsShards.getKey()); + + // paranoia: this should not be possible using the same cluster state + if (node == null) { + listener.onFailure( + new UnavailableShardsException( + oneNodeAndItsShards.getValue().iterator().next(), + "Node not found for [{}] shards", + oneNodeAndItsShards.getValue().size() ) ); + return; } + + logger.trace("get checkpoints from node {}", node); + GetCheckpointNodeAction.Request nodeCheckpointsRequest = new GetCheckpointNodeAction.Request( + oneNodeAndItsShards.getValue(), + originalIndices, + timeout + ); + transportService.sendChildRequest( + node, + GetCheckpointNodeAction.NAME, + nodeCheckpointsRequest, + task, + TransportRequestOptions.EMPTY, + new ActionListenerResponseHandler<>( + groupedListener, + GetCheckpointNodeAction.Response::new, + TransportResponseHandler.TRANSPORT_WORKER + ) + ); } + } - private static Response mergeNodeResponses(Collection responses) { - // the final list should be ordered by key - Map checkpointsByIndexReduced = new TreeMap<>(); - - // merge the node responses - for (GetCheckpointNodeAction.Response response : responses) { - response.getCheckpoints().forEach((index, checkpoint) -> { - if (checkpointsByIndexReduced.containsKey(index)) { - long[] shardCheckpoints = checkpointsByIndexReduced.get(index); - for (int i = 0; i < checkpoint.length; ++i) { - shardCheckpoints[i] = Math.max(shardCheckpoints[i], checkpoint[i]); - } - } else { - checkpointsByIndexReduced.put(index, checkpoint); - } - }); - } + private static Response mergeNodeResponses(Collection responses) { + // the final list should be ordered by key + Map checkpointsByIndexReduced = new TreeMap<>(); - return new Response(checkpointsByIndexReduced); + // merge the node responses + for (GetCheckpointNodeAction.Response response : responses) { + response.getCheckpoints().forEach((index, checkpoint) -> { + if (checkpointsByIndexReduced.containsKey(index)) { + long[] shardCheckpoints = checkpointsByIndexReduced.get(index); + for (int i = 0; i < checkpoint.length; ++i) { + shardCheckpoints[i] = Math.max(shardCheckpoints[i], checkpoint[i]); + } + } else { + checkpointsByIndexReduced.put(index, checkpoint); + } + }); } + + return new Response(checkpointsByIndexReduced); } } diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportGetTransformStatsAction.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportGetTransformStatsAction.java index 13abc427460be..f7e60b13b50a6 100644 --- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportGetTransformStatsAction.java +++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportGetTransformStatsAction.java @@ -121,14 +121,15 @@ protected void taskOperation( TransformTask transformTask, ActionListener listener ) { - // Little extra insurance, make sure we only return transforms that aren't cancelled ClusterState clusterState = clusterService.state(); String nodeId = clusterState.nodes().getLocalNode().getId(); final TaskId parentTaskId = new TaskId(nodeId, actionTask.getId()); + // If the _stats request is cancelled there is no point in continuing this work on the task level if (actionTask.notifyIfCancelled(listener)) { return; } + // Little extra insurance, make sure we only return transforms that aren't cancelled if (transformTask.isCancelled()) { listener.onResponse(new Response(Collections.emptyList())); return; diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/checkpoint/DefaultCheckpointProvider.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/checkpoint/DefaultCheckpointProvider.java index aa1332b95fe84..b9b7d9d8477cb 100644 --- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/checkpoint/DefaultCheckpointProvider.java +++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/checkpoint/DefaultCheckpointProvider.java @@ -23,6 +23,7 @@ import org.elasticsearch.common.util.set.Sets; import org.elasticsearch.core.Strings; import org.elasticsearch.core.TimeValue; +import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.transport.ActionNotFoundTransportException; import org.elasticsearch.transport.RemoteClusterService; import org.elasticsearch.xpack.core.ClientHelper; @@ -134,14 +135,16 @@ protected void getIndexCheckpoints(TimeValue timeout, ActionListener> remoteIndex : resolvedIndexes.getRemoteIndicesPerClusterAlias().entrySet()) { + String cluster = remoteIndex.getKey(); ParentTaskAssigningClient remoteClient = new ParentTaskAssigningClient( - client.getRemoteClusterClient(remoteIndex.getKey(), EsExecutors.DIRECT_EXECUTOR_SERVICE), + client.getRemoteClusterClient(cluster, EsExecutors.DIRECT_EXECUTOR_SERVICE), client.getParentTask() ); getCheckpointsFromOneCluster( @@ -149,7 +152,8 @@ protected void getIndexCheckpoints(TimeValue timeout, ActionListener headers, String[] indices, + QueryBuilder query, String cluster, ActionListener> listener ) { if (fallbackToBWC.contains(cluster)) { getCheckpointsFromOneClusterBWC(client, timeout, headers, indices, cluster, listener); } else { - getCheckpointsFromOneClusterV2(client, timeout, headers, indices, cluster, ActionListener.wrap(response -> { + getCheckpointsFromOneClusterV2(client, timeout, headers, indices, query, cluster, ActionListener.wrap(response -> { logger.debug( "[{}] Successfully retrieved checkpoints from cluster [{}] using transform checkpoint API", transformConfig.getId(), @@ -200,12 +205,15 @@ private static void getCheckpointsFromOneClusterV2( TimeValue timeout, Map headers, String[] indices, + QueryBuilder query, String cluster, ActionListener> listener ) { GetCheckpointAction.Request getCheckpointRequest = new GetCheckpointAction.Request( indices, IndicesOptions.LENIENT_EXPAND_OPEN, + query, + cluster, timeout ); ActionListener checkpointListener; @@ -239,7 +247,6 @@ private static void getCheckpointsFromOneClusterV2( getCheckpointRequest, checkpointListener ); - } /** diff --git a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/action/TransportGetCheckpointActionTests.java b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/action/TransportGetCheckpointActionTests.java new file mode 100644 index 0000000000000..0d2d9619aca68 --- /dev/null +++ b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/action/TransportGetCheckpointActionTests.java @@ -0,0 +1,134 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.transform.action; + +import org.elasticsearch.action.search.SearchShardsGroup; +import org.elasticsearch.action.search.SearchShardsResponse; +import org.elasticsearch.index.Index; +import org.elasticsearch.index.shard.ShardId; +import org.elasticsearch.test.ESTestCase; + +import java.util.List; +import java.util.Map; +import java.util.Set; + +import static org.hamcrest.Matchers.anEmptyMap; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; + +public class TransportGetCheckpointActionTests extends ESTestCase { + + private static final String NODE_0 = "node-0"; + private static final String NODE_1 = "node-1"; + private static final String NODE_2 = "node-2"; + private static final Index INDEX_A = new Index("my-index-A", "A"); + private static final Index INDEX_B = new Index("my-index-B", "B"); + private static final Index INDEX_C = new Index("my-index-C", "C"); + private static final ShardId SHARD_A_0 = new ShardId(INDEX_A, 0); + private static final ShardId SHARD_A_1 = new ShardId(INDEX_A, 1); + private static final ShardId SHARD_B_0 = new ShardId(INDEX_B, 0); + private static final ShardId SHARD_B_1 = new ShardId(INDEX_B, 1); + + private static final Map> NODES_AND_SHARDS = Map.of( + NODE_0, + Set.of(SHARD_A_0, SHARD_A_1, SHARD_B_0, SHARD_B_1), + NODE_1, + Set.of(SHARD_A_0, SHARD_A_1, SHARD_B_0, SHARD_B_1), + NODE_2, + Set.of(SHARD_A_0, SHARD_A_1, SHARD_B_0, SHARD_B_1) + ); + + public void testFilterOutSkippedShards_EmptyNodesAndShards() { + SearchShardsResponse searchShardsResponse = new SearchShardsResponse( + Set.of( + new SearchShardsGroup(SHARD_A_0, List.of(NODE_0, NODE_1), true), + new SearchShardsGroup(SHARD_B_0, List.of(NODE_1, NODE_2), false), + new SearchShardsGroup(SHARD_B_1, List.of(NODE_0, NODE_2), true) + ), + Set.of(), + Map.of() + ); + Map> filteredNodesAndShards = TransportGetCheckpointAction.filterOutSkippedShards( + Map.of(), + searchShardsResponse + ); + assertThat(filteredNodesAndShards, is(anEmptyMap())); + } + + public void testFilterOutSkippedShards_EmptySearchShardsResponse() { + SearchShardsResponse searchShardsResponse = new SearchShardsResponse(Set.of(), Set.of(), Map.of()); + Map> filteredNodesAndShards = TransportGetCheckpointAction.filterOutSkippedShards( + NODES_AND_SHARDS, + searchShardsResponse + ); + assertThat(filteredNodesAndShards, is(equalTo(NODES_AND_SHARDS))); + } + + public void testFilterOutSkippedShards_SomeNodesEmptyAfterFiltering() { + SearchShardsResponse searchShardsResponse = new SearchShardsResponse( + Set.of( + new SearchShardsGroup(SHARD_A_0, List.of(NODE_0, NODE_2), true), + new SearchShardsGroup(SHARD_A_1, List.of(NODE_0, NODE_2), true), + new SearchShardsGroup(SHARD_B_0, List.of(NODE_0, NODE_2), true), + new SearchShardsGroup(SHARD_B_1, List.of(NODE_0, NODE_2), true) + ), + Set.of(), + Map.of() + ); + Map> filteredNodesAndShards = TransportGetCheckpointAction.filterOutSkippedShards( + NODES_AND_SHARDS, + searchShardsResponse + ); + Map> expectedFilteredNodesAndShards = Map.of(NODE_1, Set.of(SHARD_A_0, SHARD_A_1, SHARD_B_0, SHARD_B_1)); + assertThat(filteredNodesAndShards, is(equalTo(expectedFilteredNodesAndShards))); + } + + public void testFilterOutSkippedShards_AllNodesEmptyAfterFiltering() { + SearchShardsResponse searchShardsResponse = new SearchShardsResponse( + Set.of( + new SearchShardsGroup(SHARD_A_0, List.of(NODE_0, NODE_1, NODE_2), true), + new SearchShardsGroup(SHARD_A_1, List.of(NODE_0, NODE_1, NODE_2), true), + new SearchShardsGroup(SHARD_B_0, List.of(NODE_0, NODE_1, NODE_2), true), + new SearchShardsGroup(SHARD_B_1, List.of(NODE_0, NODE_1, NODE_2), true) + ), + Set.of(), + Map.of() + ); + Map> filteredNodesAndShards = TransportGetCheckpointAction.filterOutSkippedShards( + NODES_AND_SHARDS, + searchShardsResponse + ); + assertThat(filteredNodesAndShards, is(equalTo(Map.of()))); + } + + public void testFilterOutSkippedShards() { + SearchShardsResponse searchShardsResponse = new SearchShardsResponse( + Set.of( + new SearchShardsGroup(SHARD_A_0, List.of(NODE_0, NODE_1), true), + new SearchShardsGroup(SHARD_B_0, List.of(NODE_1, NODE_2), false), + new SearchShardsGroup(SHARD_B_1, List.of(NODE_0, NODE_2), true), + new SearchShardsGroup(new ShardId(INDEX_C, 0), List.of(NODE_0, NODE_1, NODE_2), true) + ), + Set.of(), + Map.of() + ); + Map> filteredNodesAndShards = TransportGetCheckpointAction.filterOutSkippedShards( + NODES_AND_SHARDS, + searchShardsResponse + ); + Map> expectedFilteredNodesAndShards = Map.of( + NODE_0, + Set.of(SHARD_A_1, SHARD_B_0), + NODE_1, + Set.of(SHARD_A_1, SHARD_B_0, SHARD_B_1), + NODE_2, + Set.of(SHARD_A_0, SHARD_A_1, SHARD_B_0) + ); + assertThat(filteredNodesAndShards, is(equalTo(expectedFilteredNodesAndShards))); + } +} From 4106ad274dc64fab48f61ce6392202a1e76f0634 Mon Sep 17 00:00:00 2001 From: Nhat Nguyen Date: Mon, 20 Nov 2023 07:07:28 -0800 Subject: [PATCH 23/28] Fix AsyncOperatorTests#testFailure (#102335) There is a bug in the test where we check the failed flag immediately after the Driver starts, instead of waiting until the Driver has completed. Also, we should check for failure in the AsyncOperator. Closes #102264 --- .../compute/operator/AsyncOperator.java | 1 + .../compute/operator/AsyncOperatorTests.java | 13 +++++++------ 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/AsyncOperator.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/AsyncOperator.java index 8eea50226253b..f2011d1cdb987 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/AsyncOperator.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/AsyncOperator.java @@ -166,6 +166,7 @@ public void finish() { @Override public boolean isFinished() { + checkFailure(); return finished && checkpoint.getPersistedCheckpoint() == checkpoint.getMaxSeqNo(); } diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/AsyncOperatorTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/AsyncOperatorTests.java index b35dc8b7b2e80..00b046abdca24 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/AsyncOperatorTests.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/AsyncOperatorTests.java @@ -185,7 +185,6 @@ protected void doClose() { operator.close(); } - @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/102264") public void testFailure() throws Exception { DriverContext driverContext = driverContext(); final SequenceLongBlockSourceOperator sourceOperator = new SequenceLongBlockSourceOperator( @@ -226,15 +225,17 @@ protected void doClose() { PlainActionFuture future = new PlainActionFuture<>(); Driver driver = new Driver(driverContext, sourceOperator, List.of(asyncOperator), outputOperator, () -> {}); Driver.start(threadPool.getThreadContext(), threadPool.executor(ESQL_TEST_EXECUTOR), driver, between(1, 1000), future); - assertBusy(() -> { - assertTrue(asyncOperator.isFinished()); - assertTrue(future.isDone()); - }); + assertBusy(() -> assertTrue(future.isDone())); if (failed.get()) { ElasticsearchException error = expectThrows(ElasticsearchException.class, future::actionGet); assertThat(error.getMessage(), containsString("simulated")); + error = expectThrows(ElasticsearchException.class, asyncOperator::isFinished); + assertThat(error.getMessage(), containsString("simulated")); + error = expectThrows(ElasticsearchException.class, asyncOperator::getOutput); + assertThat(error.getMessage(), containsString("simulated")); } else { - future.actionGet(); + assertTrue(asyncOperator.isFinished()); + assertNull(asyncOperator.getOutput()); } } From 588eabe185ad319c0268a13480465966cef058cd Mon Sep 17 00:00:00 2001 From: Nhat Nguyen Date: Mon, 20 Nov 2023 07:44:22 -0800 Subject: [PATCH 24/28] Avoid spawn new nodes in EsqlActionIT (#102363) We don't need to launch new nodes in these tests if we already have at least two data nodes. --- .../xpack/esql/action/EsqlActionIT.java | 28 +++++++++++++++---- 1 file changed, 23 insertions(+), 5 deletions(-) diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlActionIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlActionIT.java index 06fd9bd469b84..a141db7037263 100644 --- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlActionIT.java +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlActionIT.java @@ -16,6 +16,7 @@ import org.elasticsearch.action.support.WriteRequest; import org.elasticsearch.client.internal.ClusterAdminClient; import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.Index; @@ -1242,8 +1243,15 @@ public void testFilterNestedFields() { } public void testStatsNestFields() { - String node1 = internalCluster().startDataOnlyNode(); - String node2 = internalCluster().startDataOnlyNode(); + final String node1, node2; + if (randomBoolean()) { + internalCluster().ensureAtLeastNumDataNodes(2); + node1 = randomDataNode().getName(); + node2 = randomValueOtherThan(node1, () -> randomDataNode().getName()); + } else { + node1 = randomDataNode().getName(); + node2 = randomDataNode().getName(); + } assertAcked( client().admin() .indices() @@ -1276,8 +1284,15 @@ public void testStatsNestFields() { } public void testStatsMissingFields() { - String node1 = internalCluster().startDataOnlyNode(); - String node2 = internalCluster().startDataOnlyNode(); + final String node1, node2; + if (randomBoolean()) { + internalCluster().ensureAtLeastNumDataNodes(2); + node1 = randomDataNode().getName(); + node2 = randomValueOtherThan(node1, () -> randomDataNode().getName()); + } else { + node1 = randomDataNode().getName(); + node2 = randomDataNode().getName(); + } assertAcked( client().admin() .indices() @@ -1292,7 +1307,6 @@ public void testStatsMissingFields() { .setSettings(Settings.builder().put("index.routing.allocation.require._name", node2)) .setMapping("bar_int", "type=integer", "bar_long", "type=long", "bar_float", "type=float", "bar_double", "type=double") ); - var fields = List.of("foo_int", "foo_long", "foo_float", "foo_double"); var functions = List.of("sum", "count", "avg", "count_distinct"); for (String field : fields) { @@ -1510,4 +1524,8 @@ private void clearPersistentSettings(Setting... settings) { var clearSettingsRequest = new ClusterUpdateSettingsRequest().persistentSettings(clearedSettings.build()); admin().cluster().updateSettings(clearSettingsRequest).actionGet(); } + + private DiscoveryNode randomDataNode() { + return randomFrom(clusterService().state().nodes().getDataNodes().values()); + } } From 7cc711ad25898d1659662a464df84175606bb289 Mon Sep 17 00:00:00 2001 From: David Turner Date: Mon, 20 Nov 2023 17:22:19 +0000 Subject: [PATCH 25/28] Add test logging to S3BlobStoreRepositoryTests#testMetrics (#102387) Relates #101608 --- .../repositories/s3/S3BlobStoreRepositoryTests.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/modules/repository-s3/src/internalClusterTest/java/org/elasticsearch/repositories/s3/S3BlobStoreRepositoryTests.java b/modules/repository-s3/src/internalClusterTest/java/org/elasticsearch/repositories/s3/S3BlobStoreRepositoryTests.java index aee61361ebd10..67b0202ac1eae 100644 --- a/modules/repository-s3/src/internalClusterTest/java/org/elasticsearch/repositories/s3/S3BlobStoreRepositoryTests.java +++ b/modules/repository-s3/src/internalClusterTest/java/org/elasticsearch/repositories/s3/S3BlobStoreRepositoryTests.java @@ -57,7 +57,7 @@ import org.elasticsearch.telemetry.metric.MeterRegistry; import org.elasticsearch.test.BackgroundIndexer; import org.elasticsearch.test.ESIntegTestCase; -import org.elasticsearch.test.junit.annotations.TestLogging; +import org.elasticsearch.test.junit.annotations.TestIssueLogging; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.xcontent.NamedXContentRegistry; import org.elasticsearch.xcontent.XContentFactory; @@ -179,7 +179,7 @@ protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) { } @Override - @TestLogging(reason = "Enable request logging to debug #88841", value = "com.amazonaws.request:DEBUG") + @TestIssueLogging(issueUrl = "https://github.com/elastic/elasticsearch/issues/88841", value = "com.amazonaws.request:DEBUG") public void testRequestStats() throws Exception { super.testRequestStats(); } @@ -225,6 +225,7 @@ public void testAbortRequestStats() throws Exception { assertEquals(assertionErrorMsg, mockCalls, sdkRequestCounts); } + @TestIssueLogging(issueUrl = "https://github.com/elastic/elasticsearch/issues/101608", value = "com.amazonaws.request:DEBUG") public void testMetrics() throws Exception { // Create the repository and perform some activities final String repository = createRepository(randomRepositoryName()); From c0782c1289059cbbebc8171de3e7688e604d9b14 Mon Sep 17 00:00:00 2001 From: Nhat Nguyen Date: Mon, 20 Nov 2023 09:45:16 -0800 Subject: [PATCH 26/28] Move ESQL yaml tests to core (#102340) This PR moves ESQL YAML tests to the core so that we can run them with different modules (e.g., the mixed cluster tests) and even with serverless in the future. YAML tests for other plugins are also in the core. --- x-pack/plugin/esql/qa/server/single-node/build.gradle | 7 +++---- .../resources/rest-api-spec/test/esql}/100_bug_fix.yml | 9 +++++++-- .../resources/rest-api-spec/test/esql}/10_basic.yml | 2 ++ .../resources/rest-api-spec/test/esql}/20_aggs.yml | 2 ++ .../resources/rest-api-spec/test/esql}/30_types.yml | 2 ++ .../resources/rest-api-spec/test/esql}/40_tsdb.yml | 2 ++ .../rest-api-spec/test/esql}/40_unsupported_types.yml | 2 ++ .../rest-api-spec/test/esql}/45_non_tsdb_counter.yml | 2 ++ .../rest-api-spec/test/esql}/50_index_patterns.yml | 2 ++ .../resources/rest-api-spec/test/esql}/60_enrich.yml | 7 +++++++ .../resources/rest-api-spec/test/esql}/60_usage.yml | 4 ++++ .../resources/rest-api-spec/test/esql}/61_enrich_ip.yml | 7 +++++++ .../resources/rest-api-spec/test/esql}/70_locale.yml | 2 ++ .../resources/rest-api-spec/test/esql}/80_text.yml | 2 ++ .../rest-api-spec/test/esql}/90_non_indexed.yml | 2 ++ 15 files changed, 48 insertions(+), 6 deletions(-) rename x-pack/plugin/{esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test => src/yamlRestTest/resources/rest-api-spec/test/esql}/100_bug_fix.yml (94%) rename x-pack/plugin/{esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test => src/yamlRestTest/resources/rest-api-spec/test/esql}/10_basic.yml (99%) rename x-pack/plugin/{esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test => src/yamlRestTest/resources/rest-api-spec/test/esql}/20_aggs.yml (99%) rename x-pack/plugin/{esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test => src/yamlRestTest/resources/rest-api-spec/test/esql}/30_types.yml (99%) rename x-pack/plugin/{esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test => src/yamlRestTest/resources/rest-api-spec/test/esql}/40_tsdb.yml (99%) rename x-pack/plugin/{esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test => src/yamlRestTest/resources/rest-api-spec/test/esql}/40_unsupported_types.yml (99%) rename x-pack/plugin/{esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test => src/yamlRestTest/resources/rest-api-spec/test/esql}/45_non_tsdb_counter.yml (98%) rename x-pack/plugin/{esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test => src/yamlRestTest/resources/rest-api-spec/test/esql}/50_index_patterns.yml (99%) rename x-pack/plugin/{esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test => src/yamlRestTest/resources/rest-api-spec/test/esql}/60_enrich.yml (95%) rename x-pack/plugin/{esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test => src/yamlRestTest/resources/rest-api-spec/test/esql}/60_usage.yml (96%) rename x-pack/plugin/{esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test => src/yamlRestTest/resources/rest-api-spec/test/esql}/61_enrich_ip.yml (94%) rename x-pack/plugin/{esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test => src/yamlRestTest/resources/rest-api-spec/test/esql}/70_locale.yml (96%) rename x-pack/plugin/{esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test => src/yamlRestTest/resources/rest-api-spec/test/esql}/80_text.yml (99%) rename x-pack/plugin/{esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test => src/yamlRestTest/resources/rest-api-spec/test/esql}/90_non_indexed.yml (97%) diff --git a/x-pack/plugin/esql/qa/server/single-node/build.gradle b/x-pack/plugin/esql/qa/server/single-node/build.gradle index 3131b4176ee25..2d430965efb21 100644 --- a/x-pack/plugin/esql/qa/server/single-node/build.gradle +++ b/x-pack/plugin/esql/qa/server/single-node/build.gradle @@ -9,10 +9,9 @@ restResources { restApi { include '_common', 'bulk', 'get', 'indices', 'esql', 'xpack', 'enrich', 'cluster' } -} - -artifacts { - restXpackTests(new File(projectDir, "src/yamlRestTest/resources/rest-api-spec/test")) + restTests { + includeXpack 'esql' + } } testClusters.configureEach { diff --git a/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/100_bug_fix.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/100_bug_fix.yml similarity index 94% rename from x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/100_bug_fix.yml rename to x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/100_bug_fix.yml index d5f5bee46f50a..1876d1a6d3881 100644 --- a/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/100_bug_fix.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/100_bug_fix.yml @@ -1,6 +1,8 @@ --- -"Bug fix https://github.com/elastic/elasticsearch/issues/99472": +"Coalesce and to_ip functions": - skip: + version: " - 8.11.99" + reason: "fixes in 8.12 or later" features: warnings - do: bulk: @@ -54,7 +56,10 @@ - match: { values.1: [ 20, null, "255.255.255.255", "255.255.255.255"] } --- -"Bug fix https://github.com/elastic/elasticsearch/issues/101489": +"unsupported and invalid mapped fields": + - skip: + version: " - 8.11.99" + reason: "fixes in 8.12 or later" - do: indices.create: index: index1 diff --git a/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/10_basic.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/10_basic.yml similarity index 99% rename from x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/10_basic.yml rename to x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/10_basic.yml index a3b2de27bcb5b..e15372bc3088e 100644 --- a/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/10_basic.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/10_basic.yml @@ -1,6 +1,8 @@ --- setup: - skip: + version: " - 8.10.99" + reason: "ESQL is available in 8.11+" features: allowed_warnings_regex - do: indices.create: diff --git a/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/20_aggs.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/20_aggs.yml similarity index 99% rename from x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/20_aggs.yml rename to x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/20_aggs.yml index e94cb6ccd8e3c..4019b3a303345 100644 --- a/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/20_aggs.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/20_aggs.yml @@ -1,6 +1,8 @@ --- setup: - skip: + version: " - 8.10.99" + reason: "ESQL is available in 8.11+" features: warnings - do: indices.create: diff --git a/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/30_types.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/30_types.yml similarity index 99% rename from x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/30_types.yml rename to x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/30_types.yml index bf159455d00ca..406ae169872a2 100644 --- a/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/30_types.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/30_types.yml @@ -1,6 +1,8 @@ --- setup: - skip: + version: " - 8.10.99" + reason: "ESQL is available in 8.11+" features: warnings --- diff --git a/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/40_tsdb.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/40_tsdb.yml similarity index 99% rename from x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/40_tsdb.yml rename to x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/40_tsdb.yml index 6a90fc5a7b8f8..1f9dc67dbfbbd 100644 --- a/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/40_tsdb.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/40_tsdb.yml @@ -1,5 +1,7 @@ setup: - skip: + version: " - 8.10.99" + reason: "ESQL is available in 8.11+" features: allowed_warnings_regex - do: indices.create: diff --git a/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/40_unsupported_types.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/40_unsupported_types.yml similarity index 99% rename from x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/40_unsupported_types.yml rename to x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/40_unsupported_types.yml index c06456f7f127d..be5b43433983e 100644 --- a/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/40_unsupported_types.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/40_unsupported_types.yml @@ -1,5 +1,7 @@ setup: - skip: + version: " - 8.10.99" + reason: "ESQL is available in 8.11+" features: allowed_warnings_regex - do: indices.create: diff --git a/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/45_non_tsdb_counter.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/45_non_tsdb_counter.yml similarity index 98% rename from x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/45_non_tsdb_counter.yml rename to x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/45_non_tsdb_counter.yml index beb7200f01230..13a88d0c2f79f 100644 --- a/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/45_non_tsdb_counter.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/45_non_tsdb_counter.yml @@ -1,5 +1,7 @@ setup: - skip: + version: " - 8.10.99" + reason: "ESQL is available in 8.11+" features: allowed_warnings_regex - do: indices.create: diff --git a/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/50_index_patterns.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/50_index_patterns.yml similarity index 99% rename from x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/50_index_patterns.yml rename to x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/50_index_patterns.yml index 2098b9ee60d1e..38023b7791709 100644 --- a/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/50_index_patterns.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/50_index_patterns.yml @@ -1,5 +1,7 @@ setup: - skip: + version: " - 8.10.99" + reason: "ESQL is available in 8.11+" features: allowed_warnings_regex --- diff --git a/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/60_enrich.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/60_enrich.yml similarity index 95% rename from x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/60_enrich.yml rename to x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/60_enrich.yml index 84d8682508733..1673453824584 100644 --- a/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/60_enrich.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/60_enrich.yml @@ -1,6 +1,8 @@ --- setup: - skip: + version: " - 8.10.99" + reason: "ESQL is available in 8.11+" features: allowed_warnings_regex - do: indices.create: @@ -127,3 +129,8 @@ setup: - match: { values.1: [ "Bob", "nyc", "USA" ] } - match: { values.2: [ "Denise", "sgn", null ] } - match: { values.3: [ "Mario", "rom", "Italy" ] } + + - do: + enrich.delete_policy: + name: cities_policy + - is_true: acknowledged diff --git a/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/60_usage.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/60_usage.yml similarity index 96% rename from x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/60_usage.yml rename to x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/60_usage.yml index d7998651540d8..ad46a3c2d9c3e 100644 --- a/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/60_usage.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/60_usage.yml @@ -1,5 +1,9 @@ --- setup: + - skip: + version: " - 8.10.99" + reason: "ESQL is available in 8.11+" + - do: indices.create: index: test diff --git a/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/61_enrich_ip.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/61_enrich_ip.yml similarity index 94% rename from x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/61_enrich_ip.yml rename to x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/61_enrich_ip.yml index bd89af2fd3f79..0d49f169fc4b2 100644 --- a/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/61_enrich_ip.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/61_enrich_ip.yml @@ -1,6 +1,8 @@ --- setup: - skip: + version: " - 8.10.99" + reason: "ESQL is available in 8.11+" features: allowed_warnings_regex - do: indices.create: @@ -95,3 +97,8 @@ setup: - match: { values.1: [ [ "10.100.0.21", "10.101.0.107" ], [ "Production", "QA" ], [ "OPS","Engineering" ], "sending messages" ] } - match: { values.2: [ "10.101.0.107" , "QA", "Engineering", "network disconnected" ] } - match: { values.3: [ "13.101.0.114" , null, null, "authentication failed" ] } + + - do: + enrich.delete_policy: + name: networks-policy + - is_true: acknowledged diff --git a/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/70_locale.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/70_locale.yml similarity index 96% rename from x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/70_locale.yml rename to x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/70_locale.yml index a77e0569668de..bcae5e7cf24a2 100644 --- a/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/70_locale.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/70_locale.yml @@ -1,6 +1,8 @@ --- setup: - skip: + version: " - 8.10.99" + reason: "ESQL is available in 8.11+" features: allowed_warnings_regex - do: indices.create: diff --git a/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/80_text.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/80_text.yml similarity index 99% rename from x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/80_text.yml rename to x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/80_text.yml index d6d20fa0a0aee..cef7f88506de8 100644 --- a/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/80_text.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/80_text.yml @@ -1,6 +1,8 @@ --- setup: - skip: + version: " - 8.10.99" + reason: "ESQL is available in 8.11+" features: allowed_warnings_regex - do: indices.create: diff --git a/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/90_non_indexed.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/90_non_indexed.yml similarity index 97% rename from x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/90_non_indexed.yml rename to x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/90_non_indexed.yml index 9138a9454c571..c6124e7f75e96 100644 --- a/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/resources/rest-api-spec/test/90_non_indexed.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/90_non_indexed.yml @@ -1,5 +1,7 @@ setup: - skip: + version: " - 8.11.99" + reason: "extracting non-indexed fields available in 8.12+" features: allowed_warnings - do: indices.create: From 86e05996c171b8068104d5dcb130149cf7d56b29 Mon Sep 17 00:00:00 2001 From: David Turner Date: Mon, 20 Nov 2023 18:11:04 +0000 Subject: [PATCH 27/28] Add descriptions to assertions in testMetrics (#102386) In #101608 we saw one of these assertions fail, but it's impossible to know which one without some more details. This commit adds descriptions to the assertions in the loop. --- .../repositories/s3/S3BlobStoreRepositoryTests.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/modules/repository-s3/src/internalClusterTest/java/org/elasticsearch/repositories/s3/S3BlobStoreRepositoryTests.java b/modules/repository-s3/src/internalClusterTest/java/org/elasticsearch/repositories/s3/S3BlobStoreRepositoryTests.java index 67b0202ac1eae..7f46440647a54 100644 --- a/modules/repository-s3/src/internalClusterTest/java/org/elasticsearch/repositories/s3/S3BlobStoreRepositoryTests.java +++ b/modules/repository-s3/src/internalClusterTest/java/org/elasticsearch/repositories/s3/S3BlobStoreRepositoryTests.java @@ -282,8 +282,12 @@ public void testMetrics() throws Exception { operation, OperationPurpose.parse((String) metric.attributes().get("purpose")) ); - assertThat(statsCollectors, hasKey(statsKey)); - assertThat(metric.getLong(), equalTo(statsCollectors.get(statsKey).counter.sum())); + assertThat(nodeName + "/" + statsKey + " exists", statsCollectors, hasKey(statsKey)); + assertThat( + nodeName + "/" + statsKey + " has correct sum", + metric.getLong(), + equalTo(statsCollectors.get(statsKey).counter.sum()) + ); aggregatedMetrics.compute(operation.getKey(), (k, v) -> v == null ? metric.getLong() : v + metric.getLong()); }); From 36a2f9bc43900a77e6d88c5c22b7664be5891104 Mon Sep 17 00:00:00 2001 From: David Turner Date: Mon, 20 Nov 2023 18:44:21 +0000 Subject: [PATCH 28/28] Remove exception mangling around LoggingTaskListener (#102369) Replaces `ListenableActionFuture` with `SubscribableListener` at both call sites. --- .../reindex/AbstractBaseReindexRestHandler.java | 8 ++++---- .../rest/action/admin/indices/RestForceMergeAction.java | 8 ++++---- .../java/org/elasticsearch/tasks/LoggingTaskListener.java | 1 - 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/modules/reindex/src/main/java/org/elasticsearch/reindex/AbstractBaseReindexRestHandler.java b/modules/reindex/src/main/java/org/elasticsearch/reindex/AbstractBaseReindexRestHandler.java index 952dd0585e7ba..8e7fab68ac697 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/reindex/AbstractBaseReindexRestHandler.java +++ b/modules/reindex/src/main/java/org/elasticsearch/reindex/AbstractBaseReindexRestHandler.java @@ -11,7 +11,7 @@ import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.ActionType; import org.elasticsearch.action.support.ActiveShardCount; -import org.elasticsearch.action.support.ListenableActionFuture; +import org.elasticsearch.action.support.SubscribableListener; import org.elasticsearch.client.internal.node.NodeClient; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.index.reindex.AbstractBulkByScrollRequest; @@ -64,9 +64,9 @@ protected RestChannelConsumer doPrepareRequest(RestRequest request, NodeClient c if (validationException != null) { throw validationException; } - final var responseFuture = new ListenableActionFuture(); - final var task = client.executeLocally(action, internal, responseFuture); - responseFuture.addListener(new LoggingTaskListener<>(task)); + final var responseListener = new SubscribableListener(); + final var task = client.executeLocally(action, internal, responseListener); + responseListener.addListener(new LoggingTaskListener<>(task)); return sendTask(client.getLocalNodeId(), task); } diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestForceMergeAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestForceMergeAction.java index a04e23f289379..4c9ac8fcb9a3c 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestForceMergeAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestForceMergeAction.java @@ -13,7 +13,7 @@ import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeRequest; import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeResponse; import org.elasticsearch.action.support.IndicesOptions; -import org.elasticsearch.action.support.ListenableActionFuture; +import org.elasticsearch.action.support.SubscribableListener; import org.elasticsearch.client.internal.node.NodeClient; import org.elasticsearch.common.Strings; import org.elasticsearch.rest.BaseRestHandler; @@ -65,9 +65,9 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC if (validationException != null) { throw validationException; } - final var responseFuture = new ListenableActionFuture(); - final var task = client.executeLocally(ForceMergeAction.INSTANCE, mergeRequest, responseFuture); - responseFuture.addListener(new LoggingTaskListener<>(task)); + final var responseListener = new SubscribableListener(); + final var task = client.executeLocally(ForceMergeAction.INSTANCE, mergeRequest, responseListener); + responseListener.addListener(new LoggingTaskListener<>(task)); return sendTask(client.getLocalNodeId(), task); } } diff --git a/server/src/main/java/org/elasticsearch/tasks/LoggingTaskListener.java b/server/src/main/java/org/elasticsearch/tasks/LoggingTaskListener.java index c99194d933131..63e17bd62f8ee 100644 --- a/server/src/main/java/org/elasticsearch/tasks/LoggingTaskListener.java +++ b/server/src/main/java/org/elasticsearch/tasks/LoggingTaskListener.java @@ -31,7 +31,6 @@ public LoggingTaskListener(Task task) { @Override public void onResponse(Response response) { logger.info("{} finished with response {}", task.getId(), response); - } @Override