Merge branch 'main' into rcs2-rebuild-connections

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchBuildCompletePlugin.java

@@ -134,7 +134,7 @@ public void execute(BuildFinishedFlowAction.Parameters parameters) throws FileNo
             }
             uploadFile.getParentFile().mkdirs();
             createBuildArchiveTar(parameters.getFilteredFiles().get(), parameters.getProjectDir().get(), uploadFile);
-            if (uploadFile.exists() && System.getenv("BUILDKITE").equals("true")) {
+            if (uploadFile.exists() && "true".equals(System.getenv("BUILDKITE"))) {
                 String uploadFilePath = "build/" + uploadFile.getName();
                 try {
                     System.out.println("Uploading buildkite artifact: " + uploadFilePath + "...");

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchTestBasePlugin.java

@@ -108,10 +108,7 @@ public void execute(Task t) {
                 "--add-opens=java.base/java.nio.file=ALL-UNNAMED",
                 "--add-opens=java.base/java.time=ALL-UNNAMED",
                 "--add-opens=java.management/java.lang.management=ALL-UNNAMED",
-                "-XX:+HeapDumpOnOutOfMemoryError",
-                // REMOVE once bumped to a JDK greater than 21.0.1, https://github.com/elastic/elasticsearch/issues/103004
-                "-XX:CompileCommand=exclude,org.apache.lucene.util.MSBRadixSorter::computeCommonPrefixLengthAndBuildHistogram",
-                "-XX:CompileCommand=exclude,org.apache.lucene.util.RadixSelector::computeCommonPrefixLengthAndBuildHistogram"
+                "-XX:+HeapDumpOnOutOfMemoryError"
             );
 
             test.getJvmArgumentProviders().add(new SimpleCommandLineArgumentProvider("-XX:HeapDumpPath=" + heapdumpDir));

build-tools-internal/version.properties

@@ -1,5 +1,5 @@
 elasticsearch     = 8.13.0
-lucene            = 9.9.0-snapshot-bb4fec631e6
+lucene            = 9.9.1
 
 bundled_jdk_vendor = openjdk
 bundled_jdk = 21.0.1+12@415e3f918a1f4062a0074a2794853d0d

distribution/src/config/jvm.options

@@ -58,10 +58,6 @@
 # result in less optimal vector performance
 20-:--add-modules=jdk.incubator.vector
 
-# REMOVE once bumped to a JDK greater than 21.0.1, https://github.com/elastic/elasticsearch/issues/103004
-19-21:-XX:CompileCommand=exclude,org.apache.lucene.util.MSBRadixSorter::computeCommonPrefixLengthAndBuildHistogram
-19-21:-XX:CompileCommand=exclude,org.apache.lucene.util.RadixSelector::computeCommonPrefixLengthAndBuildHistogram
-
 ## heap dumps
 
 # generate a heap dump when an allocation from the Java heap fails; heap dumps

distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/cli/PluginsConfig.java

@@ -160,12 +160,11 @@ static PluginsConfig parseConfig(Path configPath, XContent xContent) throws IOEx
         parser.declareStringOrNull(PluginsConfig::setProxy, new ParseField("proxy"));
         parser.declareObjectArrayOrNull(PluginsConfig::setPlugins, descriptorParser, new ParseField("plugins"));
 
-        final XContentParser yamlXContentParser = xContent.createParser(
-            XContentParserConfiguration.EMPTY,
-            Files.newInputStream(configPath)
-        );
-
-        return parser.parse(yamlXContentParser, null);
+        try (
+            XContentParser yamlXContentParser = xContent.createParser(XContentParserConfiguration.EMPTY, Files.newInputStream(configPath))
+        ) {
+            return parser.parse(yamlXContentParser, null);
+        }
     }
 
     /**

distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/MachineDependentHeap.java

@@ -86,8 +86,7 @@ static class NodeRoleParser {
         @SuppressWarnings("unchecked")
         public static MachineNodeRole parse(InputStream config) {
             final Settings settings;
-            try {
-                var parser = YamlXContent.yamlXContent.createParser(XContentParserConfiguration.EMPTY, config);
+            try (var parser = YamlXContent.yamlXContent.createParser(XContentParserConfiguration.EMPTY, config)) {
                 if (parser.currentToken() == null && parser.nextToken() == null) {
                     settings = null;
                 } else {

docs/Versions.asciidoc

@@ -1,8 +1,8 @@
 
 include::{docs-root}/shared/versions/stack/{source_branch}.asciidoc[]
 
-:lucene_version:        9.9.0
-:lucene_version_path:   9_9_0
+:lucene_version:        9.9.1
+:lucene_version_path:   9_9_1
 :jdk:                   11.0.2
 :jdk_major:             11
 :build_type:            tar

docs/changelog/102557.yaml

@@ -0,0 +1,5 @@
+pr: 102557
+summary: Metrics for search latencies
+area: Search
+type: enhancement
+issues: []

docs/changelog/103150.yaml

@@ -0,0 +1,6 @@
+pr: 103150
+summary: "ES|QL: Fix NPE on single value detection"
+area: ES|QL
+type: bug
+issues:
+ - 103141

docs/changelog/103387.yaml

@@ -0,0 +1,5 @@
+pr: 103387
+summary: Upgrade to Lucene 9.9.1
+area: Search
+type: upgrade
+issues: []

docs/changelog/103434.yaml

@@ -0,0 +1,11 @@
+pr: 103434
+summary: Lower the `look_ahead_time` index setting's max value from 7 days to 2 hours.
+area: TSDB
+type: breaking
+issues: []
+breaking:
+  title: Lower the `look_ahead_time` index setting's max value
+  area: Index setting
+  details: "Lower the `look_ahead_time` index setting's max value from 7 days to 2 hours."
+  impact: "Any value between 2 hours and 7 days will be as a look ahead time  of 2 hours is defined"
+  notable: false

docs/changelog/103520.yaml

@@ -0,0 +1,5 @@
+pr: 103520
+summary: Request indexing memory pressure in APM node metrics publisher
+area: Distributed
+type: bug
+issues: []

docs/changelog/103530.yaml

@@ -0,0 +1,5 @@
+pr: 103530
+summary: Exclude quantiles when fetching model snapshots where possible
+area: Machine Learning
+type: bug
+issues: []

docs/changelog/103574.yaml

@@ -0,0 +1,5 @@
+pr: 103574
+summary: Samples should check if the aggregations result is empty or null
+area: EQL
+type: bug
+issues: []

docs/changelog/103580.yaml

@@ -0,0 +1,6 @@
+pr: 103580
+summary: Copy counter field properties to downsampled index
+area: Downsampling
+type: bug
+issues:
+ - 103569

docs/reference/analysis/tokenfilters/synonym-graph-tokenfilter.asciidoc

@@ -45,6 +45,13 @@ Use `synonyms_set` configuration option to provide a synonym set created via Syn
   }
 ----
 
+[WARNING]
+======
+Synonyms sets must exist before they can be added to indices.
+If an index is created referencing a nonexistent synonyms set, the index will remain in a partially created and inoperable state.
+The only way to recover from this scenario is to ensure the synonyms set exists then either delete and re-create the index, or close and re-open the index.
+======
+
 Use `synonyms_path` to provide a synonym file :
 
 [source,JSON]

docs/reference/analysis/tokenfilters/synonym-tokenfilter.asciidoc

@@ -33,6 +33,13 @@ Use `synonyms_set` configuration option to provide a synonym set created via Syn
   }
 ----
 
+[WARNING]
+======
+Synonyms sets must exist before they can be added to indices.
+If an index is created referencing a nonexistent synonyms set, the index will remain in a partially created and inoperable state.
+The only way to recover from this scenario is to ensure the synonyms set exists then either delete and re-create the index, or close and re-open the index.
+======
+
 Use `synonyms_path` to provide a synonym file :
 
 [source,JSON]

docs/reference/connector/apis/connector-apis.asciidoc

@@ -27,7 +27,9 @@ Use the following APIs to manage connectors:
 * <<get-connector-api>>
 * <<list-connector-api>>
 * <<check-in-connector-api>>
+* <<update-connector-configuration-api>>
 * <<update-connector-error-api>>
+* <<update-connector-filtering-api>>
 * <<update-connector-last-sync-api>>
 * <<update-connector-name-description-api>>
 * <<update-connector-pipeline-api>>
@@ -66,7 +68,9 @@ include::list-connectors-api.asciidoc[]
 include::list-connector-sync-jobs-api.asciidoc[]
 include::set-connector-sync-job-error-api.asciidoc[]
 include::set-connector-sync-job-stats-api.asciidoc[]
+include::update-connector-configuration-api.asciidoc[]
 include::update-connector-error-api.asciidoc[]
+include::update-connector-filtering-api.asciidoc[]
 include::update-connector-last-sync-api.asciidoc[]
 include::update-connector-name-description-api.asciidoc[]
 include::update-connector-pipeline-api.asciidoc[]

docs/reference/connector/apis/update-connector-configuration-api.asciidoc

@@ -0,0 +1,154 @@
+[[update-connector-configuration-api]]
+=== Update connector configuration API
+
+preview::[]
+
+++++
+<titleabbrev>Update connector configuration</titleabbrev>
+++++
+
+Updates the `configuration` of a connector.
+
+
+[[update-connector-configuration-api-request]]
+==== {api-request-title}
+
+`PUT _connector/<connector_id>/_configuration`
+
+[[update-connector-configuration-api-prereq]]
+==== {api-prereq-title}
+
+* To sync data using connectors, it's essential to have the Elastic connectors service running.
+* The `connector_id` parameter should reference an existing connector.
+* The configuration fields definition must be compatible with the specific connector type being used.
+
+[[update-connector-configuration-api-path-params]]
+==== {api-path-parms-title}
+
+`<connector_id>`::
+(Required, string)
+
+[role="child_attributes"]
+[[update-connector-configuration-api-request-body]]
+==== {api-request-body-title}
+
+`configuration`::
+(Required, object) The configuration for the connector. The configuration field is a map where each key represents a specific configuration field name, and the value is a `ConnectorConfiguration` object.
+
+Each `ConnectorConfiguration` object contains the following attributes:
+
+* `category` (Optional, string) The category of the configuration field. This helps in grouping related configurations together in the user interface.
+
+* `default_value` (Required, string | number | bool) The default value for the configuration. This value is used if the value field is empty, applicable only for non-required fields.
+
+* `depends_on` (Required, array of `ConfigurationDependency`) An array of dependencies on other configurations. A field will not be enabled unless these dependencies are met. Each dependency specifies a field key and the required value for the dependency to be considered fulfilled.
+
+* `display` (Required, string) The display type for the UI element that represents this configuration. This defines how the field should be rendered in the user interface. Supported types are: `text`, `textbox`, `textarea`, `numeric`, `toggle` and `dropdown`.
+
+* `label` (Required, string) The display label for the configuration field. This label is shown in the user interface, adjacent to the field.
+
+* `options` (Required, array of `ConfigurationSelectOption`) An array of options for list-type fields. These options are used for inputs in the user interface, each having a label for display and a value.
+
+* `order` (Required, number) The order in which this configuration appears in the user interface. This helps in organizing fields logically.
+
+* `placeholder` (Required, string) Placeholder text for the configuration field. This text is displayed inside the field before a value is entered.
+
+* `required` (Required, boolean) Indicates whether the configuration is mandatory. If true, a value must be provided for the field.
+
+* `sensitive` (Required, boolean) Indicates whether the configuration contains sensitive information. Sensitive fields may be obfuscated in the user interface.
+
+* `tooltip` (Optional, string) Tooltip text providing additional information about the configuration. This text appears when the user hovers over the info icon next to the configuration field.
+
+* `type` (Required, string) The type of the configuration field, such as `str`, `int`, `bool`, `list`. This defines the data type and format of the field's value.
+
+* `ui_restrictions` (Required, array of strings) A list of UI restrictions. These restrictions define where in the user interface this field should be available or restricted.
+
+* `validations` (Required, array of `ConfigurationValidation`) An array of rules for validating the field's value. Each validation specifies a type and a constraint that the field's value must meet.
+
+* `value` (Required, string | number | bool) The current value of the configuration. This is the actual value set for the field and is used by the connector during its operations.
+
+`ConfigurationDependency` represents a dependency that a configuration field has on another field's value. It contains the following attributes:
+
+* `field` (Required, string) The name of the field in the configuration that this dependency relates to.
+
+* `value` (Required, string | number | bool) The required value of the specified field for this dependency to be met.
+
+`ConfigurationSelectOption` defines an option within a selectable configuration field. It contains the following attributes:
+
+* `label` (Required, string) The display label for the option.
+
+* `value` (Required, string) The actual value associated with the option.
+
+`ConfigurationValidation` specifies validation rules for configuration fields. Each ConfigurationValidation instance enforces a specific type of validation based on its type and constraint. It contains the following attributes:
+
+* `constraint` (Required, string | number) The validation constraint. The nature of this constraint depends on the validation type. It could be a numeric value, a list, a regular expression pattern.
+
+* `type` (Required, ConfigurationValidationType) The type of validation to be performed. Possible values include: `less_than`, `greater_than`, `list_type`, `included_in`, `regex` and `unset`.
+
+
+[[update-connector-configuration-api-response-codes]]
+==== {api-response-codes-title}
+
+`200`::
+Connector configuration was successfully updated.
+
+`400`::
+The `connector_id` was not provided or the request payload was malformed.
+
+`404` (Missing resources)::
+No connector matching `connector_id` could be found.
+
+[[update-connector-configuration-api-example]]
+==== {api-examples-title}
+
+The following example updates the `configuration` for the connector with ID `my-connector`:
+
+////
+[source, console]
+--------------------------------------------------
+PUT _connector/my-connector
+{
+  "index_name": "search-google-drive",
+  "name": "My Connector",
+  "service_type": "google_drive"
+}
+--------------------------------------------------
+// TESTSETUP
+
+[source,console]
+--------------------------------------------------
+DELETE _connector/my-connector
+--------------------------------------------------
+// TEARDOWN
+////
+
+[source,console]
+----
+PUT _connector/my-connector/_configuration
+{
+    "configuration": {
+        "service_account_credentials": {
+            "default_value": null,
+            "depends_on": [],
+            "display": "textarea",
+            "label": "Google Drive service account JSON",
+            "options": [],
+            "order": 1,
+            "required": true,
+            "sensitive": true,
+            "tooltip": "This connectors authenticates as a service account to synchronize content from Google Drive.",
+            "type": "str",
+            "ui_restrictions": [],
+            "validations": [],
+            "value": "...service account JSON..."
+        }
+    }
+}
+----
+
+[source,console-result]
+----
+{
+    "result": "updated"
+}
+----