Merge branch '8.16' into backport/8.16/pr-116809

n1v0lg · Nov 25, 2024 · cd35607 · cd35607
2 parents 20889fd + f0469a4
commit cd35607
Show file tree

Hide file tree

Showing 28 changed files with 396 additions and 87 deletions.
diff --git a/build-tools-internal/gradle/wrapper/gradle-wrapper.properties b/build-tools-internal/gradle/wrapper/gradle-wrapper.properties
@@ -1,7 +1,7 @@
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionSha256Sum=2ab88d6de2c23e6adae7363ae6e29cbdd2a709e992929b48b6530fd0c7133bd6
-distributionUrl=https\://services.gradle.org/distributions/gradle-8.10.2-all.zip
+distributionSha256Sum=89d4e70e4e84e2d2dfbb63e4daa53e21b25017cc70c37e4eea31ee51fb15098a
+distributionUrl=https\://services.gradle.org/distributions/gradle-8.11.1-all.zip
 networkTimeout=10000
 validateDistributionUrl=true
 zipStoreBase=GRADLE_USER_HOME

diff --git a/...-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchJavadocPlugin.java b/...-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchJavadocPlugin.java
@@ -82,12 +82,15 @@ private void configureJavadocForConfiguration(Project project, boolean shadow, C
             .sorted(Comparator.comparing(Dependency::getGroup))
             .filter(d -> d instanceof ProjectDependency)
             .map(d -> (ProjectDependency) d)
-            .filter(p -> p.getDependencyProject() != null)
             .forEach(projectDependency -> configureDependency(project, shadow, projectDependency));
     }
 
     private void configureDependency(Project project, boolean shadowed, ProjectDependency dep) {
-        var upstreamProject = dep.getDependencyProject();
+        // we should use variant aware dependency management to resolve artifacts required for javadoc here
+        Project upstreamProject = project.project(dep.getPath());
+        if (upstreamProject == null) {
+            return;
+        }
         if (shadowed) {
             /*
              * Include the source of shadowed upstream projects so we don't

diff --git a/...rnal/src/main/java/org/elasticsearch/gradle/internal/test/TestWithDependenciesPlugin.java b/...rnal/src/main/java/org/elasticsearch/gradle/internal/test/TestWithDependenciesPlugin.java
@@ -10,11 +10,14 @@
 package org.elasticsearch.gradle.internal.test;
 
 import org.apache.commons.lang.StringUtils;
-import org.elasticsearch.gradle.plugin.PluginBuildPlugin;
 import org.gradle.api.Plugin;
 import org.gradle.api.Project;
 import org.gradle.api.artifacts.Configuration;
+import org.gradle.api.artifacts.Dependency;
 import org.gradle.api.artifacts.ProjectDependency;
+import org.gradle.api.artifacts.dsl.DependencyHandler;
+import org.gradle.api.attributes.Attribute;
+import org.gradle.api.attributes.LibraryElements;
 import org.gradle.api.plugins.ExtraPropertiesExtension;
 import org.gradle.api.tasks.Copy;
 import org.gradle.api.tasks.SourceSetContainer;
@@ -45,23 +48,31 @@ public void apply(final Project project) {
 
         Configuration testImplementationConfig = project.getConfigurations().getByName("testImplementation");
         testImplementationConfig.getDependencies().all(dep -> {
-            if (dep instanceof ProjectDependency
-                && ((ProjectDependency) dep).getDependencyProject().getPlugins().hasPlugin(PluginBuildPlugin.class)) {
-                project.getGradle()
-                    .projectsEvaluated(gradle -> addPluginResources(project, ((ProjectDependency) dep).getDependencyProject()));
+            if (dep instanceof ProjectDependency && dep.getGroup().contains("plugin")) {
+                addPluginResources(project, ((ProjectDependency) dep));
             }
         });
     }
 
-    private static void addPluginResources(final Project project, final Project pluginProject) {
-        final File outputDir = new File(project.getBuildDir(), "/generated-test-resources/" + pluginProject.getName());
-        String camelProjectName = stream(pluginProject.getName().split("-")).map(t -> StringUtils.capitalize(t))
+    private static void addPluginResources(final Project project, final ProjectDependency projectDependency) {
+        final File outputDir = new File(project.getBuildDir(), "/generated-test-resources/" + projectDependency.getName());
+        String camelProjectName = stream(projectDependency.getName().split("-")).map(t -> StringUtils.capitalize(t))
             .collect(Collectors.joining());
         String taskName = "copy" + camelProjectName + "Metadata";
+        String metadataConfiguration = "resolved" + camelProjectName + "Metadata";
+        Configuration pluginMetadata = project.getConfigurations().maybeCreate(metadataConfiguration);
+        pluginMetadata.getAttributes().attribute(Attribute.of("pluginMetadata", Boolean.class), true);
+        pluginMetadata.getAttributes()
+            .attribute(
+                LibraryElements.LIBRARY_ELEMENTS_ATTRIBUTE,
+                project.getObjects().named(LibraryElements.class, LibraryElements.RESOURCES)
+            );
+        DependencyHandler dependencyHandler = project.getDependencies();
+        Dependency pluginMetadataDependency = dependencyHandler.project(Map.of("path", projectDependency.getPath()));
+        dependencyHandler.add(metadataConfiguration, pluginMetadataDependency);
         project.getTasks().register(taskName, Copy.class, copy -> {
             copy.into(outputDir);
-            copy.from(pluginProject.getTasks().named("pluginProperties"));
-            copy.from(pluginProject.file("src/main/plugin-metadata"));
+            copy.from(pluginMetadata);
         });
 
         Map<String, Object> map = Map.of("builtBy", taskName);

diff --git a/...nternal/src/main/java/org/elasticsearch/gradle/internal/test/rest/RestTestBasePlugin.java b/...nternal/src/main/java/org/elasticsearch/gradle/internal/test/rest/RestTestBasePlugin.java
@@ -44,6 +44,7 @@
 import org.gradle.api.file.ConfigurableFileCollection;
 import org.gradle.api.file.FileCollection;
 import org.gradle.api.file.FileTree;
+import org.gradle.api.internal.artifacts.dependencies.ProjectDependencyInternal;
 import org.gradle.api.provider.ProviderFactory;
 import org.gradle.api.tasks.ClasspathNormalizer;
 import org.gradle.api.tasks.PathSensitivity;
@@ -245,7 +246,7 @@ private void copyDependencies(Project project, DependencySet dependencies, Confi
         configuration.getDependencies()
             .stream()
             .filter(d -> d instanceof ProjectDependency)
-            .map(d -> project.getDependencies().project(Map.of("path", ((ProjectDependency) d).getDependencyProject().getPath())))
+            .map(d -> project.getDependencies().project(Map.of("path", ((ProjectDependencyInternal) d).getPath())))
             .forEach(dependencies::add);
     }
 
@@ -322,8 +323,9 @@ private Configuration createPluginConfiguration(Project project, String name, bo
                     Collection<Dependency> additionalDependencies = new LinkedHashSet<>();
                     for (Iterator<Dependency> iterator = dependencies.iterator(); iterator.hasNext();) {
                         Dependency dependency = iterator.next();
+                        // this logic of relying on other projects metadata should probably live in a build service
                         if (dependency instanceof ProjectDependency projectDependency) {
-                            Project dependencyProject = projectDependency.getDependencyProject();
+                            Project dependencyProject = project.project(projectDependency.getPath());
                             List<String> extendedPlugins = dependencyProject.getExtensions()
                                 .getByType(PluginPropertiesExtension.class)
                                 .getExtendedPlugins();
@@ -333,8 +335,8 @@ private Configuration createPluginConfiguration(Project project, String name, bo
                                 iterator.remove();
                                 additionalDependencies.add(
                                     useExploded
-                                        ? getExplodedBundleDependency(project, dependencyProject.getPath())
-                                        : getBundleZipTaskDependency(project, dependencyProject.getPath())
+                                        ? getExplodedBundleDependency(project, projectDependency.getPath())
+                                        : getBundleZipTaskDependency(project, projectDependency.getPath())
                                 );
                             }
 

diff --git a/...java/org/elasticsearch/gradle/internal/toolchain/AbstractCustomJavaToolchainResolver.java b/...java/org/elasticsearch/gradle/internal/toolchain/AbstractCustomJavaToolchainResolver.java
@@ -34,6 +34,7 @@ static String toArchString(Architecture architecture) {
             case X86_64 -> "x64";
             case AARCH64 -> "aarch64";
             case X86 -> "x86";
+            default -> throw new UnsupportedOperationException("Architecture " + architecture);
         };
     }
 

diff --git a/build-tools-internal/src/main/resources/minimumGradleVersion b/build-tools-internal/src/main/resources/minimumGradleVersion
@@ -1 +1 @@
-8.10.2
+8.11.1
diff --git a/build-tools/src/main/java/org/elasticsearch/gradle/LazyFileOutputStream.java b/build-tools/src/main/java/org/elasticsearch/gradle/LazyFileOutputStream.java
@@ -39,6 +39,12 @@ public void write(byte b[], int off, int len) throws IOException {
                 bootstrap();
                 delegate.write(b, off, len);
             }
+
+            @Override
+            public void write(byte b[]) throws IOException {
+                bootstrap();
+                delegate.write(b);
+            }
         };
     }
 

diff --git a/build-tools/src/main/java/org/elasticsearch/gradle/plugin/BasePluginBuildPlugin.java b/build-tools/src/main/java/org/elasticsearch/gradle/plugin/BasePluginBuildPlugin.java
@@ -24,6 +24,8 @@
 import org.gradle.api.Task;
 import org.gradle.api.Transformer;
 import org.gradle.api.artifacts.type.ArtifactTypeDefinition;
+import org.gradle.api.attributes.Attribute;
+import org.gradle.api.attributes.LibraryElements;
 import org.gradle.api.file.CopySpec;
 import org.gradle.api.file.FileCollection;
 import org.gradle.api.file.RegularFile;
@@ -126,9 +128,27 @@ private TaskProvider<Zip> createBundleTasks(final Project project, PluginPropert
         // know about the plugin (used by test security code to statically initialize the plugin in unit tests)
         var testSourceSet = project.getExtensions().getByType(SourceSetContainer.class).getByName("test");
         Map<String, Object> map = Map.of("builtBy", buildProperties);
-        testSourceSet.getOutput().dir(map, new File(project.getBuildDir(), "generated-resources"));
+
+        File generatedResources = new File(project.getBuildDir(), "generated-resources");
+        testSourceSet.getOutput().dir(map, generatedResources);
         testSourceSet.getResources().srcDir(pluginMetadata);
 
+        // expose the plugin properties and metadata for other plugins to use in their tests.
+        // See TestWithDependenciesPlugin for how this is used.
+        project.getConfigurations().create("pluginMetadata", conf -> {
+            conf.getAttributes().attribute(Attribute.of("pluginMetadata", Boolean.class), true);
+            conf.getAttributes()
+                .attribute(
+                    LibraryElements.LIBRARY_ELEMENTS_ATTRIBUTE,
+                    project.getObjects().named(LibraryElements.class, LibraryElements.RESOURCES)
+                );
+        });
+
+        project.getArtifacts().add("pluginMetadata", new File(project.getBuildDir(), "generated-descriptor"), artifact -> {
+            artifact.builtBy(buildProperties);
+        });
+        project.getArtifacts().add("pluginMetadata", pluginMetadata);
+        // getAttributes().attribute(ArtifactTypeDefinition.ARTIFACT_TYPE_ATTRIBUTE, "plugin-metadata");
         var bundleSpec = createBundleSpec(project, pluginMetadata, buildProperties);
         extension.setBundleSpec(bundleSpec);
         // create the actual bundle task, which zips up all the files for the plugin

diff --git a/distribution/packages/build.gradle b/distribution/packages/build.gradle
@@ -44,7 +44,7 @@ import java.util.regex.Pattern
  */
 
 plugins {
-  id "com.netflix.nebula.ospackage-base" version "11.9.1"
+  id "com.netflix.nebula.ospackage-base" version "11.10.0"
 }
 
 ['deb', 'rpm'].each { type ->
@@ -196,7 +196,7 @@ def commonPackageConfig(String type, String architecture) {
     configurationFile '/etc/elasticsearch/users_roles'
     from("${packagingFiles}") {
       dirPermissions {
-        unix(02750)
+        unix(0750)
       }
       into('/etc')
       permissionGroup 'elasticsearch'
@@ -209,7 +209,7 @@ def commonPackageConfig(String type, String architecture) {
     from("${packagingFiles}/etc/elasticsearch") {
       into('/etc/elasticsearch')
       dirPermissions {
-        unix(02750)
+        unix(0750)
       }
       setgid = true
       filePermissions {
@@ -261,7 +261,7 @@ def commonPackageConfig(String type, String architecture) {
 
     // ========= empty dirs =========
     // NOTE: these are created under packagingFiles as empty, but the permissions are set here
-    Closure copyEmptyDir = { path, u, g, mode ->
+    Closure copyEmptyDir = { path, u, g, gid, mode ->
       File file = new File(path)
       into(file.parent) {
         from "${packagingFiles}/${file.parent}"
@@ -273,12 +273,12 @@ def commonPackageConfig(String type, String architecture) {
         dirPermissions {
           unix(mode)
         }
-        setgid (mode == 02750)
+        setgid(gid)
       }
     }
-    copyEmptyDir('/var/log/elasticsearch', 'elasticsearch', 'elasticsearch', 02750)
-    copyEmptyDir('/var/lib/elasticsearch', 'elasticsearch', 'elasticsearch', 02750)
-    copyEmptyDir('/usr/share/elasticsearch/plugins', 'root', 'root', 0755)
+    copyEmptyDir('/var/log/elasticsearch', 'elasticsearch', 'elasticsearch', true, 0750)
+    copyEmptyDir('/var/lib/elasticsearch', 'elasticsearch', 'elasticsearch', true, 0750)
+    copyEmptyDir('/usr/share/elasticsearch/plugins', 'root', 'root', false, 0755)
 
     // the oss package conflicts with the default distribution and vice versa
     conflicts('elasticsearch-oss')

diff --git a/docs/changelog/115812.yaml b/docs/changelog/115812.yaml
@@ -0,0 +1,5 @@
+pr: 115812
+summary: "Prohibit changes to index mode, source, and sort settings during resize"
+area: Logs
+type: bug
+issues: []
diff --git a/docs/changelog/117297.yaml b/docs/changelog/117297.yaml
@@ -0,0 +1,5 @@
+pr: 117297
+summary: Fix CCS exchange when multi cluster aliases point to same cluster
+area: ES|QL
+type: bug
+issues: []
diff --git a/docs/changelog/117316.yaml b/docs/changelog/117316.yaml
@@ -0,0 +1,5 @@
+pr: 117316
+summary: Fix validation of SORT by aggregate functions
+area: ES|QL
+type: bug
+issues: []
diff --git a/docs/reference/esql/functions/aggregation-functions.asciidoc b/docs/reference/esql/functions/aggregation-functions.asciidoc
@@ -20,7 +20,7 @@ The <<esql-stats-by>> command supports these aggregate functions:
 * <<esql-sum>>
 * <<esql-top>>
 * <<esql-values>>
-* experimental:[] <<esql-weighted_avg>>
+* <<esql-weighted_avg>>
 // end::agg_list[]
 
 include::layout/avg.asciidoc[]

diff --git a/docs/reference/esql/functions/mv-functions.asciidoc b/docs/reference/esql/functions/mv-functions.asciidoc
@@ -19,6 +19,7 @@
 * <<esql-mv_median>>
 * <<esql-mv_median_absolute_deviation>>
 * <<esql-mv_min>>
+* <<esql-mv_percentile>>
 * <<esql-mv_pseries_weighted_sum>>
 * <<esql-mv_sort>>
 * <<esql-mv_slice>>
@@ -37,6 +38,7 @@ include::layout/mv_max.asciidoc[]
 include::layout/mv_median.asciidoc[]
 include::layout/mv_median_absolute_deviation.asciidoc[]
 include::layout/mv_min.asciidoc[]
+include::layout/mv_percentile.asciidoc[]
 include::layout/mv_pseries_weighted_sum.asciidoc[]
 include::layout/mv_slice.asciidoc[]
 include::layout/mv_sort.asciidoc[]

diff --git a/gradle/verification-metadata.xml b/gradle/verification-metadata.xml
@@ -914,6 +914,11 @@
             <sha256 value="d694edd7bae3bc1a8e0dae2f5a22c479ff04d6b9bfcb0ab751a42f02e02d2100" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="com.netflix.nebula" name="gradle-ospackage-plugin" version="11.10.0">
+         <artifact name="gradle-ospackage-plugin-11.10.0.jar">
+            <sha256 value="93c264b05864f8db979a169a99e4bfaad84caa30b829743f67210382dff89619" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="com.netflix.nebula" name="gradle-ospackage-plugin" version="11.9.1">
          <artifact name="gradle-ospackage-plugin-11.9.1.jar">
             <sha256 value="63b2c387cb83481bb8d7e33e0be02e9000c0ec30843ca7730641392f3f5f28ad" origin="Generated by Gradle"/>
@@ -2995,6 +3000,11 @@
             <sha256 value="29b2628d0f028119715916849c19eed45b32cdc67eb13551798c73afa4aa23d6" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="org.apache.maven" name="maven-archiver" version="3.5.1">
+         <artifact name="maven-archiver-3.5.1.jar">
+            <sha256 value="d0674469f8e6fbc866dbc5c955463d7629b7755ac7c87a235b9469d8257d6b8c" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="org.apache.maven" name="maven-artifact" version="3.6.1">
          <artifact name="maven-artifact-3.6.1.jar">
             <sha256 value="797fa1bcf54d7216e863b8177c04507d09c708eec153626eca473054c0eb56f3" origin="Generated by Gradle"/>
@@ -3010,6 +3020,11 @@
             <sha256 value="c29763149b09b294c981d12d70f96ea4edb60d462238bb5dcbaae224e6acfdaa" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="org.apache.maven" name="maven-model" version="3.1.1">
+         <artifact name="maven-model-3.1.1.jar">
+            <sha256 value="e71bad235af182c43dfcaa9b649032352cf859f9d09f79e1f537268384176de1" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="org.apache.maven" name="maven-model" version="3.6.1">
          <artifact name="maven-model-3.6.1.jar">
             <sha256 value="ab10ced4a0f692cae285effacc70d458806ede856b9a2b3b318de1eb2f2f3b05" origin="Generated by Gradle"/>
@@ -3080,6 +3095,11 @@
             <sha256 value="3ba9c619893c767db0f9c3e826d5118b57c35229301bcd16d865a89cec16a7e5" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="org.apache.maven.shared" name="maven-shared-utils" version="3.3.3">
+         <artifact name="maven-shared-utils-3.3.3.jar">
+            <sha256 value="44a60c610f4e31524b03d81a698b1ecceba116320ea510babf859575b2ea7233" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="org.apache.mina" name="mina-core" version="2.0.17">
          <artifact name="mina-core-2.0.17.jar">
             <sha256 value="08316826fa2b9357b061e52fa8f19ccae75420c949ebe29e28759d2bddd9b39b" origin="Generated by Gradle"/>
@@ -3388,6 +3408,11 @@
             <sha256 value="92654f493ecfec52082e76354f0ebf87648dc3d5cec2e3c3cdb947c016747a53" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="org.codehaus.plexus" name="plexus-archiver" version="4.2.3">
+         <artifact name="plexus-archiver-4.2.3.jar">
+            <sha256 value="fff8157bfb3d6f1f099c7da7de3e14bfd061ad4ab29599afa7bbf0271368d748" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="org.codehaus.plexus" name="plexus-classworlds" version="2.6.0">
          <artifact name="plexus-classworlds-2.6.0.jar">
             <sha256 value="52f77c5ec49f787c9c417ebed5d6efd9922f44a202f217376e4f94c0d74f3549" origin="Generated by Gradle"/>
@@ -3403,6 +3428,16 @@
             <sha256 value="e003802501574637f7abdc4e83e6d509a31e9ff825d12da6d1e419acf9688705" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="org.codehaus.plexus" name="plexus-interpolation" version="1.26">
+         <artifact name="plexus-interpolation-1.26.jar">
+            <sha256 value="b3b5412ce17889103ea564bcdfcf9fb3dfa540344ffeac6b538a73c9d7182662" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
+      <component group="org.codehaus.plexus" name="plexus-io" version="3.2.0">
+         <artifact name="plexus-io-3.2.0.jar">
+            <sha256 value="15cf8cbd9e014b7156482bbb48e515613158bdd9b4b908d21e6b900f7876f6ff" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="org.codehaus.plexus" name="plexus-utils" version="3.2.0">
          <artifact name="plexus-utils-3.2.0.jar">
             <sha256 value="0b91029df4c216b8824bd95361f52e260e86ccf93a2619fd88c8f15d23dcb30d" origin="Generated by Gradle"/>
@@ -3723,6 +3758,11 @@
             <sha256 value="631656eb38639b0ae41161f706ff7fbe04313b5b8f42892da5ec656390031fc6" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="org.iq80.snappy" name="snappy" version="0.4">
+         <artifact name="snappy-0.4.jar">
+            <sha256 value="46a0c87d504ce9d6063e1ff6e4d20738feb49d8abf85b5071a7d18df4f11bac9" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="org.javassist" name="javassist" version="3.18.1-GA">
          <artifact name="javassist-3.18.1-GA.jar">
             <sha256 value="3fb71231afd098bb0f93f5eb97aa8291c8d0556379125e596f92ec8f944c6162" origin="Generated by Gradle"/>
@@ -4473,6 +4513,11 @@
             <sha256 value="211b306cfc44f8f96df3a0a3ddaf75ba8c5289eed77d60d72f889bb855f535e5" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="org.vafer" name="jdeb" version="1.10">
+         <artifact name="jdeb-1.10.jar">
+            <sha256 value="254cbb052f718f36ee2a75ae95bd2e0251b0513fc76d038b6dbbcf0942fcc11f" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="org.vafer" name="jdeb" version="1.8">
          <artifact name="jdeb-1.8.jar">
             <sha256 value="374fe02a09314da6ec8219e005598e760323a1abe4b715cc33ce3e2a9bcda932" origin="Generated by Gradle"/>