Merge branch '8.17' into backport/8.17/pr-116809

n1v0lg · Nov 25, 2024 · 5af269f · 5af269f
2 parents c8e42d8 + 11283c0
commit 5af269f
Show file tree

Hide file tree

Showing 32 changed files with 405 additions and 96 deletions.
diff --git a/build-tools-internal/gradle/wrapper/gradle-wrapper.properties b/build-tools-internal/gradle/wrapper/gradle-wrapper.properties
@@ -1,7 +1,7 @@
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionSha256Sum=2ab88d6de2c23e6adae7363ae6e29cbdd2a709e992929b48b6530fd0c7133bd6
-distributionUrl=https\://services.gradle.org/distributions/gradle-8.10.2-all.zip
+distributionSha256Sum=89d4e70e4e84e2d2dfbb63e4daa53e21b25017cc70c37e4eea31ee51fb15098a
+distributionUrl=https\://services.gradle.org/distributions/gradle-8.11.1-all.zip
 networkTimeout=10000
 validateDistributionUrl=true
 zipStoreBase=GRADLE_USER_HOME

diff --git a/...nal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchBuildCompletePlugin.java b/...nal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchBuildCompletePlugin.java
@@ -147,12 +147,17 @@ interface Parameters extends FlowParameters {
         @SuppressWarnings("checkstyle:DescendantToken")
         @Override
         public void execute(BuildFinishedFlowAction.Parameters parameters) throws FileNotFoundException {
+            List<File> filesToArchive = parameters.getFilteredFiles().get();
+            if (filesToArchive.isEmpty()) {
+                return;
+            }
             File uploadFile = parameters.getUploadFile().get();
             if (uploadFile.exists()) {
                 getFileSystemOperations().delete(spec -> spec.delete(uploadFile));
             }
             uploadFile.getParentFile().mkdirs();
-            createBuildArchiveTar(parameters.getFilteredFiles().get(), parameters.getProjectDir().get(), uploadFile);
+
+            createBuildArchiveTar(filesToArchive, parameters.getProjectDir().get(), uploadFile);
             if (uploadFile.exists() && "true".equals(System.getenv("BUILDKITE"))) {
                 String uploadFilePath = uploadFile.getName();
                 File uploadFileDir = uploadFile.getParentFile();

diff --git a/...-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchJavadocPlugin.java b/...-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchJavadocPlugin.java
@@ -82,12 +82,15 @@ private void configureJavadocForConfiguration(Project project, boolean shadow, C
             .sorted(Comparator.comparing(Dependency::getGroup))
             .filter(d -> d instanceof ProjectDependency)
             .map(d -> (ProjectDependency) d)
-            .filter(p -> p.getDependencyProject() != null)
             .forEach(projectDependency -> configureDependency(project, shadow, projectDependency));
     }
 
     private void configureDependency(Project project, boolean shadowed, ProjectDependency dep) {
-        var upstreamProject = dep.getDependencyProject();
+        // we should use variant aware dependency management to resolve artifacts required for javadoc here
+        Project upstreamProject = project.project(dep.getPath());
+        if (upstreamProject == null) {
+            return;
+        }
         if (shadowed) {
             /*
              * Include the source of shadowed upstream projects so we don't

diff --git a/...rnal/src/main/java/org/elasticsearch/gradle/internal/test/TestWithDependenciesPlugin.java b/...rnal/src/main/java/org/elasticsearch/gradle/internal/test/TestWithDependenciesPlugin.java
@@ -10,11 +10,14 @@
 package org.elasticsearch.gradle.internal.test;
 
 import org.apache.commons.lang.StringUtils;
-import org.elasticsearch.gradle.plugin.PluginBuildPlugin;
 import org.gradle.api.Plugin;
 import org.gradle.api.Project;
 import org.gradle.api.artifacts.Configuration;
+import org.gradle.api.artifacts.Dependency;
 import org.gradle.api.artifacts.ProjectDependency;
+import org.gradle.api.artifacts.dsl.DependencyHandler;
+import org.gradle.api.attributes.Attribute;
+import org.gradle.api.attributes.LibraryElements;
 import org.gradle.api.plugins.ExtraPropertiesExtension;
 import org.gradle.api.tasks.Copy;
 import org.gradle.api.tasks.SourceSetContainer;
@@ -45,23 +48,31 @@ public void apply(final Project project) {
 
         Configuration testImplementationConfig = project.getConfigurations().getByName("testImplementation");
         testImplementationConfig.getDependencies().all(dep -> {
-            if (dep instanceof ProjectDependency
-                && ((ProjectDependency) dep).getDependencyProject().getPlugins().hasPlugin(PluginBuildPlugin.class)) {
-                project.getGradle()
-                    .projectsEvaluated(gradle -> addPluginResources(project, ((ProjectDependency) dep).getDependencyProject()));
+            if (dep instanceof ProjectDependency && dep.getGroup().contains("plugin")) {
+                addPluginResources(project, ((ProjectDependency) dep));
             }
         });
     }
 
-    private static void addPluginResources(final Project project, final Project pluginProject) {
-        final File outputDir = new File(project.getBuildDir(), "/generated-test-resources/" + pluginProject.getName());
-        String camelProjectName = stream(pluginProject.getName().split("-")).map(t -> StringUtils.capitalize(t))
+    private static void addPluginResources(final Project project, final ProjectDependency projectDependency) {
+        final File outputDir = new File(project.getBuildDir(), "/generated-test-resources/" + projectDependency.getName());
+        String camelProjectName = stream(projectDependency.getName().split("-")).map(t -> StringUtils.capitalize(t))
             .collect(Collectors.joining());
         String taskName = "copy" + camelProjectName + "Metadata";
+        String metadataConfiguration = "resolved" + camelProjectName + "Metadata";
+        Configuration pluginMetadata = project.getConfigurations().maybeCreate(metadataConfiguration);
+        pluginMetadata.getAttributes().attribute(Attribute.of("pluginMetadata", Boolean.class), true);
+        pluginMetadata.getAttributes()
+            .attribute(
+                LibraryElements.LIBRARY_ELEMENTS_ATTRIBUTE,
+                project.getObjects().named(LibraryElements.class, LibraryElements.RESOURCES)
+            );
+        DependencyHandler dependencyHandler = project.getDependencies();
+        Dependency pluginMetadataDependency = dependencyHandler.project(Map.of("path", projectDependency.getPath()));
+        dependencyHandler.add(metadataConfiguration, pluginMetadataDependency);
         project.getTasks().register(taskName, Copy.class, copy -> {
             copy.into(outputDir);
-            copy.from(pluginProject.getTasks().named("pluginProperties"));
-            copy.from(pluginProject.file("src/main/plugin-metadata"));
+            copy.from(pluginMetadata);
         });
 
         Map<String, Object> map = Map.of("builtBy", taskName);

diff --git a/...nternal/src/main/java/org/elasticsearch/gradle/internal/test/rest/RestTestBasePlugin.java b/...nternal/src/main/java/org/elasticsearch/gradle/internal/test/rest/RestTestBasePlugin.java
@@ -44,6 +44,7 @@
 import org.gradle.api.file.ConfigurableFileCollection;
 import org.gradle.api.file.FileCollection;
 import org.gradle.api.file.FileTree;
+import org.gradle.api.internal.artifacts.dependencies.ProjectDependencyInternal;
 import org.gradle.api.provider.ProviderFactory;
 import org.gradle.api.tasks.ClasspathNormalizer;
 import org.gradle.api.tasks.PathSensitivity;
@@ -245,7 +246,7 @@ private void copyDependencies(Project project, DependencySet dependencies, Confi
         configuration.getDependencies()
             .stream()
             .filter(d -> d instanceof ProjectDependency)
-            .map(d -> project.getDependencies().project(Map.of("path", ((ProjectDependency) d).getDependencyProject().getPath())))
+            .map(d -> project.getDependencies().project(Map.of("path", ((ProjectDependencyInternal) d).getPath())))
             .forEach(dependencies::add);
     }
 
@@ -322,8 +323,9 @@ private Configuration createPluginConfiguration(Project project, String name, bo
                     Collection<Dependency> additionalDependencies = new LinkedHashSet<>();
                     for (Iterator<Dependency> iterator = dependencies.iterator(); iterator.hasNext();) {
                         Dependency dependency = iterator.next();
+                        // this logic of relying on other projects metadata should probably live in a build service
                         if (dependency instanceof ProjectDependency projectDependency) {
-                            Project dependencyProject = projectDependency.getDependencyProject();
+                            Project dependencyProject = project.project(projectDependency.getPath());
                             List<String> extendedPlugins = dependencyProject.getExtensions()
                                 .getByType(PluginPropertiesExtension.class)
                                 .getExtendedPlugins();
@@ -333,8 +335,8 @@ private Configuration createPluginConfiguration(Project project, String name, bo
                                 iterator.remove();
                                 additionalDependencies.add(
                                     useExploded
-                                        ? getExplodedBundleDependency(project, dependencyProject.getPath())
-                                        : getBundleZipTaskDependency(project, dependencyProject.getPath())
+                                        ? getExplodedBundleDependency(project, projectDependency.getPath())
+                                        : getBundleZipTaskDependency(project, projectDependency.getPath())
                                 );
                             }
 

diff --git a/...java/org/elasticsearch/gradle/internal/toolchain/AbstractCustomJavaToolchainResolver.java b/...java/org/elasticsearch/gradle/internal/toolchain/AbstractCustomJavaToolchainResolver.java
@@ -34,6 +34,7 @@ static String toArchString(Architecture architecture) {
             case X86_64 -> "x64";
             case AARCH64 -> "aarch64";
             case X86 -> "x86";
+            default -> throw new UnsupportedOperationException("Architecture " + architecture);
         };
     }
 

diff --git a/build-tools-internal/src/main/resources/minimumGradleVersion b/build-tools-internal/src/main/resources/minimumGradleVersion
@@ -1 +1 @@
-8.10.2
+8.11.1
diff --git a/build-tools/src/main/java/org/elasticsearch/gradle/LazyFileOutputStream.java b/build-tools/src/main/java/org/elasticsearch/gradle/LazyFileOutputStream.java
@@ -39,6 +39,12 @@ public void write(byte b[], int off, int len) throws IOException {
                 bootstrap();
                 delegate.write(b, off, len);
             }
+
+            @Override
+            public void write(byte b[]) throws IOException {
+                bootstrap();
+                delegate.write(b);
+            }
         };
     }
 

diff --git a/build-tools/src/main/java/org/elasticsearch/gradle/plugin/BasePluginBuildPlugin.java b/build-tools/src/main/java/org/elasticsearch/gradle/plugin/BasePluginBuildPlugin.java
@@ -24,6 +24,8 @@
 import org.gradle.api.Task;
 import org.gradle.api.Transformer;
 import org.gradle.api.artifacts.type.ArtifactTypeDefinition;
+import org.gradle.api.attributes.Attribute;
+import org.gradle.api.attributes.LibraryElements;
 import org.gradle.api.file.CopySpec;
 import org.gradle.api.file.FileCollection;
 import org.gradle.api.file.RegularFile;
@@ -126,9 +128,27 @@ private TaskProvider<Zip> createBundleTasks(final Project project, PluginPropert
         // know about the plugin (used by test security code to statically initialize the plugin in unit tests)
         var testSourceSet = project.getExtensions().getByType(SourceSetContainer.class).getByName("test");
         Map<String, Object> map = Map.of("builtBy", buildProperties);
-        testSourceSet.getOutput().dir(map, new File(project.getBuildDir(), "generated-resources"));
+
+        File generatedResources = new File(project.getBuildDir(), "generated-resources");
+        testSourceSet.getOutput().dir(map, generatedResources);
         testSourceSet.getResources().srcDir(pluginMetadata);
 
+        // expose the plugin properties and metadata for other plugins to use in their tests.
+        // See TestWithDependenciesPlugin for how this is used.
+        project.getConfigurations().create("pluginMetadata", conf -> {
+            conf.getAttributes().attribute(Attribute.of("pluginMetadata", Boolean.class), true);
+            conf.getAttributes()
+                .attribute(
+                    LibraryElements.LIBRARY_ELEMENTS_ATTRIBUTE,
+                    project.getObjects().named(LibraryElements.class, LibraryElements.RESOURCES)
+                );
+        });
+
+        project.getArtifacts().add("pluginMetadata", new File(project.getBuildDir(), "generated-descriptor"), artifact -> {
+            artifact.builtBy(buildProperties);
+        });
+        project.getArtifacts().add("pluginMetadata", pluginMetadata);
+        // getAttributes().attribute(ArtifactTypeDefinition.ARTIFACT_TYPE_ATTRIBUTE, "plugin-metadata");
         var bundleSpec = createBundleSpec(project, pluginMetadata, buildProperties);
         extension.setBundleSpec(bundleSpec);
         // create the actual bundle task, which zips up all the files for the plugin

diff --git a/distribution/packages/build.gradle b/distribution/packages/build.gradle
@@ -44,7 +44,7 @@ import java.util.regex.Pattern
  */
 
 plugins {
-  id "com.netflix.nebula.ospackage-base" version "11.9.1"
+  id "com.netflix.nebula.ospackage-base" version "11.10.0"
 }
 
 ['deb', 'rpm'].each { type ->
@@ -196,7 +196,7 @@ def commonPackageConfig(String type, String architecture) {
     configurationFile '/etc/elasticsearch/users_roles'
     from("${packagingFiles}") {
       dirPermissions {
-        unix(02750)
+        unix(0750)
       }
       into('/etc')
       permissionGroup 'elasticsearch'
@@ -209,7 +209,7 @@ def commonPackageConfig(String type, String architecture) {
     from("${packagingFiles}/etc/elasticsearch") {
       into('/etc/elasticsearch')
       dirPermissions {
-        unix(02750)
+        unix(0750)
       }
       setgid = true
       filePermissions {
@@ -261,7 +261,7 @@ def commonPackageConfig(String type, String architecture) {
 
     // ========= empty dirs =========
     // NOTE: these are created under packagingFiles as empty, but the permissions are set here
-    Closure copyEmptyDir = { path, u, g, mode ->
+    Closure copyEmptyDir = { path, u, g, gid, mode ->
       File file = new File(path)
       into(file.parent) {
         from "${packagingFiles}/${file.parent}"
@@ -273,12 +273,12 @@ def commonPackageConfig(String type, String architecture) {
         dirPermissions {
           unix(mode)
         }
-        setgid (mode == 02750)
+        setgid(gid)
       }
     }
-    copyEmptyDir('/var/log/elasticsearch', 'elasticsearch', 'elasticsearch', 02750)
-    copyEmptyDir('/var/lib/elasticsearch', 'elasticsearch', 'elasticsearch', 02750)
-    copyEmptyDir('/usr/share/elasticsearch/plugins', 'root', 'root', 0755)
+    copyEmptyDir('/var/log/elasticsearch', 'elasticsearch', 'elasticsearch', true, 0750)
+    copyEmptyDir('/var/lib/elasticsearch', 'elasticsearch', 'elasticsearch', true, 0750)
+    copyEmptyDir('/usr/share/elasticsearch/plugins', 'root', 'root', false, 0755)
 
     // the oss package conflicts with the default distribution and vice versa
     conflicts('elasticsearch-oss')

diff --git a/docs/changelog/117294.yaml b/docs/changelog/117294.yaml
@@ -0,0 +1,5 @@
+pr: 117294
+summary: Always Emit Inference ID in Semantic Text Mapping
+area: Mapping
+type: bug
+issues: []
diff --git a/docs/changelog/117297.yaml b/docs/changelog/117297.yaml
@@ -0,0 +1,5 @@
+pr: 117297
+summary: Fix CCS exchange when multi cluster aliases point to same cluster
+area: ES|QL
+type: bug
+issues: []
diff --git a/docs/reference/esql/functions/aggregation-functions.asciidoc b/docs/reference/esql/functions/aggregation-functions.asciidoc
@@ -20,7 +20,7 @@ The <<esql-stats-by>> command supports these aggregate functions:
 * <<esql-sum>>
 * <<esql-top>>
 * <<esql-values>>
-* experimental:[] <<esql-weighted_avg>>
+* <<esql-weighted_avg>>
 // end::agg_list[]
 
 include::layout/avg.asciidoc[]

diff --git a/docs/reference/esql/functions/mv-functions.asciidoc b/docs/reference/esql/functions/mv-functions.asciidoc
@@ -19,6 +19,7 @@
 * <<esql-mv_median>>
 * <<esql-mv_median_absolute_deviation>>
 * <<esql-mv_min>>
+* <<esql-mv_percentile>>
 * <<esql-mv_pseries_weighted_sum>>
 * <<esql-mv_sort>>
 * <<esql-mv_slice>>
@@ -37,6 +38,7 @@ include::layout/mv_max.asciidoc[]
 include::layout/mv_median.asciidoc[]
 include::layout/mv_median_absolute_deviation.asciidoc[]
 include::layout/mv_min.asciidoc[]
+include::layout/mv_percentile.asciidoc[]
 include::layout/mv_pseries_weighted_sum.asciidoc[]
 include::layout/mv_slice.asciidoc[]
 include::layout/mv_sort.asciidoc[]