HDDS-11591. Copy dependencies when building each module (apache#7325)

aswinshakil · Oct 29, 2024 · e85b32d · e85b32d
1 parent 72e56d7
commit e85b32d
Show file tree

Hide file tree

Showing 16 changed files with 61 additions and 101 deletions.
diff --git a/hadoop-hdds/pom.xml b/hadoop-hdds/pom.xml
@@ -322,36 +322,5 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd">
         </plugins>
       </build>
     </profile>
-    <profile>
-      <id>add-classpath-descriptor</id>
-      <activation>
-        <file>
-          <exists>src/main/java</exists>
-        </file>
-      </activation>
-      <build>
-        <plugins>
-          <plugin>
-            <groupId>org.apache.maven.plugins</groupId>
-            <artifactId>maven-dependency-plugin</artifactId>
-            <executions>
-              <execution>
-                <id>add-classpath-descriptor</id>
-                <phase>prepare-package</phase>
-                <goals>
-                  <goal>build-classpath</goal>
-                </goals>
-                <configuration>
-                  <outputFile>${project.build.outputDirectory}/${project.artifactId}.classpath</outputFile>
-                  <prefix>$HDDS_LIB_JARS_DIR</prefix>
-                  <outputFilterFile>true</outputFilterFile>
-                  <includeScope>runtime</includeScope>
-                </configuration>
-              </execution>
-            </executions>
-          </plugin>
-        </plugins>
-      </build>
-    </profile>
   </profiles>
 </project>
diff --git a/hadoop-hdds/rocks-native/pom.xml b/hadoop-hdds/rocks-native/pom.xml
@@ -385,7 +385,7 @@
             <artifactId>maven-dependency-plugin</artifactId>
             <executions>
               <execution>
-                <id>copy-jars</id>
+                <id>copy-dependencies</id>
                 <phase>process-sources</phase>
                 <goals>
                   <goal>copy-dependencies</goal>

diff --git a/hadoop-hdds/server-scm/pom.xml b/hadoop-hdds/server-scm/pom.xml
@@ -29,6 +29,7 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd">
   <packaging>jar</packaging>
 
   <properties>
+    <classpath.skip>false</classpath.skip>
   </properties>
 
   <dependencies>

diff --git a/hadoop-ozone/csi/pom.xml b/hadoop-ozone/csi/pom.xml
@@ -29,6 +29,7 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
   <packaging>jar</packaging>
 
   <properties>
+    <classpath.skip>false</classpath.skip>
     <maven.test.skip>true</maven.test.skip> <!-- no tests in this module so far -->
   </properties>
 

diff --git a/hadoop-ozone/datanode/pom.xml b/hadoop-ozone/datanode/pom.xml
@@ -27,6 +27,7 @@
   <version>2.0.0-SNAPSHOT</version>
 
   <properties>
+    <classpath.skip>false</classpath.skip>
     <maven.test.skip>true</maven.test.skip> <!-- no tests in this module so far -->
     <spotbugs.skip>true</spotbugs.skip>
   </properties>

diff --git a/hadoop-ozone/dev-support/checks/dependency.sh b/hadoop-ozone/dev-support/checks/dependency.sh
@@ -32,8 +32,8 @@ cp ${src_dir}/current.txt "$REPORT_DIR"/
 
 #implementation of sort cli is not exactly the same everywhere. It's better to sort with the same command locally
 (diff -uw \
-  <(sort ${src_dir}/jar-report.txt) \
-  <(sort ${src_dir}/current.txt) \
+  <(sort -u ${src_dir}/jar-report.txt) \
+  <(sort -u ${src_dir}/current.txt) \
   || true) \
   > "$REPORT_FILE"
 

diff --git a/hadoop-ozone/dist/dev-support/bin/dist-layout-stitching b/hadoop-ozone/dist/dev-support/bin/dist-layout-stitching
@@ -71,6 +71,8 @@ run cp -p "${ROOT}/HISTORY.md" .
 run cp -p "${ROOT}/SECURITY.md" .
 run cp -p "${ROOT}/CONTRIBUTING.md" .
 
+run mkdir -p ./share/ozone/classpath
+run mkdir -p ./share/ozone/lib
 run mkdir -p ./share/ozone/web
 run mkdir -p ./bin
 run mkdir -p ./sbin
@@ -129,6 +131,14 @@ run cp -p -r "${ROOT}/hadoop-ozone/dist/target/Dockerfile" .
 #Copy pre-generated keytabs
 run cp -p -R "${ROOT}/hadoop-ozone/dist/src/main/keytabs" compose/_keytabs
 
+for file in $(find "${ROOT}" -path '*/target/classes/*.classpath' | sort); do
+  cp -n -p -v "$file" share/ozone/classpath/
+done
+
+for file in $(find "${ROOT}" -path '*/share/ozone/lib/*jar' | sort); do
+  cp -n -p -v "$file" share/ozone/lib/
+done
+
 #workaround for https://issues.apache.org/jira/browse/MRESOURCES-236
 find ./compose -name "*.sh" -exec chmod 755 {} \;
 find ./kubernetes -name "*.sh" -exec chmod 755 {} \;
diff --git a/hadoop-ozone/dist/pom.xml b/hadoop-ozone/dist/pom.xml
@@ -71,24 +71,7 @@
         <artifactId>maven-dependency-plugin</artifactId>
         <executions>
           <execution>
-            <id>copy-classpath-files</id>
-            <phase>prepare-package</phase>
-            <goals>
-              <goal>unpack-dependencies</goal>
-            </goals>
-            <configuration>
-              <outputDirectory>
-                target/ozone-${ozone.version}/share/ozone/classpath
-              </outputDirectory>
-              <includes>*.classpath</includes>
-              <includeArtifactIds>
-                hdds-server-scm,ozone-common,ozone-csi,ozone-datanode,ozone-httpfsgateway,
-                ozone-insight,ozone-manager,ozone-recon,ozone-s3gateway,ozone-tools,hdds-rocks-native,ozone-s3-secret-store
-              </includeArtifactIds>
-            </configuration>
-          </execution>
-          <execution>
-            <id>copy-jars</id>
+            <id>copy-omitted-jars</id>
             <phase>prepare-package</phase>
             <goals>
               <goal>copy-dependencies</goal>
@@ -98,24 +81,6 @@
               <includeScope>runtime</includeScope>
             </configuration>
           </execution>
-          <execution>
-            <id>copy-omitted-jars</id>
-            <phase>prepare-package</phase>
-            <goals>
-              <goal>copy</goal>
-            </goals>
-            <configuration>
-              <outputDirectory>target/ozone-${ozone.version}/share/ozone/lib
-              </outputDirectory>
-              <artifactItems>
-                <artifactItem>
-                  <groupId>com.google.protobuf</groupId>
-                  <artifactId>protobuf-java</artifactId>
-                  <version>${grpc.protobuf-compile.version}</version>
-                </artifactItem>
-              </artifactItems>
-            </configuration>
-          </execution>
         </executions>
       </plugin>
       <plugin>

diff --git a/hadoop-ozone/httpfsgateway/pom.xml b/hadoop-ozone/httpfsgateway/pom.xml
@@ -32,6 +32,7 @@
   <description>Apache Ozone HttpFS</description>
 
   <properties>
+    <classpath.skip>false</classpath.skip>
     <httpfs.source.repository>REPO NOT AVAIL</httpfs.source.repository>
     <httpfs.source.revision>REVISION NOT AVAIL</httpfs.source.revision>
     <maven.build.timestamp.format>yyyy-MM-dd'T'HH:mm:ssZ</maven.build.timestamp.format>

diff --git a/hadoop-ozone/insight/pom.xml b/hadoop-ozone/insight/pom.xml
@@ -27,7 +27,9 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd">
   <description>Apache Ozone Insight Tool</description>
   <name>Apache Ozone Insight Tool</name>
   <packaging>jar</packaging>
+
   <properties>
+    <classpath.skip>false</classpath.skip>
   </properties>
 
   <dependencies>

diff --git a/hadoop-ozone/ozone-manager/pom.xml b/hadoop-ozone/ozone-manager/pom.xml
@@ -29,6 +29,7 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd">
   <packaging>jar</packaging>
 
   <properties>
+    <classpath.skip>false</classpath.skip>
   </properties>
 
   <dependencies>

diff --git a/hadoop-ozone/pom.xml b/hadoop-ozone/pom.xml
@@ -455,36 +455,5 @@
         </plugins>
       </build>
     </profile>
-    <profile>
-      <id>add-classpath-descriptor</id>
-      <activation>
-        <file>
-          <exists>src/main/java</exists>
-        </file>
-      </activation>
-      <build>
-        <plugins>
-          <plugin>
-            <groupId>org.apache.maven.plugins</groupId>
-            <artifactId>maven-dependency-plugin</artifactId>
-            <executions>
-              <execution>
-                <id>add-classpath-descriptor</id>
-                <phase>prepare-package</phase>
-                <goals>
-                  <goal>build-classpath</goal>
-                </goals>
-                <configuration>
-                  <outputFile>${project.build.outputDirectory}/${project.artifactId}.classpath</outputFile>
-                  <prefix>$HDDS_LIB_JARS_DIR</prefix>
-                  <outputFilterFile>true</outputFilterFile>
-                  <includeScope>runtime</includeScope>
-                </configuration>
-              </execution>
-            </executions>
-          </plugin>
-        </plugins>
-      </build>
-    </profile>
   </profiles>
 </project>
diff --git a/hadoop-ozone/recon/pom.xml b/hadoop-ozone/recon/pom.xml
@@ -24,6 +24,7 @@
   <modelVersion>4.0.0</modelVersion>
   <artifactId>ozone-recon</artifactId>
   <properties>
+    <classpath.skip>false</classpath.skip>
     <pnpm.version>8.15.7</pnpm.version>
   </properties>
   <build>

diff --git a/hadoop-ozone/s3gateway/pom.xml b/hadoop-ozone/s3gateway/pom.xml
@@ -26,6 +26,7 @@
   <packaging>jar</packaging>
   <version>2.0.0-SNAPSHOT</version>
   <properties>
+    <classpath.skip>false</classpath.skip>
     <file.encoding>UTF-8</file.encoding>
     <downloadSources>true</downloadSources>
   </properties>

diff --git a/hadoop-ozone/tools/pom.xml b/hadoop-ozone/tools/pom.xml
@@ -28,6 +28,10 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd">
   <name>Apache Ozone Tools</name>
   <packaging>jar</packaging>
 
+  <properties>
+    <classpath.skip>false</classpath.skip>
+  </properties>
+
   <dependencies>
     <dependency>
       <groupId>org.apache.ozone</groupId>

diff --git a/pom.xml b/pom.xml
@@ -98,6 +98,8 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xs
     <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
     <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
 
+    <classpath.skip>true</classpath.skip>
+
     <maven-gpg-plugin.version>3.2.7</maven-gpg-plugin.version>
     <shell-executable>bash</shell-executable>
 
@@ -1707,6 +1709,38 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xs
     </pluginManagement>
 
     <plugins>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-dependency-plugin</artifactId>
+        <executions>
+          <execution>
+            <id>add-classpath-descriptor</id>
+            <phase>prepare-package</phase>
+            <goals>
+              <goal>build-classpath</goal>
+            </goals>
+            <configuration>
+              <outputFile>${project.build.outputDirectory}/${project.artifactId}.classpath</outputFile>
+              <prefix>$HDDS_LIB_JARS_DIR</prefix>
+              <outputFilterFile>true</outputFilterFile>
+              <includeScope>runtime</includeScope>
+              <skip>${classpath.skip}</skip>
+            </configuration>
+          </execution>
+          <execution>
+            <id>copy-jars</id>
+            <phase>prepare-package</phase>
+            <goals>
+              <goal>copy-dependencies</goal>
+            </goals>
+            <configuration>
+              <outputDirectory>${project.build.directory}/share/ozone/lib</outputDirectory>
+              <includeScope>runtime</includeScope>
+              <skip>${classpath.skip}</skip>
+            </configuration>
+          </execution>
+        </executions>
+      </plugin>
       <plugin>
         <artifactId>maven-clean-plugin</artifactId>
         <configuration>