diff --git a/depclean-gradle-plugin/src/main/java/se/kth/depclean/DepCleanGradleAction.java b/depclean-gradle-plugin/src/main/java/se/kth/depclean/DepCleanGradleAction.java index 450c9819..db0904d8 100644 --- a/depclean-gradle-plugin/src/main/java/se/kth/depclean/DepCleanGradleAction.java +++ b/depclean-gradle-plugin/src/main/java/se/kth/depclean/DepCleanGradleAction.java @@ -1,5 +1,7 @@ package se.kth.depclean; +import com.google.common.collect.ArrayListMultimap; +import com.google.common.collect.Multimap; import java.io.File; import java.io.IOException; import java.nio.file.Path; @@ -16,6 +18,7 @@ import lombok.SneakyThrows; import org.apache.commons.io.FileUtils; import org.gradle.api.Action; +import org.gradle.api.GradleException; import org.gradle.api.Project; import org.gradle.api.artifacts.Configuration; import org.gradle.api.artifacts.ResolvedArtifact; @@ -24,12 +27,12 @@ import org.gradle.api.logging.Logger; import org.jetbrains.annotations.NotNull; import se.kth.depclean.utils.DependencyUtils; +import se.kth.depclean.utils.GradleWritingUtils; import se.kth.depclean.analysis.DefaultGradleProjectDependencyAnalyzer; import se.kth.depclean.analysis.GradleProjectDependencyAnalysis; import se.kth.depclean.core.analysis.ProjectDependencyAnalyzerException; import se.kth.depclean.util.JarUtils; - /** * Depclean default and only action. */ @@ -49,12 +52,41 @@ public class DepCleanGradleAction implements Action { */ private static final Map SizeOfDependencies = new HashMap<>(); + // Extensions fields ===================================== + private Project project; + private boolean skipDepClean; + private boolean isIgnoreTest; + private boolean failIfUnusedDirect; + private boolean failIfUnusedTransitive; + private boolean failIfUnusedInherited; + private boolean createBuildDebloated; + // TODO : The implementation of next two parameters will be done later. + private boolean createResultJson; + private boolean createClassUsageCsv; + private Set ignoreConfiguration; + private Set ignoreDependencies; + @SneakyThrows @Override public void execute(@NotNull Project project) { Logger logger = project.getLogger(); + // If the user provided some configuration. + DepCleanGradlePluginExtension extension = project.getExtensions() + .getByType(DepCleanGradlePluginExtension.class); + getPluginExtensions(extension); + + if (skipDepClean) { + logger.lifecycle("Skipping DepClean plugin execution"); + return; + } + + // If the project is not the default one. + if (this.project != null) { + project = this.project; + } + // Path to the project directory. final Path projectDirPath = Paths.get(project.getProjectDir().getAbsolutePath()); @@ -130,7 +162,6 @@ public void execute(@NotNull Project project) { /* Decompress dependencies */ decompressDependencies(dependencyDirectory, dependencyDirPath.toString()); - final boolean isIgnoreTest = true; /* Analyze dependencies usage status */ GradleProjectDependencyAnalysis projectDependencyAnalysis = null; DefaultGradleProjectDependencyAnalyzer dependencyAnalyzer = @@ -203,6 +234,28 @@ public void execute(@NotNull Project project) { unusedTransitiveArtifactsCoordinates.removeAll(unusedDirectArtifactsCoordinates); unusedTransitiveArtifactsCoordinates.removeAll(unusedInheritedArtifactsCoordinates); + // Exclude dependencies with specific scopes from the post analysis result. + if (ignoreConfiguration != null) { + usedDirectArtifactsCoordinates = excludeConfiguration(usedDirectArtifactsCoordinates); + usedTransitiveArtifactsCoordinates = excludeConfiguration(usedTransitiveArtifactsCoordinates); + usedInheritedArtifactsCoordinates = excludeConfiguration(usedInheritedArtifactsCoordinates); + unusedDirectArtifactsCoordinates = excludeConfiguration(unusedDirectArtifactsCoordinates); + unusedTransitiveArtifactsCoordinates = excludeConfiguration(unusedTransitiveArtifactsCoordinates); + unusedInheritedArtifactsCoordinates = excludeConfiguration(unusedInheritedArtifactsCoordinates); + } + + // Excluding dependencies ignored by the user from post analysis result. + // TODO : If a direct dependency is ignored by the user then it' corresponding + // transitive and inherited dependencies should also be ignore. + if (ignoreDependencies != null) { + usedDirectArtifactsCoordinates = excludeDependencies(usedDirectArtifactsCoordinates); + usedTransitiveArtifactsCoordinates = excludeDependencies(usedTransitiveArtifactsCoordinates); + usedInheritedArtifactsCoordinates = excludeDependencies(usedInheritedArtifactsCoordinates); + unusedDirectArtifactsCoordinates = excludeDependencies(unusedDirectArtifactsCoordinates); + unusedTransitiveArtifactsCoordinates = excludeDependencies(unusedTransitiveArtifactsCoordinates); + unusedInheritedArtifactsCoordinates = excludeDependencies(unusedInheritedArtifactsCoordinates); + } + /* Printing the results to the terminal */ printString(SEPARATOR); printString(" D E P C L E A N A N A L Y S I S R E S U L T S"); @@ -230,6 +283,144 @@ public void execute(@NotNull Project project) { + " [" + allUnresolvedDependencies.size() + "]" + ": "); allUnresolvedDependencies.forEach(s -> printString("\t" + s)); } + + // Configurations ignored by the depclean analysis on user's wish. + if (ignoreConfiguration != null) { + printString( + "\nConfigurations ignored in the analysis by the user : " + + " [" + ignoreConfiguration.size() + "]" + ": "); + ignoreConfiguration.forEach(s -> printString("\t" + s)); + } + + // Dependencies ignored by depclean analysis on user's wish. + if (ignoreDependencies != null) { + printString( + "\nDependencies ignored in the analysis by the user" + + " [" + ignoreDependencies.size() + "]" + ": "); + ignoreDependencies.forEach(s -> printString("\t" + s)); + } + + /* Fail the build if there are unused direct dependencies */ + if (failIfUnusedDirect && !unusedDirectArtifactsCoordinates.isEmpty()) { + throw new GradleException("Build failed due to unused direct dependencies" + + " in the dependency tree of the project."); + } + + /* Fail the build if there are unused direct dependencies */ + if (failIfUnusedTransitive && !unusedTransitiveArtifactsCoordinates.isEmpty()) { + throw new GradleException("Build failed due to unused transitive dependencies" + + " in the dependency tree of the project."); + } + + /* Fail the build if there are unused direct dependencies */ + if (failIfUnusedInherited && !unusedInheritedArtifactsCoordinates.isEmpty()) { + throw new GradleException("Build failed due to unused inherited dependencies" + + " in the dependency tree of the project."); + } + + /* Writing the debloated version of the pom */ + if (createBuildDebloated) { + logger.lifecycle("Starting debloating dependencies"); + + // All dependencies which will be added directly to the desired file. + Set dependenciesToAdd = new HashSet<>(); + + /* Adding used direct dependencies */ + try { + logger + .lifecycle("Adding " + usedDirectArtifacts.size() + + " used direct dependencies"); + dependenciesToAdd.addAll(usedDirectArtifacts); + } catch (Exception e) { + throw new GradleException(e.getMessage(), e); + } + + /* Add used transitive as direct dependencies */ + try { + if (!usedTransitiveArtifacts.isEmpty()) { + logger + .lifecycle("Adding " + usedTransitiveArtifacts.size() + + " used transitive dependencies as direct dependencies."); + dependenciesToAdd.addAll(usedTransitiveArtifacts); + } + } catch (Exception e) { + throw new GradleException(e.getMessage(), e); + } + + /* Exclude unused transitive dependencies */ + + /* A multi-map [parent] -> [child] i.e. this will keep a track of from which dependency + the unused transitive dependencies should be excluded. Also, here multi-map is preferred + as one transitive dependency can have more than one parent. */ + Multimap excludedTransitiveArtifactsMap = ArrayListMultimap.create(); + + // A set that contains all the transitive children of project's dependencies. + Set allChildren = getAllChildren(allDependencies); + try { + if (!unusedTransitiveArtifacts.isEmpty()) { + logger + .lifecycle("Excluding " + unusedTransitiveArtifactsCoordinates.size() + + " unused transitive dependencies one-by-one."); + for (String artifact : unusedTransitiveArtifactsCoordinates) { + String unusedTransitiveDependencyId = getArtifactGroupArtifactId(artifact); + for (ResolvedDependency dependency : allChildren) { + if (dependency.getName().equals(unusedTransitiveDependencyId)) { + // i.e. this dependency should be excluded from all it's parents. + Set parents = dependency.getParents(); + parents.forEach(s -> excludedTransitiveArtifactsMap + .put(s.getName(), unusedTransitiveDependencyId)); + break; // Not need to check further. + } + } + } + } + } catch (Exception e) { + throw new GradleException(e.getMessage(), e); + } + + /* Write the debloated-dependencies.gradle file */ + final Path pathToDebloatedDependencies = projectDirPath.resolve("debloated-dependencies.gradle"); + File debloatedDependencies = pathToDebloatedDependencies.toFile(); + try { + // Delete the previous existence (if exist). + if (debloatedDependencies.exists()) { + se.kth.depclean.util.FileUtils.forceDelete(debloatedDependencies); + debloatedDependencies.createNewFile(); + } + } catch (IOException e) { + e.printStackTrace(); + } + + try { + GradleWritingUtils.writeGradle(debloatedDependencies, + dependenciesToAdd, + excludedTransitiveArtifactsMap); + } catch (IOException e) { + throw new GradleException(e.getMessage(), e); + } + logger.lifecycle("Dependencies debloated successfully"); + logger.lifecycle("debloated-dependencies.gradle file created in: " + + pathToDebloatedDependencies); + } + } + + /** + * A utility method to get the additional configuration of the plugin. + * + * @param extension Plugin extension class. + */ + public void getPluginExtensions(final DepCleanGradlePluginExtension extension) { + this.project = extension.getProject(); + this.skipDepClean = extension.isSkipDepClean(); + this.isIgnoreTest = extension.isIgnoreTest(); + this.failIfUnusedDirect = extension.isFailIfUnusedDirect(); + this.failIfUnusedTransitive = extension.isFailIfUnusedTransitive(); + this.failIfUnusedInherited = extension.isFailIfUnusedInherited(); + this.createBuildDebloated = extension.isCreateBuildDebloated(); + this.createResultJson = extension.isCreateResultJson(); + this.createClassUsageCsv = extension.isCreateClassUsageCsv(); + this.ignoreConfiguration = extension.getIgnoreConfiguration(); + this.ignoreDependencies = extension.getIgnoreDependency(); } /** @@ -374,4 +565,64 @@ public static String getName(final ResolvedArtifact artifact) { String[] artifactGroupArtifactId = artifactGroupArtifactIds[1].split("\\)"); return artifactGroupArtifactId[0] + ":" + ArtifactConfigurationMap.get(artifact); } + + /** + * Remove those artifacts coordinates which belong to the configuration, ignored by the user. + * + * @param artifactCoordinates Coordinates of the artifact. + * @return Un-ignored coordinates. + */ + public Set excludeConfiguration(final Set artifactCoordinates) { + Set nonExcludedConfigurations = new HashSet<>(); + for (String coordinates : artifactCoordinates) { + String configuration = coordinates.split(":")[3]; + if (!ignoreConfiguration.contains(configuration)) { + nonExcludedConfigurations.add(coordinates); + } + } + return nonExcludedConfigurations; + } + + /** + * Remove those artifact coordinates which are ignores by the user. + * + * @param artifactCoordinates Coordinates of the artifact. + * @return Un-ignored coordinates. + */ + public Set excludeDependencies(final Set artifactCoordinates) { + Set nonExcludedDependencies = new HashSet<>(); + for (String coordinates : artifactCoordinates) { + if (!ignoreDependencies.contains(coordinates)) { + nonExcludedDependencies.add(coordinates); + ignoreDependencies.remove(coordinates); + } + } + return nonExcludedDependencies; + } + + /** + * Get coordinates(name) without scopes or configuration. + * + * @param artifact Artifact + * @return Name of artifact without scope. + */ + public static String getArtifactGroupArtifactId(final String artifact) { + String[] parts = artifact.split(":"); + return parts[0] + ":" + parts[1] + ":" + parts[2]; + } + + /** + * Get all the transitive children of all the project's dependencies. + * + * @param allDependencies Set of all dependencies + * @return Set of all children + */ + public Set getAllChildren(final Set allDependencies) { + Set allChildren = new HashSet<>(); + for (ResolvedDependency dependency : allDependencies) { + allChildren.addAll(dependency.getChildren()); + } + return allChildren; + } + } diff --git a/depclean-gradle-plugin/src/main/java/se/kth/depclean/DepCleanGradlePluginExtension.java b/depclean-gradle-plugin/src/main/java/se/kth/depclean/DepCleanGradlePluginExtension.java new file mode 100644 index 00000000..ee0f6e16 --- /dev/null +++ b/depclean-gradle-plugin/src/main/java/se/kth/depclean/DepCleanGradlePluginExtension.java @@ -0,0 +1,115 @@ +package se.kth.depclean; + +import java.util.Set; +import org.gradle.api.Project; + +/** + * This extension class allows you to add optional parameters to the default (debloat) task. + */ +public class DepCleanGradlePluginExtension { + + /** + * The Gradle project to analyze. + */ + public Project project = null; + + /** + * Skip plugin execution completely. + */ + public boolean skipDepClean = false; + + /** + * If this is true, DepClean will not analyze the test sources in the project, and, therefore, + * the dependencies that are only used for testing will be considered unused. This property is + * useful to detect dependencies that have a compile scope but are only used during testing. + * Hence, these dependencies should have a test scope. + */ + public boolean ignoreTest = false; + + /** + * If this is true, and DepClean reported any unused direct dependency in the dependency tree, + * then the project's build fails immediately after running DepClean. + */ + public boolean failIfUnusedDirect = false; + + /** + * If this is true, and DepClean reported any unused transitive dependency in the dependency tree, + * then the project's build fails immediately after running DepClean. + */ + public boolean failIfUnusedTransitive = false; + + /** + * If this is true, and DepClean reported any unused inherited dependency in the dependency tree, + * then the project's build fails immediately after running DepClean. + */ + public boolean failIfUnusedInherited = false; + + /** + * If this is true, DepClean creates a debloated version of the build.gradle without + * unused dependencies, called "debloated-build.gradle", in root of the project. + */ + public boolean createBuildDebloated = false; + + /** + * If this is true, DepClean creates a JSON file with the result of the analysis. + * The file is called "debloat-result.json" and it is located in /build. + */ + public boolean createResultJson = false; + + /** + * If this is true, DepClean creates a CSV file with the result of the analysis + * with the columns: OriginClass,TargetClass,Dependency. The file is called + * "class-usage.csv" and it is located in /target. + */ + public boolean createClassUsageCsv; + + /** + * Ignore dependencies with specific configurations from the DepClean analysis. + */ + public Set ignoreConfiguration; + + /** + * Add a list of dependencies, identified by their coordinates, to be ignored by DepClean during + * the analysis and considered as used dependencies. Useful to override incomplete result caused + * by bytecode-level analysis Dependency format is groupId:artifactId:version. + */ + public Set ignoreDependency; + + // Getters ========================================== + + public Project getProject() { + return project; + } + + public boolean isSkipDepClean() { + return skipDepClean; + } + + public boolean isIgnoreTest() { + return ignoreTest; + } + + public boolean isFailIfUnusedDirect() { return failIfUnusedDirect; } + + public boolean isFailIfUnusedTransitive() { + return failIfUnusedTransitive; + } + + public boolean isFailIfUnusedInherited() { + return failIfUnusedInherited; + } + + public boolean isCreateBuildDebloated() { return createBuildDebloated; } + + public boolean isCreateResultJson() { return createResultJson; } + + public boolean isCreateClassUsageCsv() { return createClassUsageCsv; } + + public Set getIgnoreConfiguration() { + return ignoreConfiguration; + } + + public Set getIgnoreDependency() { + return ignoreDependency; + } +} diff --git a/depclean-gradle-plugin/src/main/java/se/kth/depclean/utils/GradleWritingUtils.java b/depclean-gradle-plugin/src/main/java/se/kth/depclean/utils/GradleWritingUtils.java new file mode 100644 index 00000000..993f2ff9 --- /dev/null +++ b/depclean-gradle-plugin/src/main/java/se/kth/depclean/utils/GradleWritingUtils.java @@ -0,0 +1,138 @@ +package se.kth.depclean.utils; + +import com.google.common.collect.ArrayListMultimap; +import com.google.common.collect.Multimap; +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.io.PrintWriter; +import java.util.Collection; +import java.util.HashSet; +import java.util.Set; +import org.gradle.api.artifacts.ResolvedArtifact; +import se.kth.depclean.DepCleanGradleAction; + +public class GradleWritingUtils { + + /** + * Writes the debloated-dependencies.gradle. + * + * @param file Target + * @param dependenciesToAdd Direct dependencies to be written directly. + * @param excludedTransitiveArtifactsMap Map [dependency] -> [excluded transitive child] + * @throws IOException In case of IO issues. + */ + public static void writeGradle(final File file, final Set dependenciesToAdd, + final Multimap excludedTransitiveArtifactsMap) + throws IOException { + /* A multi-map [configuration] -> [dependency] */ + Multimap configurationDependencyMap = getNewConfigurations(dependenciesToAdd); + + /* Writing starts */ + FileWriter fileWriter = new FileWriter(file, true); + BufferedWriter bufferedWriter = new BufferedWriter(fileWriter); + PrintWriter writer = new PrintWriter(bufferedWriter); + + writer.println("dependencies {"); + + for (String configuration : configurationDependencyMap.keySet()) { + writer.print("\t" + configuration); + + /* Getting all the dependencies with specified configuration and converting + it to an array for ease in writing. */ + Collection dependency = configurationDependencyMap.get(configuration); + String[] dep = dependency.toArray(new String[dependency.size()]); + + /* Writing those dependencies which do not have to exclude any dependency(s). + Simultaneously, also getting those dependencies which have to exclude + some transitive dependencies. */ + Set excludeChildrenDependencies = + writeNonExcluded(writer, dep, excludedTransitiveArtifactsMap); + + /* Writing those dependencies which have to exclude any dependency(s). */ + if (!excludeChildrenDependencies.isEmpty()) { + writeExcluded(writer, configuration, + excludeChildrenDependencies, excludedTransitiveArtifactsMap); + } + } + writer.println("}"); + writer.close(); + } + + // TODO: To modify later. + /** + * There are some dependencies configurations that are removed by Gradle above 7.0.0, + * like runtime converted to implementation. To know more visit here., but still + * $dependencies.getConfiguration() still returns those deprecated scopes.
+ * So, currently we just divide dependencies into two parts i.e. implementation + * & testImplementation. + * + * @param dependenciesToAdd All dependencies to be added. + * @return A multi-map with value as a dependency and key as it's configuration. + */ + public static Multimap getNewConfigurations( + final Set dependenciesToAdd) { + Multimap configurationDependencyMap = ArrayListMultimap.create(); + for (ResolvedArtifact artifact : dependenciesToAdd) { + String artifactName = DepCleanGradleAction.getName(artifact); + String dependency = DepCleanGradleAction.getArtifactGroupArtifactId(artifactName); + String oldConfiguration = artifactName.split(":")[3]; + String configuration = + oldConfiguration.startsWith("test") || oldConfiguration.endsWith("Elements") + ? "testImplementation" : "implementation"; + configurationDependencyMap.put(configuration, dependency); + } + return configurationDependencyMap; + } + + /** + * Writes those dependencies which don't have to exclude any transitive dependencies of their own. + * Simultaneously, it also returns the set of dependencies which have to exclude some + * transitive dependencies to write them separately. + * + * @param writer For writing. + * @param dep Dependencies to be printed. + * @param excludedTransitiveArtifactsMap [dependency] -> [excluded transitive dependencies]. + * @return A set of dependencies. + */ + public static Set writeNonExcluded(final PrintWriter writer, + final String[] dep, + final Multimap excludedTransitiveArtifactsMap) { + Set excludeChildrenDependencies = new HashSet<>(); + int size = dep.length - 1; + for (int i = 0; i < size; i++) { + if (excludedTransitiveArtifactsMap.containsKey(dep[i])) { + excludeChildrenDependencies.add(dep[i]); + } else { + writer.println("\t\t\t'" + dep[i] + "',"); + } + } + writer.println("\t\t\t'" + dep[size] + "'\n"); + return excludeChildrenDependencies; + } + + /** + * Writes those dependencies which have to exclude some of their transitive dependency(s). + * + * @param writer For writing. + * @param configuration Corresponding configuration. + * @param excludeChildrenDependencies Transitive dependencies to be excluded. + * @param excludedTransitiveArtifactsMap [dependency] -> [excluded transitive dependencies]. + */ + public static void writeExcluded(final PrintWriter writer, + final String configuration, + final Set excludeChildrenDependencies, + final Multimap excludedTransitiveArtifactsMap) { + for (String excludeDep : excludeChildrenDependencies) { + writer.println("\t" + configuration + " ('" + excludeDep + "') {"); + Collection excludeDependencies = excludedTransitiveArtifactsMap.get(excludeDep); + excludeDependencies.forEach(s -> + writer.println("\t\t\texclude group: '" + s.split(":")[0] + + "', module: '" + s.split(":")[1] + "'")); + writer.println("\t}"); + } + } + +}