Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

use predictable ordering as much as possible #3317

Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions cli/src/main/java/org/owasp/dependencycheck/App.java
Original file line number Diff line number Diff line change
Expand Up @@ -21,9 +21,9 @@
import java.io.FileNotFoundException;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.TreeSet;

import org.apache.commons.cli.ParseException;
import org.apache.tools.ant.DirectoryScanner;
Expand Down Expand Up @@ -316,7 +316,7 @@ private int determineReturnCode(Engine engine, float cvssFailScore) {
* @return returns the set of identified files
*/
private Set<File> scanAntStylePaths(List<String> antStylePaths, int symLinkDepth, String[] excludes) {
final Set<File> paths = new HashSet<>();
final Set<File> paths = new TreeSet<>();
for (String file : antStylePaths) {
LOGGER.debug("Scanning {}", file);
final DirectoryScanner scanner = new DirectoryScanner();
Expand Down
16 changes: 9 additions & 7 deletions core/src/main/java/org/owasp/dependencycheck/Engine.java
Original file line number Diff line number Diff line change
Expand Up @@ -544,6 +544,7 @@ protected synchronized Dependency scanFile(@NotNull final File file, @Nullable f
for (Dependency existing : dependencies) {
if (sha1.equals(existing.getSha1sum())) {
if (existing.getFileName().contains(": ") || dependency.getFileName().contains(": ")) {
LOGGER.warn("continue for {} and {}", dependency.getFileName(), existing.getFileName());
//TODO this won't be quite right 100% of the time. Its possible that the ": " would get added later
continue;
}
Expand All @@ -553,7 +554,8 @@ protected synchronized Dependency scanFile(@NotNull final File file, @Nullable f
}
if (existing.getActualFilePath() != null && dependency.getActualFilePath() != null
&& !existing.getActualFilePath().equals(dependency.getActualFilePath())) {
existing.addRelatedDependency(dependency);
LOGGER.warn("adding {} as relatedDependancy to {}", dependency.getActualFilePath(), existing.getActualFilePath());
existing.addRelatedDependency(dependency);
} else {
dependency = existing;
}
Expand Down Expand Up @@ -763,14 +765,14 @@ protected synchronized List<AnalysisTask> getAnalysisTasks(Analyzer analyzer, Li
* @return the executor service
*/
protected ExecutorService getExecutorService(Analyzer analyzer) {
if (analyzer.supportsParallelProcessing()) {
final int maximumNumberOfThreads = Runtime.getRuntime().availableProcessors();
LOGGER.debug("Parallel processing with up to {} threads: {}.", maximumNumberOfThreads, analyzer.getName());
return Executors.newFixedThreadPool(maximumNumberOfThreads);
} else {
// if (analyzer.supportsParallelProcessing()) {
// final int maximumNumberOfThreads = Runtime.getRuntime().availableProcessors();
// LOGGER.debug("Parallel processing with up to {} threads: {}.", maximumNumberOfThreads, analyzer.getName());
// return Executors.newFixedThreadPool(maximumNumberOfThreads);
// } else {
LOGGER.debug("Parallel processing is not supported: {}.", analyzer.getName());
return Executors.newSingleThreadExecutor();
}
// }
}

/**
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,8 @@
import org.owasp.dependencycheck.Engine;
import org.owasp.dependencycheck.analyzer.exception.AnalysisException;
import org.owasp.dependencycheck.dependency.Dependency;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

/**
* <p>
Expand All @@ -41,6 +43,8 @@
@ThreadSafe
public abstract class AbstractDependencyComparingAnalyzer extends AbstractAnalyzer {

private static final Logger LOGGER = LoggerFactory.getLogger(AbstractDependencyComparingAnalyzer.class);

/**
* a flag indicating if this analyzer has run. This analyzer only runs once.
*/
Expand Down Expand Up @@ -91,6 +95,7 @@ protected synchronized void analyzeDependency(Dependency ignore, Engine engine)
}
Arrays.sort(dependencies, (l, r) -> (l.getDisplayFileName() + l.getActualFilePath()).compareTo(r.getDisplayFileName() + r.getActualFilePath()));
for (int x = 0; x < dependencies.length - 1; x++) {
// LOGGER.info(dependencies[x].getDisplayFileName() + " | " + dependencies[x].getActualFilePath());
final Dependency dependency = dependencies[x];
if (!dependenciesToRemove.contains(dependency)) {
for (int y = x + 1; y < dependencies.length; y++) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.TreeSet;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
import javax.annotation.concurrent.ThreadSafe;
Expand Down Expand Up @@ -797,7 +798,7 @@ protected boolean determineIdentifiers(Dependency dependency, String vendor, Str
final CpeBuilder cpeBuilder = new CpeBuilder();

final Set<CpePlus> cpePlusEntries = cve.getCPEs(vendor, product);
final Set<Cpe> cpes = filterEcosystem(dependency.getEcosystem(), cpePlusEntries);
final Set<Cpe> cpes = new TreeSet<>(filterEcosystem(dependency.getEcosystem(), cpePlusEntries));
if (cpes == null || cpes.isEmpty()) {
return false;
}
Expand All @@ -811,7 +812,7 @@ protected boolean determineIdentifiers(Dependency dependency, String vendor, Str
String bestGuessUpdate = null;
Confidence bestGuessConf = null;
String bestGuessURL = null;
final Set<IdentifierMatch> collected = new HashSet<>();
final Set<IdentifierMatch> collected = new TreeSet<>();

if (dependency.getVersion() != null && !dependency.getVersion().isEmpty()) {
//we shouldn't always use the dependency version - in some cases this causes FP
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
import com.github.packageurl.PackageURL;
import org.apache.commons.lang3.builder.EqualsBuilder;
import org.apache.commons.lang3.builder.HashCodeBuilder;
import org.jetbrains.annotations.NotNull;
import org.owasp.dependencycheck.data.nexus.MavenArtifact;
import org.owasp.dependencycheck.utils.Checksum;
import org.slf4j.Logger;
Expand All @@ -33,7 +34,6 @@
import java.security.NoSuchAlgorithmException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Optional;
import java.util.Set;
Expand All @@ -54,7 +54,7 @@
* @author Jeremy Long
*/
@ThreadSafe
public class Dependency extends EvidenceCollection implements Serializable {
public class Dependency extends EvidenceCollection implements Serializable, Comparable<Dependency> {

/**
* The serial version UID for serialization.
Expand Down Expand Up @@ -91,19 +91,19 @@ public class Dependency extends EvidenceCollection implements Serializable {
/**
* A set of vulnerabilities that have been suppressed.
*/
private final Set<Vulnerability> suppressedVulnerabilities = new HashSet<>();
private final Set<Vulnerability> suppressedVulnerabilities = new TreeSet<>();
/**
* A list of vulnerabilities for this dependency.
*/
private final Set<Vulnerability> vulnerabilities = new HashSet<>();
private final Set<Vulnerability> vulnerabilities = new TreeSet<>();
/**
* A collection of related dependencies.
*/
private final Set<Dependency> relatedDependencies = new HashSet<>();
private final Set<Dependency> relatedDependencies = new TreeSet<>();
/**
* A list of projects that reference this dependency.
*/
private final Set<String> projectReferences = new HashSet<>();
private final Set<String> projectReferences = new TreeSet<>();
/**
* A list of available versions.
*/
Expand Down Expand Up @@ -741,7 +741,7 @@ public synchronized void removeVulnerability(Vulnerability v) {
* @return the unmodifiable set of relatedDependencies
*/
public synchronized Set<Dependency> getRelatedDependencies() {
return Collections.unmodifiableSet(new HashSet<>(relatedDependencies));
return Collections.unmodifiableSet(new TreeSet<>(relatedDependencies));
}

/**
Expand All @@ -750,7 +750,7 @@ public synchronized Set<Dependency> getRelatedDependencies() {
* @return the unmodifiable set of projectReferences
*/
public synchronized Set<String> getProjectReferences() {
return Collections.unmodifiableSet(new HashSet<>(projectReferences));
return Collections.unmodifiableSet(new TreeSet<>(projectReferences));
}

/**
Expand Down Expand Up @@ -957,4 +957,18 @@ interface HashingFunction {
*/
String hash(File file) throws IOException, NoSuchAlgorithmException;
}


/**
* Implementation of the comparable interface.
*
* @param o the dependency being compared
* @return an integer indicating the ordering of the two objects
*/
@SuppressWarnings("deprecation")
@Override
public int compareTo(@NotNull Dependency o) {
return (o.getDisplayFileName() + o.getActualFilePath()).compareTo(this.getDisplayFileName() + this.getActualFilePath());
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,9 @@

import java.io.Serializable;
import java.util.Collections;
import java.util.HashSet;
import java.util.Set;
import java.util.TreeSet;

import javax.annotation.concurrent.ThreadSafe;
import org.apache.commons.lang3.builder.EqualsBuilder;
import org.apache.commons.lang3.builder.HashCodeBuilder;
Expand All @@ -41,23 +42,23 @@ class EvidenceCollection implements Serializable {
/**
* A collection of vendor evidence.
*/
private final Set<Evidence> vendors = new HashSet<>();
private final Set<Evidence> vendors = new TreeSet<>();
/**
* A collection of strings used to adjust Lucene's vendor term weighting.
*/
private final Set<String> vendorWeightings = new HashSet<>();
private final Set<String> vendorWeightings = new TreeSet<>();
/**
* A collection of product evidence.
*/
private final Set<Evidence> products = new HashSet<>();
private final Set<Evidence> products = new TreeSet<>();
/**
* A collection of strings used to adjust Lucene's product term weighting.
*/
private final Set<String> productWeightings = new HashSet<>();
/**
private final Set<String> productWeightings = new TreeSet<>();
/**TreeSet
* A collection of version evidence.
*/
private final Set<Evidence> versions = new HashSet<>();
private final Set<Evidence> versions = new TreeSet<>();

/**
* Used to iterate over highest confidence evidence contained in the
Expand Down Expand Up @@ -113,13 +114,13 @@ public synchronized Iterable<Evidence> getIterator(EvidenceType type, Confidence

switch (type) {
case VENDOR:
list = Collections.unmodifiableSet(new HashSet<>(vendors));
list = Collections.unmodifiableSet(new TreeSet<>(vendors));
break;
case PRODUCT:
list = Collections.unmodifiableSet(new HashSet<>(products));
list = Collections.unmodifiableSet(new TreeSet<>(products));
break;
case VERSION:
list = Collections.unmodifiableSet(new HashSet<>(versions));
list = Collections.unmodifiableSet(new TreeSet<>(versions));
break;
default:
return null;
Expand Down Expand Up @@ -248,7 +249,7 @@ public synchronized void addProductWeighting(String str) {
* @return an unmodifiable set of vendor weighting strings
*/
public synchronized Set<String> getVendorWeightings() {
return Collections.unmodifiableSet(new HashSet<>(vendorWeightings));
return Collections.unmodifiableSet(new TreeSet<>(vendorWeightings));
}

/**
Expand All @@ -259,7 +260,7 @@ public synchronized Set<String> getVendorWeightings() {
* @return an unmodifiable set of vendor weighting strings
*/
public synchronized Set<String> getProductWeightings() {
return Collections.unmodifiableSet(new HashSet<>(productWeightings));
return Collections.unmodifiableSet(new TreeSet<>(productWeightings));
}

/**
Expand All @@ -272,11 +273,11 @@ public synchronized Set<Evidence> getEvidence(EvidenceType type) {
if (null != type) {
switch (type) {
case VENDOR:
return Collections.unmodifiableSet(new HashSet<>(vendors));
return Collections.unmodifiableSet(new TreeSet<>(vendors));
case PRODUCT:
return Collections.unmodifiableSet(new HashSet<>(products));
return Collections.unmodifiableSet(new TreeSet<>(products));
case VERSION:
return Collections.unmodifiableSet(new HashSet<>(versions));
return Collections.unmodifiableSet(new TreeSet<>(versions));
default:
break;
}
Expand All @@ -290,7 +291,7 @@ public synchronized Set<Evidence> getEvidence(EvidenceType type) {
* @return the unmodifiable set of evidence
*/
public synchronized Set<Evidence> getEvidence() {
final Set<Evidence> e = new HashSet<>(vendors);
final Set<Evidence> e = new TreeSet<>(vendors);
e.addAll(products);
e.addAll(versions);
return Collections.unmodifiableSet(e);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -191,9 +191,7 @@ public int compareTo(@NotNull Identifier o) {
if (o instanceof CpeIdentifier) {
final CpeIdentifier other = (CpeIdentifier) o;
return new CompareToBuilder()
.append(this.cpe, other.cpe)
.append(this.url, other.getUrl())
.append(this.confidence, other.getConfidence())
.append(this.toString(), other.toString())
.toComparison();

}
Expand Down
8 changes: 4 additions & 4 deletions core/src/main/resources/templates/xmlReport.vsl
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
#**
This file is part of Dependency-Check.
This file is part of Dependency-Check.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
Expand Down Expand Up @@ -59,7 +59,7 @@ Copyright (c) 2018 Jeremy Long. All Rights Reserved.
#if ($groupID)
<groupID>$enc.xml($groupID)</groupID>
#end
#if ($artifactID)
#if ($artifactID)
<artifactID>$enc.xml($artifactID)</artifactID>
#end
#if ($applicationVersion)
Expand Down Expand Up @@ -257,7 +257,7 @@ Copyright (c) 2018 Jeremy Long. All Rights Reserved.
#end
</references>
<vulnerableSoftware>
#foreach($vs in $vuln.getVulnerableSoftware())
#foreach($vs in $vuln.getVulnerableSoftware(true))
<software#if($vs == $vuln.matchedVulnerableSoftware) vulnerabilityIdMatched="true"#end
#if($vs.versionStartIncluding) versionStartIncluding="$enc.xml($vs.versionStartIncluding)"#end
#if($vs.versionStartExcluding) versionStartExcluding="$enc.xml($vs.versionStartExcluding)"#end
Expand Down Expand Up @@ -337,7 +337,7 @@ Copyright (c) 2018 Jeremy Long. All Rights Reserved.
#if($vs.versionEndIncluding) versionEndIncluding="$enc.xml($vs.versionEndIncluding)"#end
#if($vs.versionEndExcluding) versionEndExcluding="$enc.xml($vs.versionEndExcluding)"#end
#if(!$vs.vulnerable) vulnerable="$vs.vulnerable"#end
>$enc.xml($vs.toCpe23FS())</software>
>$enc.xml($vs.toCpe23FS())</software>
#end
</vulnerableSoftware>
</suppressedVulnerability>
Expand Down