From f4a68083c9c667b6830d5d87430153dcff266552 Mon Sep 17 00:00:00 2001 From: Keshav Priyadarshi Date: Wed, 29 Mar 2023 15:09:44 +0530 Subject: [PATCH 1/5] Support retrieval of advisories for a CVE in DataSource - VendorData now includes PURL, necessary for organizing advisories received via CVE queries Signed-off-by: Keshav Priyadarshi --- vulntotal/validator.py | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/vulntotal/validator.py b/vulntotal/validator.py index 4601390cd..415678ba1 100644 --- a/vulntotal/validator.py +++ b/vulntotal/validator.py @@ -11,29 +11,48 @@ from typing import Iterable from typing import List +from packageurl import PackageURL + @dataclasses.dataclass(order=True) class VendorData: + purl: PackageURL aliases: List[str] = dataclasses.field(default_factory=list) affected_versions: List[str] = dataclasses.field(default_factory=list) fixed_versions: List[str] = dataclasses.field(default_factory=list) def to_dict(self): return { + "purl": str(self.purl), "affected_versions": self.affected_versions, "fixed_versions": self.fixed_versions, "aliases": self.aliases, } +class InvalidCVEError(Exception): + def __init__(self, message="CVE identifier must start with 'CVE-'"): + self.message = message + super().__init__(self.message) + + class DataSource: def __init__(self): self._raw_dump = [] - def datasource_advisory(self, purl) -> Iterable[VendorData]: + def datasource_advisory(self, purl: PackageURL) -> Iterable[VendorData]: + """ + Yield VendorData object for crossponding PURL. + """ + return NotImplementedError + + def datasource_advisory_from_cve(self, cve: str) -> Iterable[VendorData]: """ - Yield VendorData object corresponding to DataSource + Yield VendorData objects for a given CVE identifier. """ + if not cve.upper().startswith("CVE-"): + raise InvalidCVEError + return NotImplementedError @classmethod From dbcf200cb86cdbad5b620b0ed242cdbd5a0b19ca Mon Sep 17 00:00:00 2001 From: Keshav Priyadarshi Date: Wed, 29 Mar 2023 15:22:22 +0530 Subject: [PATCH 2/5] Add support for CVE query in GithubDataSource Signed-off-by: Keshav Priyadarshi --- vulntotal/datasources/github.py | 242 ++++++++++++++++-- .../graphql_cve-2022-2922_response.json | 186 ++++++++++++++ .../github/graphql_payload-expected.json | 18 +- .../github/graphql_pyaload_cve-expected.json | 3 + .../group_advisory_by_package-expected.json | 34 +++ .../github/parse_advisory-expected.json | 4 + vulntotal/tests/test_github.py | 25 +- 7 files changed, 473 insertions(+), 39 deletions(-) create mode 100644 vulntotal/tests/test_data/github/graphql_cve-2022-2922_response.json create mode 100644 vulntotal/tests/test_data/github/graphql_pyaload_cve-expected.json create mode 100644 vulntotal/tests/test_data/github/group_advisory_by_package-expected.json diff --git a/vulntotal/datasources/github.py b/vulntotal/datasources/github.py index c90d48e11..459a45f93 100644 --- a/vulntotal/datasources/github.py +++ b/vulntotal/datasources/github.py @@ -11,9 +11,11 @@ from typing import Iterable from dotenv import load_dotenv +from packageurl import PackageURL from vulnerabilities import utils from vulntotal.validator import DataSource +from vulntotal.validator import InvalidCVEError from vulntotal.validator import VendorData from vulntotal.vulntotal_utils import get_item from vulntotal.vulntotal_utils import github_constraints_satisfied @@ -27,7 +29,7 @@ class GithubDataSource(DataSource): def fetch_github(self, graphql_query): """ - Requires GitHub API key in .env file + Requires GitHub API key in .env file. For example:: GH_TOKEN="your-github-token" @@ -39,7 +41,7 @@ def datasource_advisory(self, purl) -> Iterable[VendorData]: end_cursor = "" interesting_edges = [] while True: - queryset = generate_graphql_payload(purl, end_cursor) + queryset = generate_graphql_payload_from_purl(purl, end_cursor) response = self.fetch_github(queryset) self._raw_dump.append(response) security_advisories = get_item(response, "data", "securityVulnerabilities") @@ -47,7 +49,28 @@ def datasource_advisory(self, purl) -> Iterable[VendorData]: end_cursor = get_item(security_advisories, "pageInfo", "endCursor") if not security_advisories["pageInfo"]["hasNextPage"]: break - return parse_advisory(interesting_edges) + return parse_advisory(interesting_edges, purl) + + def datasource_advisory_from_cve(self, cve: str) -> Iterable[VendorData]: + if not cve.upper().startswith("CVE-"): + raise InvalidCVEError + + queryset = generate_graphql_payload_from_cve(cve) + response = self.fetch_github(queryset) + self._raw_dump = [response] + grouped_advisory = group_advisory_by_package(response, cve) + + for advisory in grouped_advisory: + ecosystem = get_item(advisory, "package", "ecosystem") + ecosystem = get_purl_type(ecosystem) + package_name = get_item(advisory, "package", "name") + purl = PackageURL.from_string(f"pkg:{ecosystem}/{package_name}") + yield VendorData( + purl=purl, + aliases=sorted(list(set(advisory.get("identifiers", None)))), + affected_versions=sorted(list(set(advisory.get("firstPatchedVersion", None)))), + fixed_versions=sorted(list(set(advisory.get("vulnerableVersionRange", None)))), + ) @classmethod def supported_ecosystem(cls): @@ -61,10 +84,21 @@ def supported_ecosystem(cls): "rust": "RUST", "npm": "NPM", "erlang": "ERLANG", + "pub": "PUB", } -def parse_advisory(interesting_edges) -> Iterable[VendorData]: +def parse_advisory(interesting_edges, purl) -> Iterable[VendorData]: + """ + Parses the GraphQL response and yields VendorData instances. + + Parameters: + interesting_edges (list): List of edges containing security advisory. + purl (PackageURL): PURL to be included in VendorData. + + Yields: + VendorData instance containing purl, aliases, affected_versions and fixed_versions. + """ for edge in interesting_edges: node = edge["node"] aliases = [aliase["value"] for aliase in get_item(node, "advisory", "identifiers")] @@ -72,6 +106,7 @@ def parse_advisory(interesting_edges) -> Iterable[VendorData]: parsed_fixed_versions = get_item(node, "firstPatchedVersion", "identifier") fixed_versions = [parsed_fixed_versions] if parsed_fixed_versions else [] yield VendorData( + purl=PackageURL(purl.type, purl.namespace, purl.name), aliases=sorted(list(set(aliases))), affected_versions=sorted(list(set(affected_versions))), fixed_versions=sorted(list(set(fixed_versions))), @@ -86,39 +121,49 @@ def extract_interesting_edge(edges, purl): return interesting_edges -def generate_graphql_payload(purl, end_cursor): +def generate_graphql_payload_from_purl(purl, end_cursor=""): + """ + Generates a GraphQL payload for querying security vulnerabilities related to a PURL. + + Parameters: + purl (PackageURL): The PURL to search for vulnerabilities. + end_cursor (str): An optional end cursor to use for pagination. + + Returns: + dict: A dictionary containing the GraphQL query string with ecosystem and package. + """ GRAPHQL_QUERY_TEMPLATE = """ query{ securityVulnerabilities(first: 100, ecosystem: %s, package: "%s", %s){ edges { - node { - advisory { - identifiers { - type - value + node { + advisory { + identifiers { + type + value + } + summary + references { + url + } + severity + publishedAt } - summary - references { - url + firstPatchedVersion{ + identifier } - severity - publishedAt - } - firstPatchedVersion{ - identifier - } - package { - name + package { + name + } + vulnerableVersionRange } - vulnerableVersionRange } - } - pageInfo { - hasNextPage - endCursor + pageInfo { + hasNextPage + endCursor + } } } - } """ supported_ecosystem = GithubDataSource.supported_ecosystem() @@ -149,3 +194,146 @@ def generate_graphql_payload(purl, end_cursor): package_name = f"{purl.namespace}/{purl.name}" return {"query": GRAPHQL_QUERY_TEMPLATE % (ecosystem, package_name, end_cursor_exp)} + + +def generate_graphql_payload_from_cve(cve: str): + """ + Generates a GraphQL payload for querying security advisories related to a CVE. + + Parameters: + - cve (str): CVE identifier string to search for. + + Returns: + - dict: Dictionary containing the GraphQL query string with the CVE identifier substituted in. + """ + GRAPHQL_QUERY_TEMPLATE = """ + query { + securityAdvisories(first: 100, identifier: { type: CVE, value: "%s" }) { + nodes { + vulnerabilities(first: 100) { + nodes { + package { + ecosystem + name + } + advisory { + identifiers { + type + value + } + } + firstPatchedVersion { + identifier + } + vulnerableVersionRange + } + } + } + } + } + """ + return {"query": GRAPHQL_QUERY_TEMPLATE % (cve)} + + +def get_purl_type(github_ecosystem): + """ + Returns the corresponding purl type for a given GitHub ecosystem string. + + Parameters: + github_ecosystem (str): The GitHub ecosystem string. + + Returns: + str or None: The corresponding purl type string, or None if the ecosystem is not supported. + """ + ecosystems = GithubDataSource.supported_ecosystem() + for key, val in ecosystems.items(): + if val == github_ecosystem.upper(): + return key.lower() + return None + + +def group_advisory_by_package(advisories_dict, cve): + """ + Extracts security advisory information from a dictionary and groups them by package. + + Parameters: + advisories_dict (dict): Dictionary containing security advisory. The dictionary + should have the following structure: + { + "data":{ + "securityAdvisories":{ + "nodes":[ + { + "vulnerabilities":{ + "nodes":[ + { + "package": { + "ecosystem": str, + "name": str + }, + "advisory":{ + "identifiers":[ + { "value": str }, + ... + ] + }, + "firstPatchedVersion":{ + "identifier": str + }, + "vulnerableVersionRange": str + }, + ... + ] + } + }, + ... + ] + } + } + } + + cve (str): Used for filtering out advisory non maching CVEs. + + Returns: + list: List of dict containing advisory for package. Each dict + in the list represents advisory for a package and has the following keys: + + package (dict): Dict containing ecosystem and package name. + identifiers (list of str): List of identifiers CVE and GHSA. + firstPatchedVersion (list of str): List of first patched versions. + vulnerableVersionRange (list of str): List of vulnerable version ranges. + """ + advisories = advisories_dict["data"]["securityAdvisories"]["nodes"] + output = [] + + for advisory in advisories: + for vulnerability in advisory["vulnerabilities"]["nodes"]: + package = vulnerability["package"] + advisory_ids = [ + identifier["value"] for identifier in vulnerability["advisory"]["identifiers"] + ] + + # Skip advisory if required CVE is not present in advisory. + # GraphQL query for `CVE-2022-2922` may also include advisory for `CVE-2022-29221` + # `CVE-2022-29222` and `CVE-2022-29229` + if cve not in advisory_ids: + continue + first_patched_version = vulnerability["firstPatchedVersion"]["identifier"] + vulnerable_version_range = vulnerability["vulnerableVersionRange"] + + # Check if a vulnerability for the same package is already in the output list + existing_vulnerability = next((v for v in output if v["package"] == package), None) + if existing_vulnerability: + existing_vulnerability["identifiers"] += advisory_ids + existing_vulnerability["firstPatchedVersion"].append(first_patched_version) + existing_vulnerability["vulnerableVersionRange"].append(vulnerable_version_range) + else: + output.append( + { + "package": package, + "identifiers": advisory_ids, + "firstPatchedVersion": [first_patched_version], + "vulnerableVersionRange": [vulnerable_version_range], + } + ) + return output diff --git a/vulntotal/tests/test_data/github/graphql_cve-2022-2922_response.json b/vulntotal/tests/test_data/github/graphql_cve-2022-2922_response.json new file mode 100644 index 000000000..2d88f0187 --- /dev/null +++ b/vulntotal/tests/test_data/github/graphql_cve-2022-2922_response.json @@ -0,0 +1,186 @@ +{ + "data":{ + "securityAdvisories":{ + "nodes":[ + { + "vulnerabilities":{ + "nodes":[ + { + "package":{ + "ecosystem":"GO", + "name":"github.com/pion/dtls/v2" + }, + "advisory":{ + "identifiers":[ + { + "type":"GHSA", + "value":"GHSA-w45j-f832-hxvh" + }, + { + "type":"CVE", + "value":"CVE-2022-29222" + } + ] + }, + "firstPatchedVersion":{ + "identifier":"2.1.5" + }, + "vulnerableVersionRange":"< 2.1.5" + }, + { + "package":{ + "ecosystem":"GO", + "name":"github.com/pion/dtls" + }, + "advisory":{ + "identifiers":[ + { + "type":"GHSA", + "value":"GHSA-w45j-f832-hxvh" + }, + { + "type":"CVE", + "value":"CVE-2022-29222" + } + ] + }, + "firstPatchedVersion":{ + "identifier":"2.1.5" + }, + "vulnerableVersionRange":"< 2.1.5" + } + ] + } + }, + { + "vulnerabilities":{ + "nodes":[ + { + "package":{ + "ecosystem":"NUGET", + "name":"DotNetNuke.Web" + }, + "advisory":{ + "identifiers":[ + { + "type":"GHSA", + "value":"GHSA-9w72-2f23-57gm" + }, + { + "type":"CVE", + "value":"CVE-2022-2922" + } + ] + }, + "firstPatchedVersion":{ + "identifier":"9.11.0" + }, + "vulnerableVersionRange":"< 9.11.0" + }, + { + "package":{ + "ecosystem":"NUGET", + "name":"DotNetNuke.Core" + }, + "advisory":{ + "identifiers":[ + { + "type":"GHSA", + "value":"GHSA-9w72-2f23-57gm" + }, + { + "type":"CVE", + "value":"CVE-2022-2922" + } + ] + }, + "firstPatchedVersion":{ + "identifier":"9.11.0" + }, + "vulnerableVersionRange":"< 9.11.0" + } + ] + } + }, + { + "vulnerabilities":{ + "nodes":[ + { + "package":{ + "ecosystem":"NPM", + "name":"cassproject" + }, + "advisory":{ + "identifiers":[ + { + "type":"GHSA", + "value":"GHSA-7qcx-4p32-qcmx" + }, + { + "type":"CVE", + "value":"CVE-2022-29229" + } + ] + }, + "firstPatchedVersion":{ + "identifier":"1.5.8" + }, + "vulnerableVersionRange":"< 1.5.8" + } + ] + } + }, + { + "vulnerabilities":{ + "nodes":[ + { + "package":{ + "ecosystem":"COMPOSER", + "name":"smarty/smarty" + }, + "advisory":{ + "identifiers":[ + { + "type":"GHSA", + "value":"GHSA-634x-pc3q-cf4c" + }, + { + "type":"CVE", + "value":"CVE-2022-29221" + } + ] + }, + "firstPatchedVersion":{ + "identifier":"4.1.1" + }, + "vulnerableVersionRange":">= 4.0.0, < 4.1.1" + }, + { + "package":{ + "ecosystem":"COMPOSER", + "name":"smarty/smarty" + }, + "advisory":{ + "identifiers":[ + { + "type":"GHSA", + "value":"GHSA-634x-pc3q-cf4c" + }, + { + "type":"CVE", + "value":"CVE-2022-29221" + } + ] + }, + "firstPatchedVersion":{ + "identifier":"3.1.45" + }, + "vulnerableVersionRange":"< 3.1.45" + } + ] + } + } + ] + } + } +} \ No newline at end of file diff --git a/vulntotal/tests/test_data/github/graphql_payload-expected.json b/vulntotal/tests/test_data/github/graphql_payload-expected.json index 6203de0d9..a1a437cf0 100644 --- a/vulntotal/tests/test_data/github/graphql_payload-expected.json +++ b/vulntotal/tests/test_data/github/graphql_payload-expected.json @@ -1,29 +1,29 @@ [ { - "query": "\n query{\n securityVulnerabilities(first: 100, ecosystem: PIP, package: \"jinja2\", ){\n edges {\n node {\n advisory {\n identifiers {\n type\n value\n }\n summary\n references {\n url\n }\n severity\n publishedAt\n }\n firstPatchedVersion{\n identifier\n }\n package {\n name\n }\n vulnerableVersionRange\n }\n }\n pageInfo {\n hasNextPage\n endCursor\n }\n }\n }\n " + "query": "\n query{\n securityVulnerabilities(first: 100, ecosystem: PIP, package: \"jinja2\", ){\n edges {\n node {\n advisory {\n identifiers {\n type\n value\n }\n summary\n references {\n url\n }\n severity\n publishedAt\n }\n firstPatchedVersion{\n identifier\n }\n package {\n name\n }\n vulnerableVersionRange\n }\n }\n pageInfo {\n hasNextPage\n endCursor\n }\n }\n }\n " }, { - "query": "\n query{\n securityVulnerabilities(first: 100, ecosystem: MAVEN, package: \"org.apache.tomcat:tomcat\", ){\n edges {\n node {\n advisory {\n identifiers {\n type\n value\n }\n summary\n references {\n url\n }\n severity\n publishedAt\n }\n firstPatchedVersion{\n identifier\n }\n package {\n name\n }\n vulnerableVersionRange\n }\n }\n pageInfo {\n hasNextPage\n endCursor\n }\n }\n }\n " + "query": "\n query{\n securityVulnerabilities(first: 100, ecosystem: MAVEN, package: \"org.apache.tomcat:tomcat\", ){\n edges {\n node {\n advisory {\n identifiers {\n type\n value\n }\n summary\n references {\n url\n }\n severity\n publishedAt\n }\n firstPatchedVersion{\n identifier\n }\n package {\n name\n }\n vulnerableVersionRange\n }\n }\n pageInfo {\n hasNextPage\n endCursor\n }\n }\n }\n " }, { - "query": "\n query{\n securityVulnerabilities(first: 100, ecosystem: NUGET, package: \"moment.js\", ){\n edges {\n node {\n advisory {\n identifiers {\n type\n value\n }\n summary\n references {\n url\n }\n severity\n publishedAt\n }\n firstPatchedVersion{\n identifier\n }\n package {\n name\n }\n vulnerableVersionRange\n }\n }\n pageInfo {\n hasNextPage\n endCursor\n }\n }\n }\n " + "query": "\n query{\n securityVulnerabilities(first: 100, ecosystem: NUGET, package: \"moment.js\", ){\n edges {\n node {\n advisory {\n identifiers {\n type\n value\n }\n summary\n references {\n url\n }\n severity\n publishedAt\n }\n firstPatchedVersion{\n identifier\n }\n package {\n name\n }\n vulnerableVersionRange\n }\n }\n pageInfo {\n hasNextPage\n endCursor\n }\n }\n }\n " }, { - "query": "\n query{\n securityVulnerabilities(first: 100, ecosystem: NPM, package: \"semver-regex\", ){\n edges {\n node {\n advisory {\n identifiers {\n type\n value\n }\n summary\n references {\n url\n }\n severity\n publishedAt\n }\n firstPatchedVersion{\n identifier\n }\n package {\n name\n }\n vulnerableVersionRange\n }\n }\n pageInfo {\n hasNextPage\n endCursor\n }\n }\n }\n " + "query": "\n query{\n securityVulnerabilities(first: 100, ecosystem: NPM, package: \"semver-regex\", ){\n edges {\n node {\n advisory {\n identifiers {\n type\n value\n }\n summary\n references {\n url\n }\n severity\n publishedAt\n }\n firstPatchedVersion{\n identifier\n }\n package {\n name\n }\n vulnerableVersionRange\n }\n }\n pageInfo {\n hasNextPage\n endCursor\n }\n }\n }\n " }, { - "query": "\n query{\n securityVulnerabilities(first: 100, ecosystem: GO, package: \"github.com/cloudflare/cfrpki\", ){\n edges {\n node {\n advisory {\n identifiers {\n type\n value\n }\n summary\n references {\n url\n }\n severity\n publishedAt\n }\n firstPatchedVersion{\n identifier\n }\n package {\n name\n }\n vulnerableVersionRange\n }\n }\n pageInfo {\n hasNextPage\n endCursor\n }\n }\n }\n " + "query": "\n query{\n securityVulnerabilities(first: 100, ecosystem: GO, package: \"github.com/cloudflare/cfrpki\", ){\n edges {\n node {\n advisory {\n identifiers {\n type\n value\n }\n summary\n references {\n url\n }\n severity\n publishedAt\n }\n firstPatchedVersion{\n identifier\n }\n package {\n name\n }\n vulnerableVersionRange\n }\n }\n pageInfo {\n hasNextPage\n endCursor\n }\n }\n }\n " }, { - "query": "\n query{\n securityVulnerabilities(first: 100, ecosystem: COMPOSER, package: \"symfony/symfony\", ){\n edges {\n node {\n advisory {\n identifiers {\n type\n value\n }\n summary\n references {\n url\n }\n severity\n publishedAt\n }\n firstPatchedVersion{\n identifier\n }\n package {\n name\n }\n vulnerableVersionRange\n }\n }\n pageInfo {\n hasNextPage\n endCursor\n }\n }\n }\n " + "query": "\n query{\n securityVulnerabilities(first: 100, ecosystem: COMPOSER, package: \"symfony/symfony\", ){\n edges {\n node {\n advisory {\n identifiers {\n type\n value\n }\n summary\n references {\n url\n }\n severity\n publishedAt\n }\n firstPatchedVersion{\n identifier\n }\n package {\n name\n }\n vulnerableVersionRange\n }\n }\n pageInfo {\n hasNextPage\n endCursor\n }\n }\n }\n " }, { - "query": "\n query{\n securityVulnerabilities(first: 100, ecosystem: RUST, package: \"slice-deque\", ){\n edges {\n node {\n advisory {\n identifiers {\n type\n value\n }\n summary\n references {\n url\n }\n severity\n publishedAt\n }\n firstPatchedVersion{\n identifier\n }\n package {\n name\n }\n vulnerableVersionRange\n }\n }\n pageInfo {\n hasNextPage\n endCursor\n }\n }\n }\n " + "query": "\n query{\n securityVulnerabilities(first: 100, ecosystem: RUST, package: \"slice-deque\", ){\n edges {\n node {\n advisory {\n identifiers {\n type\n value\n }\n summary\n references {\n url\n }\n severity\n publishedAt\n }\n firstPatchedVersion{\n identifier\n }\n package {\n name\n }\n vulnerableVersionRange\n }\n }\n pageInfo {\n hasNextPage\n endCursor\n }\n }\n }\n " }, { - "query": "\n query{\n securityVulnerabilities(first: 100, ecosystem: ERLANG, package: \"alchemist.vim\", ){\n edges {\n node {\n advisory {\n identifiers {\n type\n value\n }\n summary\n references {\n url\n }\n severity\n publishedAt\n }\n firstPatchedVersion{\n identifier\n }\n package {\n name\n }\n vulnerableVersionRange\n }\n }\n pageInfo {\n hasNextPage\n endCursor\n }\n }\n }\n " + "query": "\n query{\n securityVulnerabilities(first: 100, ecosystem: ERLANG, package: \"alchemist.vim\", ){\n edges {\n node {\n advisory {\n identifiers {\n type\n value\n }\n summary\n references {\n url\n }\n severity\n publishedAt\n }\n firstPatchedVersion{\n identifier\n }\n package {\n name\n }\n vulnerableVersionRange\n }\n }\n pageInfo {\n hasNextPage\n endCursor\n }\n }\n }\n " }, { - "query": "\n query{\n securityVulnerabilities(first: 100, ecosystem: RUBYGEMS, package: \"ftpd\", ){\n edges {\n node {\n advisory {\n identifiers {\n type\n value\n }\n summary\n references {\n url\n }\n severity\n publishedAt\n }\n firstPatchedVersion{\n identifier\n }\n package {\n name\n }\n vulnerableVersionRange\n }\n }\n pageInfo {\n hasNextPage\n endCursor\n }\n }\n }\n " + "query": "\n query{\n securityVulnerabilities(first: 100, ecosystem: RUBYGEMS, package: \"ftpd\", ){\n edges {\n node {\n advisory {\n identifiers {\n type\n value\n }\n summary\n references {\n url\n }\n severity\n publishedAt\n }\n firstPatchedVersion{\n identifier\n }\n package {\n name\n }\n vulnerableVersionRange\n }\n }\n pageInfo {\n hasNextPage\n endCursor\n }\n }\n }\n " } ] \ No newline at end of file diff --git a/vulntotal/tests/test_data/github/graphql_pyaload_cve-expected.json b/vulntotal/tests/test_data/github/graphql_pyaload_cve-expected.json new file mode 100644 index 000000000..b7af802f4 --- /dev/null +++ b/vulntotal/tests/test_data/github/graphql_pyaload_cve-expected.json @@ -0,0 +1,3 @@ +{ + "query": "\n query {\n securityAdvisories(first: 100, identifier: { type: CVE, value: \"CVE-2022-2922\" }) {\n nodes {\n vulnerabilities(first: 100) {\n nodes {\n package {\n ecosystem\n name\n }\n advisory {\n identifiers {\n type\n value\n }\n }\n firstPatchedVersion {\n identifier\n }\n vulnerableVersionRange\n }\n }\n }\n }\n }\n " +} \ No newline at end of file diff --git a/vulntotal/tests/test_data/github/group_advisory_by_package-expected.json b/vulntotal/tests/test_data/github/group_advisory_by_package-expected.json new file mode 100644 index 000000000..3729b5895 --- /dev/null +++ b/vulntotal/tests/test_data/github/group_advisory_by_package-expected.json @@ -0,0 +1,34 @@ +[ + { + "package": { + "ecosystem": "NUGET", + "name": "DotNetNuke.Web" + }, + "identifiers": [ + "GHSA-9w72-2f23-57gm", + "CVE-2022-2922" + ], + "firstPatchedVersion": [ + "9.11.0" + ], + "vulnerableVersionRange": [ + "< 9.11.0" + ] + }, + { + "package": { + "ecosystem": "NUGET", + "name": "DotNetNuke.Core" + }, + "identifiers": [ + "GHSA-9w72-2f23-57gm", + "CVE-2022-2922" + ], + "firstPatchedVersion": [ + "9.11.0" + ], + "vulnerableVersionRange": [ + "< 9.11.0" + ] + } +] \ No newline at end of file diff --git a/vulntotal/tests/test_data/github/parse_advisory-expected.json b/vulntotal/tests/test_data/github/parse_advisory-expected.json index c335c280b..60979f260 100644 --- a/vulntotal/tests/test_data/github/parse_advisory-expected.json +++ b/vulntotal/tests/test_data/github/parse_advisory-expected.json @@ -1,5 +1,6 @@ [ { + "purl": "pkg:generic/namespace/test", "affected_versions": [ "<2.7.2" ], @@ -12,6 +13,7 @@ ] }, { + "purl": "pkg:generic/namespace/test", "affected_versions": [ "<2.11.3" ], @@ -24,6 +26,7 @@ ] }, { + "purl": "pkg:generic/namespace/test", "affected_versions": [ "<2.8.1" ], @@ -36,6 +39,7 @@ ] }, { + "purl": "pkg:generic/namespace/test", "affected_versions": [ "<2.10.1" ], diff --git a/vulntotal/tests/test_github.py b/vulntotal/tests/test_github.py index 5c48a6457..8cbe617b1 100644 --- a/vulntotal/tests/test_github.py +++ b/vulntotal/tests/test_github.py @@ -20,7 +20,7 @@ class TestGithub(testcase.FileBasedTesting): test_data_dir = str(Path(__file__).resolve().parent / "test_data" / "github") - def test_generate_graphql_payload(self): + def test_generate_graphql_payload_from_purl(self): purls = [ "pkg:pypi/jinja2@2.4.1", "pkg:maven/org.apache.tomcat/tomcat@10.1.0-M8", @@ -33,7 +33,8 @@ def test_generate_graphql_payload(self): "pkg:gem/ftpd@0.0.1", ] results = [ - github.generate_graphql_payload(PackageURL.from_string(purl), "") for purl in purls + github.generate_graphql_payload_from_purl(PackageURL.from_string(purl), "") + for purl in purls ] expected_file = self.get_test_loc("graphql_payload-expected.json", must_exist=False) util_tests.check_results_against_json(results, expected_file) @@ -54,6 +55,24 @@ def test_parse_advisory(self): advisory_file = self.get_test_loc("interesting_edge.json") with open(advisory_file) as f: advisory = json.load(f) - results = [adv.to_dict() for adv in github.parse_advisory(advisory)] + results = [ + adv.to_dict() + for adv in github.parse_advisory(advisory, PackageURL("generic", "namespace", "test")) + ] expected_file = self.get_test_loc("parse_advisory-expected.json", must_exist=False) util_tests.check_results_against_json(results, expected_file) + + def test_generate_graphql_payload_from_cve(self): + results = github.generate_graphql_payload_from_cve("CVE-2022-2922") + expected_file = self.get_test_loc("graphql_pyaload_cve-expected.json", must_exist=False) + util_tests.check_results_against_json(results, expected_file) + + def test_group_advisory_by_package(self): + file = self.get_test_loc("graphql_cve-2022-2922_response.json") + with open(file) as f: + response = json.load(f) + results = github.group_advisory_by_package(response, "CVE-2022-2922") + expected_file = self.get_test_loc( + "group_advisory_by_package-expected.json", must_exist=False + ) + util_tests.check_results_against_json(results, expected_file) From 8234a9a54b0ca42439ddfcaa145e51ad42442a26 Mon Sep 17 00:00:00 2001 From: Keshav Priyadarshi Date: Wed, 29 Mar 2023 15:28:39 +0530 Subject: [PATCH 3/5] Include PURL in VendorData Signed-off-by: Keshav Priyadarshi --- vulntotal/datasources/deps.py | 6 ++++-- vulntotal/datasources/gitlab.py | 7 +++++-- vulntotal/datasources/oss_index.py | 6 ++++-- vulntotal/datasources/osv.py | 6 ++++-- vulntotal/datasources/snyk.py | 6 ++++-- vulntotal/datasources/vulnerablecode.py | 9 ++++++--- .../test_data/deps/parse_advisory-expected.json | 1 + .../gitlab/parsed_advisory-expected.json | 4 ++++ .../oss_index/parse_advisory-expected.json | 9 +++++++++ .../osv/parse_advisory_data-expected.json | 5 +++++ .../test_data/snyk/html/0.html-expected.json | 1 + .../test_data/snyk/html/1.html-expected.json | 1 + .../test_data/snyk/html/2.html-expected.json | 1 + .../test_data/snyk/html/3.html-expected.json | 1 + .../vulnerablecode/parse_advisory-expected.json | 2 ++ vulntotal/tests/test_deps.py | 5 ++++- vulntotal/tests/test_gitlab.py | 4 +++- vulntotal/tests/test_oss_index.py | 8 +++++++- vulntotal/tests/test_osv.py | 7 +++++-- vulntotal/tests/test_snyk.py | 16 ++++++++++++---- vulntotal/tests/test_vulnerablecode.py | 6 +++++- 21 files changed, 88 insertions(+), 23 deletions(-) diff --git a/vulntotal/datasources/deps.py b/vulntotal/datasources/deps.py index f8f264d38..33c62773d 100644 --- a/vulntotal/datasources/deps.py +++ b/vulntotal/datasources/deps.py @@ -12,6 +12,7 @@ from urllib.parse import quote import requests +from packageurl import PackageURL from vulntotal.validator import DataSource from vulntotal.validator import VendorData @@ -41,7 +42,7 @@ def datasource_advisory(self, purl) -> Iterable[VendorData]: fetched_advisory = self.fetch_json_response(advisory_payload) self._raw_dump.append(fetched_advisory) if fetched_advisory: - return parse_advisory(fetched_advisory) + return parse_advisory(fetched_advisory, purl) @classmethod def supported_ecosystem(cls): @@ -56,11 +57,12 @@ def supported_ecosystem(cls): } -def parse_advisory(advisory) -> Iterable[VendorData]: +def parse_advisory(advisory, purl) -> Iterable[VendorData]: package = advisory["packages"][0] affected_versions = [event["version"] for event in package["versionsAffected"]] fixed_versions = [event["version"] for event in package["versionsUnaffected"]] yield VendorData( + purl=PackageURL(purl.type, purl.namespace, purl.name), aliases=sorted(set(advisory["aliases"])), affected_versions=sorted(set(affected_versions)), fixed_versions=sorted(set(fixed_versions)), diff --git a/vulntotal/datasources/gitlab.py b/vulntotal/datasources/gitlab.py index 5818156de..55aaa5b99 100644 --- a/vulntotal/datasources/gitlab.py +++ b/vulntotal/datasources/gitlab.py @@ -17,6 +17,7 @@ import requests import saneyaml from fetchcode import fetch +from packageurl import PackageURL from vulntotal.validator import DataSource from vulntotal.validator import VendorData @@ -39,7 +40,7 @@ def datasource_advisory(self, purl) -> Iterable[VendorData]: location = download_subtree(casesensitive_package_slug) if location: interesting_advisories = parse_interesting_advisories( - location, purl.version, delete_download=True + location, purl, delete_download=True ) return interesting_advisories clear_download(location) @@ -151,7 +152,8 @@ def get_casesensitive_slug(path, package_slug): hasnext = paginated_tree["pageInfo"]["hasNextPage"] -def parse_interesting_advisories(location, version, delete_download=False) -> Iterable[VendorData]: +def parse_interesting_advisories(location, purl, delete_download=False) -> Iterable[VendorData]: + version = purl.version path = Path(location) glob = "**/*.yml" files = (p for p in path.glob(glob) if p.is_file()) @@ -161,6 +163,7 @@ def parse_interesting_advisories(location, version, delete_download=False) -> It affected_range = gitlab_advisory["affected_range"] if gitlab_constraints_satisfied(affected_range, version): yield VendorData( + purl=PackageURL(purl.type, purl.namespace, purl.name), aliases=gitlab_advisory["identifiers"], affected_versions=[affected_range], fixed_versions=gitlab_advisory["fixed_versions"], diff --git a/vulntotal/datasources/oss_index.py b/vulntotal/datasources/oss_index.py index 3ff6e3057..d00454304 100644 --- a/vulntotal/datasources/oss_index.py +++ b/vulntotal/datasources/oss_index.py @@ -12,6 +12,7 @@ from typing import Iterable import requests +from packageurl import PackageURL from vulntotal.validator import DataSource from vulntotal.validator import VendorData @@ -57,7 +58,7 @@ def datasource_advisory(self, purl) -> Iterable[VendorData]: response = self.fetch_json_response([str(purl)]) if response: self._raw_dump.append(response) - return parse_advisory(response) + return parse_advisory(response, purl) @classmethod def supported_ecosystem(cls): @@ -79,7 +80,7 @@ def supported_ecosystem(cls): } -def parse_advisory(component) -> Iterable[VendorData]: +def parse_advisory(component, purl) -> Iterable[VendorData]: response = component[0] vulnerabilities = response.get("vulnerabilities") or [] for vuln in vulnerabilities: @@ -89,6 +90,7 @@ def parse_advisory(component) -> Iterable[VendorData]: version_ranges = vuln.get("versionRanges") or [] affected_versions.extend(version_ranges) yield VendorData( + purl=PackageURL(purl.type, purl.namespace, purl.name), aliases=aliases, affected_versions=affected_versions, fixed_versions=fixed_versions, diff --git a/vulntotal/datasources/osv.py b/vulntotal/datasources/osv.py index 3f388cdcc..e6f2ee79e 100644 --- a/vulntotal/datasources/osv.py +++ b/vulntotal/datasources/osv.py @@ -11,6 +11,7 @@ from typing import Iterable import requests +from packageurl import PackageURL from vulntotal.ecosystem.nuget import search_closest_nuget_package_name from vulntotal.validator import DataSource @@ -40,7 +41,7 @@ def datasource_advisory(self, purl) -> Iterable[VendorData]: return advisory = self.fetch_advisory(payload) self._raw_dump.append(advisory) - return parse_advisory(advisory) + return parse_advisory(advisory, purl) @classmethod def supported_ecosystem(cls): @@ -62,7 +63,7 @@ def supported_ecosystem(cls): } -def parse_advisory(response) -> Iterable[VendorData]: +def parse_advisory(response, purl) -> Iterable[VendorData]: """ Parse response from OSV API and yield VendorData """ @@ -91,6 +92,7 @@ def parse_advisory(response) -> Iterable[VendorData]: pass yield VendorData( + purl=PackageURL(purl.type, purl.namespace, purl.name), aliases=sorted(list(set(aliases))), affected_versions=sorted(list(set(affected_versions))), fixed_versions=sorted(list(set(fixed))), diff --git a/vulntotal/datasources/snyk.py b/vulntotal/datasources/snyk.py index 145be8560..d21d74f36 100644 --- a/vulntotal/datasources/snyk.py +++ b/vulntotal/datasources/snyk.py @@ -13,6 +13,7 @@ import requests from bs4 import BeautifulSoup +from packageurl import PackageURL from vulntotal.validator import DataSource from vulntotal.validator import VendorData @@ -46,7 +47,7 @@ def datasource_advisory(self, purl) -> Iterable[VendorData]: advisory_html = self.fetch(advisory_payload) self._raw_dump.append(advisory_html) if advisory_html: - yield parse_html_advisory(advisory_html, snyk_id, affected) + yield parse_html_advisory(advisory_html, snyk_id, affected, purl) @classmethod def supported_ecosystem(cls): @@ -124,7 +125,7 @@ def extract_html_json_advisories(package_advisories): return vulnerablity -def parse_html_advisory(advisory_html, snyk_id, affected) -> VendorData: +def parse_html_advisory(advisory_html, snyk_id, affected, purl) -> VendorData: aliases = [] fixed_versions = [] @@ -145,6 +146,7 @@ def parse_html_advisory(advisory_html, snyk_id, affected) -> VendorData: fixed_versions = "".join(fixed[lower + 1 : upper]).split(",") aliases.append(snyk_id) return VendorData( + purl=PackageURL(purl.type, purl.namespace, purl.name), aliases=aliases, affected_versions=affected, fixed_versions=fixed_versions, diff --git a/vulntotal/datasources/vulnerablecode.py b/vulntotal/datasources/vulnerablecode.py index db3e51c01..5d482acdf 100644 --- a/vulntotal/datasources/vulnerablecode.py +++ b/vulntotal/datasources/vulnerablecode.py @@ -53,7 +53,7 @@ def datasource_advisory(self, purl) -> Iterable[VendorData]: for advisory in metadata_advisories[0]["affected_by_vulnerabilities"]: fetched_advisory = self.fetch_get_json(advisory["url"]) self._raw_dump.append(fetched_advisory) - yield parse_advisory(fetched_advisory) + yield parse_advisory(fetched_advisory, purl) @classmethod def supported_ecosystem(cls): @@ -74,7 +74,7 @@ def supported_ecosystem(cls): } -def parse_advisory(fetched_advisory) -> VendorData: +def parse_advisory(fetched_advisory, purl) -> VendorData: aliases = [aliase["alias"] for aliase in fetched_advisory["aliases"]] affected_versions = [] fixed_versions = [] @@ -83,7 +83,10 @@ def parse_advisory(fetched_advisory) -> VendorData: for instance in fetched_advisory["fixed_packages"]: fixed_versions.append(PackageURL.from_string(instance["purl"]).version) return VendorData( - aliases=aliases, affected_versions=affected_versions, fixed_versions=fixed_versions + purl=PackageURL(purl.type, purl.namespace, purl.name), + aliases=aliases, + affected_versions=affected_versions, + fixed_versions=fixed_versions, ) diff --git a/vulntotal/tests/test_data/deps/parse_advisory-expected.json b/vulntotal/tests/test_data/deps/parse_advisory-expected.json index 4494c2c2d..0a57fead1 100644 --- a/vulntotal/tests/test_data/deps/parse_advisory-expected.json +++ b/vulntotal/tests/test_data/deps/parse_advisory-expected.json @@ -1,5 +1,6 @@ [ { + "purl": "pkg:generic/namespace/test", "affected_versions": [ "2.0", "2.0rc1", diff --git a/vulntotal/tests/test_data/gitlab/parsed_advisory-expected.json b/vulntotal/tests/test_data/gitlab/parsed_advisory-expected.json index e3598ecea..1a4e4a024 100644 --- a/vulntotal/tests/test_data/gitlab/parsed_advisory-expected.json +++ b/vulntotal/tests/test_data/gitlab/parsed_advisory-expected.json @@ -1,5 +1,6 @@ [ { + "purl": "pkg:generic/namespace/test", "affected_versions": [ "<=2.7.1" ], @@ -11,6 +12,7 @@ ] }, { + "purl": "pkg:generic/namespace/test", "affected_versions": [ "<2.8.1" ], @@ -23,6 +25,7 @@ ] }, { + "purl": "pkg:generic/namespace/test", "affected_versions": [ "<2.10.1" ], @@ -34,6 +37,7 @@ ] }, { + "purl": "pkg:generic/namespace/test", "affected_versions": [ "<2.11.3" ], diff --git a/vulntotal/tests/test_data/oss_index/parse_advisory-expected.json b/vulntotal/tests/test_data/oss_index/parse_advisory-expected.json index fa2708551..096d498f0 100644 --- a/vulntotal/tests/test_data/oss_index/parse_advisory-expected.json +++ b/vulntotal/tests/test_data/oss_index/parse_advisory-expected.json @@ -1,5 +1,6 @@ [ { + "purl": "pkg:generic/namespace/test", "affected_versions": [], "fixed_versions": [], "aliases": [ @@ -7,6 +8,7 @@ ] }, { + "purl": "pkg:generic/namespace/test", "affected_versions": [], "fixed_versions": [], "aliases": [ @@ -14,6 +16,7 @@ ] }, { + "purl": "pkg:generic/namespace/test", "affected_versions": [], "fixed_versions": [], "aliases": [ @@ -21,6 +24,7 @@ ] }, { + "purl": "pkg:generic/namespace/test", "affected_versions": [], "fixed_versions": [], "aliases": [ @@ -28,6 +32,7 @@ ] }, { + "purl": "pkg:generic/namespace/test", "affected_versions": [], "fixed_versions": [], "aliases": [ @@ -35,6 +40,7 @@ ] }, { + "purl": "pkg:generic/namespace/test", "affected_versions": [], "fixed_versions": [], "aliases": [ @@ -42,6 +48,7 @@ ] }, { + "purl": "pkg:generic/namespace/test", "affected_versions": [], "fixed_versions": [], "aliases": [ @@ -49,6 +56,7 @@ ] }, { + "purl": "pkg:generic/namespace/test", "affected_versions": [], "fixed_versions": [], "aliases": [ @@ -56,6 +64,7 @@ ] }, { + "purl": "pkg:generic/namespace/test", "affected_versions": [], "fixed_versions": [], "aliases": [ diff --git a/vulntotal/tests/test_data/osv/parse_advisory_data-expected.json b/vulntotal/tests/test_data/osv/parse_advisory_data-expected.json index c7bfc490c..d25ec7179 100644 --- a/vulntotal/tests/test_data/osv/parse_advisory_data-expected.json +++ b/vulntotal/tests/test_data/osv/parse_advisory_data-expected.json @@ -1,5 +1,6 @@ [ { + "purl": "pkg:generic/namespace/test", "affected_versions": [ "0", "2.0", @@ -31,6 +32,7 @@ ] }, { + "purl": "pkg:generic/namespace/test", "affected_versions": [ "0", "2.0", @@ -63,6 +65,7 @@ ] }, { + "purl": "pkg:generic/namespace/test", "affected_versions": [ "0", "2.0", @@ -107,6 +110,7 @@ ] }, { + "purl": "pkg:generic/namespace/test", "affected_versions": [ "0", "2.0", @@ -142,6 +146,7 @@ ] }, { + "purl": "pkg:generic/namespace/test", "affected_versions": [ "0", "2.0", diff --git a/vulntotal/tests/test_data/snyk/html/0.html-expected.json b/vulntotal/tests/test_data/snyk/html/0.html-expected.json index d97c5390c..bc858e01b 100644 --- a/vulntotal/tests/test_data/snyk/html/0.html-expected.json +++ b/vulntotal/tests/test_data/snyk/html/0.html-expected.json @@ -1,4 +1,5 @@ { + "purl": "pkg:generic/namespace/test", "affected_versions": [ "TEST-AFFECTED" ], diff --git a/vulntotal/tests/test_data/snyk/html/1.html-expected.json b/vulntotal/tests/test_data/snyk/html/1.html-expected.json index c231f1c2a..7ef633f3b 100644 --- a/vulntotal/tests/test_data/snyk/html/1.html-expected.json +++ b/vulntotal/tests/test_data/snyk/html/1.html-expected.json @@ -1,4 +1,5 @@ { + "purl": "pkg:generic/namespace/test", "affected_versions": [ "TEST-AFFECTED" ], diff --git a/vulntotal/tests/test_data/snyk/html/2.html-expected.json b/vulntotal/tests/test_data/snyk/html/2.html-expected.json index 8e1d42876..ffff11b74 100644 --- a/vulntotal/tests/test_data/snyk/html/2.html-expected.json +++ b/vulntotal/tests/test_data/snyk/html/2.html-expected.json @@ -1,4 +1,5 @@ { + "purl": "pkg:generic/namespace/test", "affected_versions": [ "TEST-AFFECTED" ], diff --git a/vulntotal/tests/test_data/snyk/html/3.html-expected.json b/vulntotal/tests/test_data/snyk/html/3.html-expected.json index 8740cbfc4..ed1142993 100644 --- a/vulntotal/tests/test_data/snyk/html/3.html-expected.json +++ b/vulntotal/tests/test_data/snyk/html/3.html-expected.json @@ -1,4 +1,5 @@ { + "purl": "pkg:generic/namespace/test", "affected_versions": [ "TEST-AFFECTED" ], diff --git a/vulntotal/tests/test_data/vulnerablecode/parse_advisory-expected.json b/vulntotal/tests/test_data/vulnerablecode/parse_advisory-expected.json index 4c04506df..4fcfe5fd3 100644 --- a/vulntotal/tests/test_data/vulnerablecode/parse_advisory-expected.json +++ b/vulntotal/tests/test_data/vulnerablecode/parse_advisory-expected.json @@ -1,5 +1,6 @@ [ { + "purl": "pkg:generic/namespace/test", "affected_versions": [ "8.5.50", "9.0.30", @@ -109,6 +110,7 @@ ] }, { + "purl": "pkg:generic/namespace/test", "affected_versions": [ "7.0.0", "7.0.81", diff --git a/vulntotal/tests/test_deps.py b/vulntotal/tests/test_deps.py index d676aefb3..b39258f99 100644 --- a/vulntotal/tests/test_deps.py +++ b/vulntotal/tests/test_deps.py @@ -55,6 +55,9 @@ def test_parse_advisory(self): advisory_file = self.get_test_loc("advisory.json") with open(advisory_file) as f: advisory = json.load(f) - results = [adv.to_dict() for adv in deps.parse_advisory(advisory)] + results = [ + adv.to_dict() + for adv in deps.parse_advisory(advisory, PackageURL("generic", "namespace", "test")) + ] expected_file = self.get_test_loc("parse_advisory-expected.json", must_exist=False) util_tests.check_results_against_json(results, expected_file) diff --git a/vulntotal/tests/test_gitlab.py b/vulntotal/tests/test_gitlab.py index 5e6f6fe91..e7f38009d 100644 --- a/vulntotal/tests/test_gitlab.py +++ b/vulntotal/tests/test_gitlab.py @@ -40,7 +40,9 @@ def test_parse_html_advisory(self): ) results = [ adv.to_dict() - for adv in gitlab.parse_interesting_advisories(advisory_folder, "0.1.1", False) + for adv in gitlab.parse_interesting_advisories( + advisory_folder, PackageURL("generic", "namespace", "test", "0.1.1"), False + ) ] expected_file = self.get_test_loc("parsed_advisory-expected.json", must_exist=False) util_tests.check_results_against_json(results, expected_file) diff --git a/vulntotal/tests/test_oss_index.py b/vulntotal/tests/test_oss_index.py index 1681c79e3..6ba02e634 100644 --- a/vulntotal/tests/test_oss_index.py +++ b/vulntotal/tests/test_oss_index.py @@ -11,6 +11,7 @@ from pathlib import Path from commoncode import testcase +from packageurl import PackageURL from vulnerabilities.tests import util_tests from vulntotal.datasources import oss_index @@ -23,6 +24,11 @@ def test_parse_advisory(self): advisory_file = self.get_test_loc("advisory.json") with open(advisory_file) as f: advisory = json.load(f) - results = [adv.to_dict() for adv in oss_index.parse_advisory(advisory)] + results = [ + adv.to_dict() + for adv in oss_index.parse_advisory( + advisory, PackageURL("generic", "namespace", "test") + ) + ] expected_file = self.get_test_loc("parse_advisory-expected.json", must_exist=False) util_tests.check_results_against_json(results, expected_file) diff --git a/vulntotal/tests/test_osv.py b/vulntotal/tests/test_osv.py index 22b7c41d4..a24647fa7 100644 --- a/vulntotal/tests/test_osv.py +++ b/vulntotal/tests/test_osv.py @@ -58,7 +58,10 @@ def test_parse_advisory(self): advisory_page = self.get_test_loc("advisory.txt") with open(advisory_page) as f: advisory = json.load(f) - results = [adv.to_dict() for adv in osv.parse_advisory(advisory)] + results = [ + adv.to_dict() + for adv in osv.parse_advisory(advisory, PackageURL("generic", "namespace", "test")) + ] expected_file = self.get_test_loc("parse_advisory_data-expected.json", must_exist=False) util_tests.check_results_against_json(results, expected_file) @@ -66,7 +69,7 @@ def test_parse_advisory(self): @pytest.mark.webtest class TestOSVLive(testcase.FileBasedTesting): def test_generate_payload_nuget_with_api_call(self): - # this test makes like API calls + # this test makes live API calls purl = PackageURL.from_string("pkg:nuget/moment.js@2.18.0") results = osv.generate_payload(purl) expected = {"package": {"ecosystem": "NuGet", "name": "Moment.js"}, "version": "2.18.0"} diff --git a/vulntotal/tests/test_snyk.py b/vulntotal/tests/test_snyk.py index 31b8e6e20..05431b96b 100644 --- a/vulntotal/tests/test_snyk.py +++ b/vulntotal/tests/test_snyk.py @@ -56,7 +56,9 @@ def test_parse_html_advisory_0(self): file = self.get_test_loc("html/0.html") with open(file) as f: page = f.read() - result = snyk.parse_html_advisory(page, "TEST-SNYKID", ["TEST-AFFECTED"]).to_dict() + result = snyk.parse_html_advisory( + page, "TEST-SNYKID", ["TEST-AFFECTED"], PackageURL("generic", "namespace", "test") + ).to_dict() expected_file = f"{file}-expected.json" util_tests.check_results_against_json(result, expected_file) @@ -64,7 +66,9 @@ def test_parse_html_advisory_1(self): file = self.get_test_loc("html/1.html") with open(file) as f: page = f.read() - result = snyk.parse_html_advisory(page, "TEST-SNYKID", ["TEST-AFFECTED"]).to_dict() + result = snyk.parse_html_advisory( + page, "TEST-SNYKID", ["TEST-AFFECTED"], PackageURL("generic", "namespace", "test") + ).to_dict() expected_file = f"{file}-expected.json" util_tests.check_results_against_json(result, expected_file) @@ -72,7 +76,9 @@ def test_parse_html_advisory_2(self): file = self.get_test_loc("html/2.html") with open(file) as f: page = f.read() - result = snyk.parse_html_advisory(page, "TEST-SNYKID", ["TEST-AFFECTED"]).to_dict() + result = snyk.parse_html_advisory( + page, "TEST-SNYKID", ["TEST-AFFECTED"], PackageURL("generic", "namespace", "test") + ).to_dict() expected_file = f"{file}-expected.json" util_tests.check_results_against_json(result, expected_file) @@ -80,6 +86,8 @@ def test_parse_html_advisory_3(self): file = self.get_test_loc("html/3.html") with open(file) as f: page = f.read() - result = snyk.parse_html_advisory(page, "TEST-SNYKID", ["TEST-AFFECTED"]).to_dict() + result = snyk.parse_html_advisory( + page, "TEST-SNYKID", ["TEST-AFFECTED"], PackageURL("generic", "namespace", "test") + ).to_dict() expected_file = f"{file}-expected.json" util_tests.check_results_against_json(result, expected_file) diff --git a/vulntotal/tests/test_vulnerablecode.py b/vulntotal/tests/test_vulnerablecode.py index 410be53a2..3d6bc54c3 100644 --- a/vulntotal/tests/test_vulnerablecode.py +++ b/vulntotal/tests/test_vulnerablecode.py @@ -11,6 +11,7 @@ from pathlib import Path from commoncode import testcase +from packageurl import PackageURL from vulnerabilities.tests import util_tests from vulntotal.datasources import vulnerablecode @@ -23,6 +24,9 @@ def test_parse_advisory(self): advisory_file = self.get_test_loc("advisory.json") with open(advisory_file) as f: advisory = json.load(f) - results = [vulnerablecode.parse_advisory(adv).to_dict() for adv in advisory] + results = [ + vulnerablecode.parse_advisory(adv, PackageURL("generic", "namespace", "test")).to_dict() + for adv in advisory + ] expected_file = self.get_test_loc("parse_advisory-expected.json", must_exist=False) util_tests.check_results_against_json(results, expected_file) From 927e8515e82f4f975c55180b625b4a9487a49d3e Mon Sep 17 00:00:00 2001 From: Keshav Priyadarshi Date: Wed, 29 Mar 2023 19:22:39 +0530 Subject: [PATCH 4/5] Use imperative mood for docstring Signed-off-by: Keshav Priyadarshi --- vulntotal/datasources/github.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/vulntotal/datasources/github.py b/vulntotal/datasources/github.py index 459a45f93..8df0fb965 100644 --- a/vulntotal/datasources/github.py +++ b/vulntotal/datasources/github.py @@ -90,7 +90,7 @@ def supported_ecosystem(cls): def parse_advisory(interesting_edges, purl) -> Iterable[VendorData]: """ - Parses the GraphQL response and yields VendorData instances. + Parse the GraphQL response and yield VendorData instances. Parameters: interesting_edges (list): List of edges containing security advisory. @@ -123,7 +123,7 @@ def extract_interesting_edge(edges, purl): def generate_graphql_payload_from_purl(purl, end_cursor=""): """ - Generates a GraphQL payload for querying security vulnerabilities related to a PURL. + Generate a GraphQL payload for querying security vulnerabilities related to a PURL. Parameters: purl (PackageURL): The PURL to search for vulnerabilities. @@ -198,7 +198,7 @@ def generate_graphql_payload_from_purl(purl, end_cursor=""): def generate_graphql_payload_from_cve(cve: str): """ - Generates a GraphQL payload for querying security advisories related to a CVE. + Generate a GraphQL payload for querying security advisories related to a CVE. Parameters: - cve (str): CVE identifier string to search for. @@ -237,7 +237,7 @@ def generate_graphql_payload_from_cve(cve: str): def get_purl_type(github_ecosystem): """ - Returns the corresponding purl type for a given GitHub ecosystem string. + Return the corresponding purl type for a given GitHub ecosystem string. Parameters: github_ecosystem (str): The GitHub ecosystem string. @@ -254,7 +254,7 @@ def get_purl_type(github_ecosystem): def group_advisory_by_package(advisories_dict, cve): """ - Extracts security advisory information from a dictionary and groups them by package. + Extract security advisory information from a dictionary and groups them by package. Parameters: advisories_dict (dict): Dictionary containing security advisory. The dictionary From 35ffb91b1244a061abc28329a3e43fbe9627a407 Mon Sep 17 00:00:00 2001 From: Keshav Priyadarshi Date: Wed, 29 Mar 2023 21:06:12 +0530 Subject: [PATCH 5/5] Regen test fixture for VCIO Signed-off-by: Keshav Priyadarshi --- vulntotal/datasources/github.py | 2 +- .../test_data/vulnerablecode/parse_advisory-expected.json | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/vulntotal/datasources/github.py b/vulntotal/datasources/github.py index e0ca36c83..3311665c4 100644 --- a/vulntotal/datasources/github.py +++ b/vulntotal/datasources/github.py @@ -83,7 +83,7 @@ def supported_ecosystem(cls): "golang": "GO", "cargo": "RUST", "npm": "NPM", - "erlang": "ERLANG", + "hex": "ERLANG", "pub": "PUB", } diff --git a/vulntotal/tests/test_data/vulnerablecode/parse_advisory-expected.json b/vulntotal/tests/test_data/vulnerablecode/parse_advisory-expected.json index 4fcfe5fd3..76407e6f2 100644 --- a/vulntotal/tests/test_data/vulnerablecode/parse_advisory-expected.json +++ b/vulntotal/tests/test_data/vulnerablecode/parse_advisory-expected.json @@ -1,6 +1,6 @@ [ { - "purl": "pkg:generic/namespace/test", + "purl": "pkg:maven/org.apache.tomcat/tomcat", "affected_versions": [ "8.5.50", "9.0.30", @@ -110,7 +110,7 @@ ] }, { - "purl": "pkg:generic/namespace/test", + "purl": "pkg:maven/org.apache.tomcat/tomcat", "affected_versions": [ "7.0.0", "7.0.81",