diff --git a/analyzer/src/funTest/assets/projects/synthetic/all-managers/packages.config b/analyzer/src/funTest/assets/projects/synthetic/all-managers/packages.config new file mode 100644 index 0000000000000..f7bc5e544973f --- /dev/null +++ b/analyzer/src/funTest/assets/projects/synthetic/all-managers/packages.config @@ -0,0 +1,5 @@ + + + + + diff --git a/analyzer/src/funTest/assets/projects/synthetic/all-managers/test.csproj b/analyzer/src/funTest/assets/projects/synthetic/all-managers/test.csproj new file mode 100644 index 0000000000000..022773de3dad9 --- /dev/null +++ b/analyzer/src/funTest/assets/projects/synthetic/all-managers/test.csproj @@ -0,0 +1,18 @@ + + + + + + + + test + is + ignored + + + + + + + + diff --git a/analyzer/src/funTest/assets/projects/synthetic/dotnet-expected-output.yml b/analyzer/src/funTest/assets/projects/synthetic/dotnet-expected-output.yml new file mode 100644 index 0000000000000..4c61455b3da8a --- /dev/null +++ b/analyzer/src/funTest/assets/projects/synthetic/dotnet-expected-output.yml @@ -0,0 +1,148 @@ +--- +project: + id: "nuget::subProjectTest:" + purl: "pkg://nuget//subProjectTest@" + definition_file_path: "analyzer/src/funTest/assets/projects/synthetic/dotnet/subProjectTest/test.csproj" + declared_licenses: [] + declared_licenses_processed: {} + vcs: + type: "Git" + url: "https://github.com/heremaps/oss-review-toolkit.git" + revision: "" + path: "" + vcs_processed: + type: "git" + url: "" + revision: "" + path: "" + homepage_url: "" + scopes: + - name: "dependencies" + dependencies: + - id: "nuget::WebGrease:1.5.2" + dependencies: + - id: "nuget::Antlr:3.4.1.9004" + - id: "nuget::Newtonsoft.Json:5.0.4" + - id: "nuget::jQuery:3.3.1" +packages: +- package: + id: "nuget::Antlr:3.4.1.9004" + purl: "pkg://nuget//Antlr@3.4.1.9004" + declared_licenses: [] + declared_licenses_processed: {} + description: "ANother Tool for Language Recognition, is a language tool that provides\ + \ a framework for constructing recognizers, interpreters, compilers, and translators\ + \ from grammatical descriptions containing actions in a variety of target languages." + homepage_url: "" + binary_artifact: + url: "https://api.nuget.org/v3-flatcontainer/antlr/3.4.1.9004/antlr.3.4.1.9004.nupkg" + hash: "t4RqqB/yvSHU8okrySS1L2KaaPsAOCDM8NPbiOy8OJw/oiNIjjUzS5igIVJds0m5k5AmYyGWV9jBVpFQcq830w==" + hash_algorithm: "SHA-512" + source_artifact: + url: "" + hash: "" + hash_algorithm: "" + vcs: + type: "" + url: "" + revision: "" + path: "" + vcs_processed: + type: "" + url: "" + revision: "" + path: "" + curations: [] +- package: + id: "nuget::Newtonsoft.Json:5.0.4" + purl: "pkg://nuget//Newtonsoft.Json@5.0.4" + declared_licenses: + - "http://json.codeplex.com/license" + declared_licenses_processed: + unmapped: + - "http://json.codeplex.com/license" + description: "Json.NET is a popular high-performance JSON framework for .NET" + homepage_url: "http://james.newtonking.com/projects/json-net.aspx" + binary_artifact: + url: "https://api.nuget.org/v3-flatcontainer/newtonsoft.json/5.0.4/newtonsoft.json.5.0.4.nupkg" + hash: "dg2TSL5YEyhbitOHlaKlr+eIvN2rT3GQ7VgcbqUkwNZ6fdW91ThPGX47WljAcKtb23BEk8yOCtY6s191O21hvw==" + hash_algorithm: "SHA-512" + source_artifact: + url: "" + hash: "" + hash_algorithm: "" + vcs: + type: "" + url: "" + revision: "" + path: "" + vcs_processed: + type: "" + url: "" + revision: "" + path: "" + curations: [] +- package: + id: "nuget::WebGrease:1.5.2" + purl: "pkg://nuget//WebGrease@1.5.2" + declared_licenses: + - "http://www.microsoft.com/web/webpi/eula/msn_webgrease_eula.htm" + declared_licenses_processed: + unmapped: + - "http://www.microsoft.com/web/webpi/eula/msn_webgrease_eula.htm" + description: "Web Grease is a suite of tools for optimizing javascript, css files\ + \ and images." + homepage_url: "" + binary_artifact: + url: "https://api.nuget.org/v3-flatcontainer/webgrease/1.5.2/webgrease.1.5.2.nupkg" + hash: "RT+no0g7Qk28J/XVwHgn5lJ/qm4hyWFp7wt0I71Uw0j3SfTfOmNqPMxfedHanAjtf3dB2/0KjQ3fQkvVewsvsA==" + hash_algorithm: "SHA-512" + source_artifact: + url: "" + hash: "" + hash_algorithm: "" + vcs: + type: "" + url: "" + revision: "" + path: "" + vcs_processed: + type: "" + url: "" + revision: "" + path: "" + curations: [] +- package: + id: "nuget::jQuery:3.3.1" + purl: "pkg://nuget//jQuery@3.3.1" + declared_licenses: + - "http://jquery.org/license" + declared_licenses_processed: + unmapped: + - "http://jquery.org/license" + description: "jQuery is a new kind of JavaScript Library.\n jQuery is a\ + \ fast and concise JavaScript Library that simplifies HTML document traversing,\ + \ event handling, animating, and Ajax interactions for rapid web development.\ + \ jQuery is designed to change the way that you write JavaScript.\n NOTE:\ + \ This package is maintained on behalf of the library owners by the NuGet Community\ + \ Packages project at http://nugetpackages.codeplex.com/" + homepage_url: "http://jquery.com/" + binary_artifact: + url: "https://api.nuget.org/v3-flatcontainer/jquery/3.3.1/jquery.3.3.1.nupkg" + hash: "Emsts1D6E2AgHVBm4cTD2dxoRyWVG+3zK1x8mYbf+k6aAArPTXHR2Ip++o1utlNbmYIzaPFnhGw3YDexp8PjlQ==" + hash_algorithm: "SHA-512" + source_artifact: + url: "" + hash: "" + hash_algorithm: "" + vcs: + type: "" + url: "" + revision: "" + path: "" + vcs_processed: + type: "" + url: "" + revision: "" + path: "" + curations: [] diff --git a/analyzer/src/funTest/assets/projects/synthetic/dotnet/subProjectTest/test.csproj b/analyzer/src/funTest/assets/projects/synthetic/dotnet/subProjectTest/test.csproj new file mode 100644 index 0000000000000..022773de3dad9 --- /dev/null +++ b/analyzer/src/funTest/assets/projects/synthetic/dotnet/subProjectTest/test.csproj @@ -0,0 +1,18 @@ + + + + + + + + test + is + ignored + + + + + + + + diff --git a/analyzer/src/funTest/assets/projects/synthetic/nuget-expected-output.yml b/analyzer/src/funTest/assets/projects/synthetic/nuget-expected-output.yml new file mode 100644 index 0000000000000..1870992b7248d --- /dev/null +++ b/analyzer/src/funTest/assets/projects/synthetic/nuget-expected-output.yml @@ -0,0 +1,148 @@ +--- +project: + id: "nuget::nuget:" + purl: "pkg://nuget//nuget@" + definition_file_path: "analyzer/src/funTest/assets/projects/synthetic/nuget/packages.config" + declared_licenses: [] + declared_licenses_processed: {} + vcs: + type: "Git" + url: "https://github.com/heremaps/oss-review-toolkit.git" + revision: "" + path: "" + vcs_processed: + type: "git" + url: "" + revision: "" + path: "" + homepage_url: "" + scopes: + - name: "dependencies" + dependencies: + - id: "nuget::WebGrease:1.5.2" + dependencies: + - id: "nuget::Antlr:3.4.1.9004" + - id: "nuget::Newtonsoft.Json:5.0.4" + - id: "nuget::jQuery:3.3.1" +packages: +- package: + id: "nuget::Antlr:3.4.1.9004" + purl: "pkg://nuget//Antlr@3.4.1.9004" + declared_licenses: [] + declared_licenses_processed: {} + description: "ANother Tool for Language Recognition, is a language tool that provides\ + \ a framework for constructing recognizers, interpreters, compilers, and translators\ + \ from grammatical descriptions containing actions in a variety of target languages." + homepage_url: "" + binary_artifact: + url: "https://api.nuget.org/v3-flatcontainer/antlr/3.4.1.9004/antlr.3.4.1.9004.nupkg" + hash: "t4RqqB/yvSHU8okrySS1L2KaaPsAOCDM8NPbiOy8OJw/oiNIjjUzS5igIVJds0m5k5AmYyGWV9jBVpFQcq830w==" + hash_algorithm: "SHA-512" + source_artifact: + url: "" + hash: "" + hash_algorithm: "" + vcs: + type: "" + url: "" + revision: "" + path: "" + vcs_processed: + type: "" + url: "" + revision: "" + path: "" + curations: [] +- package: + id: "nuget::Newtonsoft.Json:5.0.4" + purl: "pkg://nuget//Newtonsoft.Json@5.0.4" + declared_licenses: + - "http://json.codeplex.com/license" + declared_licenses_processed: + unmapped: + - "http://json.codeplex.com/license" + description: "Json.NET is a popular high-performance JSON framework for .NET" + homepage_url: "http://james.newtonking.com/projects/json-net.aspx" + binary_artifact: + url: "https://api.nuget.org/v3-flatcontainer/newtonsoft.json/5.0.4/newtonsoft.json.5.0.4.nupkg" + hash: "dg2TSL5YEyhbitOHlaKlr+eIvN2rT3GQ7VgcbqUkwNZ6fdW91ThPGX47WljAcKtb23BEk8yOCtY6s191O21hvw==" + hash_algorithm: "SHA-512" + source_artifact: + url: "" + hash: "" + hash_algorithm: "" + vcs: + type: "" + url: "" + revision: "" + path: "" + vcs_processed: + type: "" + url: "" + revision: "" + path: "" + curations: [] +- package: + id: "nuget::WebGrease:1.5.2" + purl: "pkg://nuget//WebGrease@1.5.2" + declared_licenses: + - "http://www.microsoft.com/web/webpi/eula/msn_webgrease_eula.htm" + declared_licenses_processed: + unmapped: + - "http://www.microsoft.com/web/webpi/eula/msn_webgrease_eula.htm" + description: "Web Grease is a suite of tools for optimizing javascript, css files\ + \ and images." + homepage_url: "" + binary_artifact: + url: "https://api.nuget.org/v3-flatcontainer/webgrease/1.5.2/webgrease.1.5.2.nupkg" + hash: "RT+no0g7Qk28J/XVwHgn5lJ/qm4hyWFp7wt0I71Uw0j3SfTfOmNqPMxfedHanAjtf3dB2/0KjQ3fQkvVewsvsA==" + hash_algorithm: "SHA-512" + source_artifact: + url: "" + hash: "" + hash_algorithm: "" + vcs: + type: "" + url: "" + revision: "" + path: "" + vcs_processed: + type: "" + url: "" + revision: "" + path: "" + curations: [] +- package: + id: "nuget::jQuery:3.3.1" + purl: "pkg://nuget//jQuery@3.3.1" + declared_licenses: + - "http://jquery.org/license" + declared_licenses_processed: + unmapped: + - "http://jquery.org/license" + description: "jQuery is a new kind of JavaScript Library.\n jQuery is a\ + \ fast and concise JavaScript Library that simplifies HTML document traversing,\ + \ event handling, animating, and Ajax interactions for rapid web development.\ + \ jQuery is designed to change the way that you write JavaScript.\n NOTE:\ + \ This package is maintained on behalf of the library owners by the NuGet Community\ + \ Packages project at http://nugetpackages.codeplex.com/" + homepage_url: "http://jquery.com/" + binary_artifact: + url: "https://api.nuget.org/v3-flatcontainer/jquery/3.3.1/jquery.3.3.1.nupkg" + hash: "Emsts1D6E2AgHVBm4cTD2dxoRyWVG+3zK1x8mYbf+k6aAArPTXHR2Ip++o1utlNbmYIzaPFnhGw3YDexp8PjlQ==" + hash_algorithm: "SHA-512" + source_artifact: + url: "" + hash: "" + hash_algorithm: "" + vcs: + type: "" + url: "" + revision: "" + path: "" + vcs_processed: + type: "" + url: "" + revision: "" + path: "" + curations: [] diff --git a/analyzer/src/funTest/assets/projects/synthetic/nuget/packages.config b/analyzer/src/funTest/assets/projects/synthetic/nuget/packages.config new file mode 100644 index 0000000000000..f7bc5e544973f --- /dev/null +++ b/analyzer/src/funTest/assets/projects/synthetic/nuget/packages.config @@ -0,0 +1,5 @@ + + + + + diff --git a/analyzer/src/funTest/kotlin/DotNetSupportTest.kt b/analyzer/src/funTest/kotlin/DotNetSupportTest.kt new file mode 100644 index 0000000000000..1a26d7b728994 --- /dev/null +++ b/analyzer/src/funTest/kotlin/DotNetSupportTest.kt @@ -0,0 +1,114 @@ +/* + * Copyright (C) 2019 Bosch Software Innovations + * Based on: + * Copyright (C) 2017-2019 HERE Europe B.V. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * License-Filename: LICENSE + */ + +package com.here.ort.analyzer + +import com.here.ort.model.Identifier +import com.here.ort.model.OrtIssue +import com.here.ort.model.PackageReference +import com.here.ort.model.Scope + +import io.kotlintest.shouldBe +import io.kotlintest.specs.StringSpec + +import java.io.File + +class DotNetSupportTest : StringSpec() { + private val projectDir = File("src/funTest/assets/projects/synthetic/dotnet") + + init { + "bad version is correctly fixed" { + val testPackage = Pair("jQuery", "1.3.2") + + val dotNetSupport = DotNetSupport(mapOf(testPackage), projectDir) + + val resultScope = Scope("dependencies", sortedSetOf( + PackageReference( + Identifier(type = "nuget", + namespace = "", + name = "jQuery", + version = "3.3.1") + ))) + + dotNetSupport.scope.toString() shouldBe resultScope.toString() + + val resultErrors = listOf() + + errorsToStringWithoutTimestamp(dotNetSupport.errors) shouldBe errorsToStringWithoutTimestamp(resultErrors) + } + + "non-existing project gets registered as error and not in scope" { + val testPackage = Pair("trifj", "2.0.0") + val testPackage2 = Pair("tffrifj", "2.0.0") + + val dotNetSupport = DotNetSupport(mapOf(testPackage, testPackage2), projectDir) + + val resultScope = Scope("dependencies", sortedSetOf()) + + dotNetSupport.scope.toString() shouldBe resultScope.toString() + + val resultErrors = listOf(OrtIssue(source = "nuget-API does not provide package", + message = "${testPackage.first}:${testPackage.second} can not be found on Nugets RestAPI "), + OrtIssue(source = "nuget-API does not provide package", + message = "${testPackage2.first}:${testPackage2.second} " + + "can not be found on Nugets RestAPI ")) + + errorsToStringWithoutTimestamp(dotNetSupport.errors) shouldBe errorsToStringWithoutTimestamp(resultErrors) + } + + "dependencies are detected correctly" { + val testPackage = Pair("WebGrease", "1.5.2") + + val dotNetSupport = DotNetSupport(mapOf(testPackage), projectDir) + + val resultScope = Scope("dependencies", sortedSetOf( + PackageReference( + Identifier(type = "nuget", + namespace = "", + name = "WebGrease", + version = "1.5.2"), + dependencies = sortedSetOf( + PackageReference( + Identifier(type = "nuget", + namespace = "", + name = "Antlr", + version = "3.4.1.9004") + ), + PackageReference( + Identifier(type = "nuget", + namespace = "", + name = "Newtonsoft.Json", + version = "5.0.4") + )) + ))) + dotNetSupport.scope.toString() shouldBe resultScope.toString() + } + } + + private fun errorsToStringWithoutTimestamp(errors: List): String { + var errorResult = "" + errors.forEach { issue: OrtIssue -> + if (errorResult != "") errorResult += ", " + errorResult += issue.toString().split("[ERROR]").last() + } + return "[$errorResult]" + } +} diff --git a/analyzer/src/funTest/kotlin/DotNetTest.kt b/analyzer/src/funTest/kotlin/DotNetTest.kt new file mode 100644 index 0000000000000..ab8833a314f30 --- /dev/null +++ b/analyzer/src/funTest/kotlin/DotNetTest.kt @@ -0,0 +1,81 @@ +/* + * Copyright (C) 2019 Bosch Software Innovations + * Based on: + * Copyright (C) 2017-2019 HERE Europe B.V. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * License-Filename: LICENSE + */ + +package com.here.ort.analyzer + +import com.here.ort.analyzer.managers.DotNet +import com.here.ort.downloader.VersionControlSystem +import com.here.ort.model.yamlMapper +import com.here.ort.utils.normalizeVcsUrl +import com.here.ort.utils.test.DEFAULT_ANALYZER_CONFIGURATION +import com.here.ort.utils.test.DEFAULT_REPOSITORY_CONFIGURATION +import com.here.ort.utils.test.USER_DIR +import com.here.ort.utils.test.patchExpectedResult + +import io.kotlintest.matchers.beEmpty +import io.kotlintest.should +import io.kotlintest.shouldBe +import io.kotlintest.shouldNotBe +import io.kotlintest.specs.StringSpec + +import com.fasterxml.jackson.dataformat.xml.XmlMapper +import com.fasterxml.jackson.module.kotlin.readValue +import com.fasterxml.jackson.module.kotlin.registerKotlinModule + +import java.io.File + +class DotNetTest : StringSpec() { + private val projectDir = File("src/funTest/assets/projects/synthetic/dotnet") + private val vcsDir = VersionControlSystem.forDirectory(projectDir.absoluteFile)!! + private val vcsUrl = vcsDir.getRemoteUrl() + private val vcsRevision = vcsDir.getRevision() + private val packageFile = File(projectDir, "subProjectTest/test.csproj") + + + init { + "Project dependencies are detected correctly" { + val vcsPath = vcsDir.getPathToRoot(projectDir) + val expectedResult = patchExpectedResult(File(projectDir.parentFile, + "dotnet-expected-output.yml"), + definitionFilePath = "$vcsPath/subProjectTest/test.csproj", + path = "$vcsPath/subProjectTest", + revision = vcsRevision, + url = normalizeVcsUrl(vcsUrl)) + + val result = DotNet("DotNet", DEFAULT_ANALYZER_CONFIGURATION, DEFAULT_REPOSITORY_CONFIGURATION) + .resolveDependencies(USER_DIR, listOf(packageFile))[packageFile] + + result shouldNotBe null + result!!.errors should beEmpty() + yamlMapper.writeValueAsString(result) shouldBe expectedResult + } + + "Definition File is correctly mapped" { + val mapper = XmlMapper().registerKotlinModule() + + val result: List = mapper.readValue(packageFile) + + result shouldNotBe null + result.size shouldBe 4 + result[1].packageReference?.size shouldBe 2 + } + } +} diff --git a/analyzer/src/funTest/kotlin/NugetTest.kt b/analyzer/src/funTest/kotlin/NugetTest.kt new file mode 100644 index 0000000000000..7a52d7a7bc82b --- /dev/null +++ b/analyzer/src/funTest/kotlin/NugetTest.kt @@ -0,0 +1,80 @@ +/* + * Copyright (C) 2019 Bosch Software Innovations + * Based on: + * Copyright (C) 2017-2019 HERE Europe B.V. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * License-Filename: LICENSE + */ + +package com.here.ort.analyzer + +import com.fasterxml.jackson.dataformat.xml.XmlMapper +import com.fasterxml.jackson.module.kotlin.readValue +import com.fasterxml.jackson.module.kotlin.registerKotlinModule + +import com.here.ort.analyzer.managers.Nuget +import com.here.ort.downloader.VersionControlSystem +import com.here.ort.model.yamlMapper +import com.here.ort.utils.normalizeVcsUrl +import com.here.ort.utils.test.DEFAULT_ANALYZER_CONFIGURATION +import com.here.ort.utils.test.DEFAULT_REPOSITORY_CONFIGURATION +import com.here.ort.utils.test.USER_DIR +import com.here.ort.utils.test.patchExpectedResult + +import io.kotlintest.matchers.beEmpty +import io.kotlintest.should +import io.kotlintest.shouldBe +import io.kotlintest.shouldNotBe +import io.kotlintest.specs.StringSpec + +import java.io.File + +class NugetTest : StringSpec() { + private val projectDir = File("src/funTest/assets/projects/synthetic/nuget") + private val vcsDir = VersionControlSystem.forDirectory(projectDir.absoluteFile)!! + private val vcsUrl = vcsDir.getRemoteUrl() + private val vcsRevision = vcsDir.getRevision() + private val packageFile = File(projectDir, "packages.config") + + init { + "Project dependencies are detected correctly" { + val vcsPath = vcsDir.getPathToRoot(projectDir) + val expectedResult = patchExpectedResult(File(projectDir.parentFile, + "nuget-expected-output.yml"), + definitionFilePath = "$vcsPath/packages.config", + path = vcsPath, + revision = vcsRevision, + url = normalizeVcsUrl(vcsUrl)) + + val result = Nuget("Nuget", DEFAULT_ANALYZER_CONFIGURATION, DEFAULT_REPOSITORY_CONFIGURATION) + .resolveDependencies(USER_DIR, listOf(packageFile))[packageFile] + + result shouldNotBe null + result!!.errors should beEmpty() + yamlMapper.writeValueAsString(result) shouldBe expectedResult + } + + "Definition File is correctly mapped" { + val mapper = XmlMapper().registerKotlinModule() + + val result: Nuget.Companion.Packages = mapper.readValue(packageFile) + + result shouldNotBe null + result.packages shouldNotBe null + result.packages?.size shouldBe 2 + } + } +} diff --git a/analyzer/src/main/kotlin/DotNetSupport.kt b/analyzer/src/main/kotlin/DotNetSupport.kt new file mode 100644 index 0000000000000..8e6a1b8641021 --- /dev/null +++ b/analyzer/src/main/kotlin/DotNetSupport.kt @@ -0,0 +1,319 @@ +/* + * Copyright (C) 2019 Bosch Software Innovations + * Based on: + * Copyright (C) 2017-2019 HERE Europe B.V. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * License-Filename: LICENSE + */ + +package com.here.ort.analyzer + +import com.fasterxml.jackson.databind.JsonNode +import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper +import com.here.ort.model.HashAlgorithm +import com.here.ort.model.Identifier +import com.here.ort.model.OrtIssue +import com.here.ort.model.Package +import com.here.ort.model.PackageReference +import com.here.ort.model.RemoteArtifact +import com.here.ort.model.Scope +import com.here.ort.model.VcsInfo +import com.here.ort.model.xmlMapper +import com.here.ort.utils.OkHttpClientHelper +import com.here.ort.utils.textValueOrEmpty +import okhttp3.Request +import java.io.File +import java.net.HttpURLConnection + +class DotNetSupport( + packageReferencesMap: Map, + val workingDir: File +) { + companion object { + private const val SCOPE_NAME = "dependencies" + private const val PROVIDER_NAME = "nuget" + + private fun extractRepositoryType(node: JsonNode) = + node["repository"]?.get("type")?.textValue() + ?: "" + + private fun extractRepositoryUrl(node: JsonNode) = + node["repository"]?.get("url")?.textValue() + ?: node["projectURL"]?.textValue() + ?: "" + + private fun extractRepositoryRevision(node: JsonNode): String = + node["repository"]?.get("commit")?.textValue() + ?: "" + + private fun extractRepositoryPath(node: JsonNode): String = + node["repository"]?.get("branch")?.textValue() + ?: "" + + private fun extractVcsInfo(node: JsonNode) = + VcsInfo( + type = extractRepositoryType(node), + url = extractRepositoryUrl(node), + revision = extractRepositoryRevision(node), + path = extractRepositoryPath(node) + ) + + private fun extractPackageId(node: JsonNode) = Identifier( + type = PROVIDER_NAME, + namespace = "", + name = node["id"]?.textValue() ?: "", + version = node["version"]?.textValue() ?: "" + ) + + + private fun extractDeclaredLicenses(node: JsonNode) = sortedSetOf().apply { + val license = node["license"]?.textValue() ?: node["licenseUrl"]?.textValue() ?: "" + // most nuget packages only provide a licenseUrl, + // which counts as declared license, will be changed when scanner runs + if (license.isNotEmpty()) { + add(license) + } + } + + private fun extractRemoteArtifact(node: JsonNode, nupkgUrl: String) = RemoteArtifact( + url = nupkgUrl, + hash = node["packageHash"]?.textValue() ?: "", + hashAlgorithm = HashAlgorithm.fromString( + node["packageHashAlgorithm"]?.textValue() ?: "" + ) + ) + + private fun getCatalogURL(registrationNode: JsonNode): String = + registrationNode["catalogEntry"]?.textValue() ?: "" + + private fun getNuspecURL(registrationNode: JsonNode): String = + registrationNode["packageContent"]?.textValue() ?: "" + + private fun extractVersion(range: String): String { + if (range.isEmpty()) return "" + val rangeReplaces = range.replace("[", "") + .replace(" ", "") + .replace(")", "") + return rangeReplaces.split(",").elementAt(0) + } + + } + + var packages = mutableMapOf() + var errors = mutableListOf() + var scope: Scope = Scope(SCOPE_NAME, sortedSetOf()) + // consists of map key: id, version and pair map entry: nupkg url as string and catalogentry as jsonnode + private val packageReferencesAlreadyFound = mutableMapOf, Pair>() + + init { + for (entry: Map.Entry in packageReferencesMap) { + val scopeDependency = getPackageReferenceFromRestAPI(entry.key, entry.value) + if (scopeDependency != null) scope.dependencies += scopeDependency + } + + for (packageReference: PackageReference in scope.collectDependencies()) { + val pkg = packageReferenceToPackage(packageReference) + + if (pkg != Package.EMPTY) { + packages["${pkg.id.name}:${pkg.id.version}"] = pkg + } + } + } + + private fun packageReferenceToPackage(packageReference: PackageReference): Package { + return jsonNodeToPackage(getPackageReferenceJsonContent(packageReference)) + } + + private fun getPackageReferenceJsonContent(packageReference: PackageReference): Pair { + if (packageReferencesAlreadyFound.containsKey( + Pair(packageReference.id.name, packageReference.id.version) + )) { + return packageReferencesAlreadyFound[Pair( + packageReference.id.name, + packageReference.id.version)]!! + } + return try { + val informationUrl = getInformationURL(packageReference.id.name, + packageReference.id.version) ?: throw Exception() + Pair(informationUrl.first, jacksonObjectMapper().readTree(informationUrl.second.requestFromNugetAPI())) + } catch (e: Exception) { + Pair("", xmlMapper.readTree("")) + } + } + + private fun jsonNodeToPackage(packageContent: Pair): Package { + val jsonCatalogNode = packageContent.second + val jsonNuspecNode = try { + xmlMapper.readTree(packageContent.first.replace( + "${jsonCatalogNode["version"].textValueOrEmpty()}.nupkg", + "nuspec" + ).requestFromNugetAPI()) + } catch (e: Exception) { + xmlMapper.readTree("") + } + if (jsonCatalogNode["id"]?.textValue() == null) return Package.EMPTY + + val vcsInfo = extractVcsInfo(jsonNuspecNode["metadata"] ?: xmlMapper.readTree("")) + + return Package( + id = extractPackageId(jsonCatalogNode), + declaredLicenses = extractDeclaredLicenses(jsonCatalogNode), + description = jsonCatalogNode["description"]?.textValue() ?: "", + homepageUrl = jsonCatalogNode["projectUrl"]?.textValue() ?: "", + binaryArtifact = extractRemoteArtifact(jsonCatalogNode, packageContent.first), + sourceArtifact = RemoteArtifact.EMPTY, + vcs = vcsInfo, + vcsProcessed = vcsInfo.normalize() + ) + } + + private fun getPackageReferenceFromRestAPI(packageID: String, version: String): PackageReference? { + val packageJsonNode = preparePackageReference(packageID, version) + + if (packageJsonNode == null) { + errors.add(OrtIssue( + source = "nuget-API does not provide package", + message = "$packageID:$version can not be found on Nugets RestAPI " + )) + return null + } + + val packageReference = PackageReference( + Identifier( + type = PROVIDER_NAME, + namespace = "", + name = if (packageID == packageJsonNode["id"]?.textValue()) packageID + else packageJsonNode["id"]?.textValue() ?: "", + version = packageJsonNode["version"]?.textValue() ?: "" + )) + + val dependenciesIterator = packageJsonNode["dependencyGroups"]?.elements() + + dependenciesIterator?.forEach { + val dependencyIterator = it["dependencies"]?.elements() + + if (dependencyIterator != null) while (dependencyIterator.hasNext()) { + val node = dependencyIterator.next() + val nodeAsPair = Pair( + node["id"].textValueOrEmpty(), + extractVersion(node["range"].textValueOrEmpty())) + if (!hasPackageReferenceAlready(nodeAsPair)) { + val subPackageRef = getPackageReferenceFromRestAPI( + nodeAsPair.first, + nodeAsPair.second + ) + if (subPackageRef != null) packageReference.dependencies += subPackageRef + } + } + } + return packageReference + } + + private fun preparePackageReference(packageID: String, version: String): JsonNode? { + val (first, second) = getInformationURL(packageID, version) ?: return null + val packageJsonNode = try { + jacksonObjectMapper().readTree( + second.requestFromNugetAPI()) + ?: throw Exception("No configuration URL could be fetched") + } catch (e: Exception) { + xmlMapper.readTree("") + } + + packageReferencesAlreadyFound[Pair(first = packageID, second = version)] = + Pair(first, packageJsonNode) + + return packageJsonNode + } + + private fun hasPackageReferenceAlready(nodeAsPair: Pair): Boolean { + return packageReferencesAlreadyFound.containsKey(nodeAsPair) + } + + private fun getInformationURL(packageID: String, version: String): Pair? { + val registrationInfo: String? = try { + "https://api.nuget.org/v3/registration3/$packageID/$version.json".requestFromNugetAPI() + } catch (e: Exception) { + try { + getIdUrl(packageID, version).requestFromNugetAPI() + } catch (e: Exception) { + "" + } + } + val node = jacksonObjectMapper().readTree( + registrationInfo ?: "" + ) + return if (node != null) Pair(getNuspecURL(node), getCatalogURL(node)) + else null + } + + private fun getIdUrl(packageID: String, version: String): String { + val node = getCreateSearchRestAPIURL(packageID) + + return getRightVersionUrl(node["data"]?.elements(), packageID, version) + ?: getFirstMatchingIdUrl(node["data"]?.elements(), packageID) ?: "" + } + + private fun getRightVersionUrl(dataIterator: (MutableIterator)?, + packageID: String, version: String): String? { + while (dataIterator != null && dataIterator.hasNext()) { + val packageNode = dataIterator.next() + if (packageNode["id"].textValueOrEmpty() == packageID) { + packageNode["versions"].elements().forEach { + if (it["version"].textValueOrEmpty() == version) + return it["@id"].textValueOrEmpty() + else if (!dataIterator.hasNext() && version == "latest") + return it["@id"].textValueOrEmpty() + } + } + } + return null + } + + private fun getFirstMatchingIdUrl(dataIterator: MutableIterator?, packageID: String): String? { + while (dataIterator != null && dataIterator.hasNext()) { + val packageNode = dataIterator.next() + if (packageNode["id"].textValueOrEmpty() == packageID) { + return packageNode["versions"]?.last()?.get("@id").textValueOrEmpty() + } + } + return null + } + + private fun getCreateSearchRestAPIURL(packageID: String): JsonNode { + return jacksonObjectMapper().readTree( + "https://api-v2v3search-0.nuget.org/query?q=\"$packageID\"&prerelease=false".requestFromNugetAPI()) + } + + private fun String.requestFromNugetAPI(): String { + if(this.isNullOrEmpty()) { + throw Exception("GET with URL ${this} could not be resolved") + } + val pkgRequest = Request.Builder() + .get() + .url(this) + .build() + + OkHttpClientHelper.execute(HTTP_CACHE_PATH, pkgRequest).use { response -> + val body = response.body()?.string()?.trim() + + if(response.code() != HttpURLConnection.HTTP_OK || body.isNullOrEmpty()) { + throw Exception("GET with URL ${this} could not be resolved") + } + + return body + } + } +} diff --git a/analyzer/src/main/kotlin/managers/DotNet.kt b/analyzer/src/main/kotlin/managers/DotNet.kt new file mode 100644 index 0000000000000..c301a0f3a9c77 --- /dev/null +++ b/analyzer/src/main/kotlin/managers/DotNet.kt @@ -0,0 +1,121 @@ +/* + * Copyright (C) 2019 Bosch Software Innovations + * Based on: + * Copyright (C) 2017-2019 HERE Europe B.V. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * License-Filename: LICENSE + */ + +package com.here.ort.analyzer.managers + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties +import com.fasterxml.jackson.annotation.JsonProperty +import com.fasterxml.jackson.dataformat.xml.XmlMapper +import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlElementWrapper +import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlProperty +import com.fasterxml.jackson.module.kotlin.readValue +import com.fasterxml.jackson.module.kotlin.registerKotlinModule + +import com.here.ort.analyzer.AbstractPackageManagerFactory +import com.here.ort.analyzer.DotNetSupport +import com.here.ort.analyzer.PackageManager +import com.here.ort.downloader.VersionControlSystem +import com.here.ort.model.Identifier +import com.here.ort.model.Project +import com.here.ort.model.ProjectAnalyzerResult +import com.here.ort.model.config.AnalyzerConfiguration +import com.here.ort.model.config.RepositoryConfiguration + +import java.io.File + +/** + * Dotnet package manager + */ +open class DotNet(name: String, analyzerConfig: AnalyzerConfiguration, repoConfig: RepositoryConfiguration) : + PackageManager(name, analyzerConfig, repoConfig) { + companion object { + fun mapPackageReferences(workingDir: File): Map { + val map = mutableMapOf() + + val mapper = XmlMapper().registerKotlinModule() + + val mappedFile: List = mapper.readValue(workingDir) + + mappedFile.forEach { it -> + it.packageReference?.forEach { + if (!it.Include.isNullOrEmpty()) { + map[it.Include] = it.Version ?: " " + } + } + } + return map + } + + @JsonIgnoreProperties(ignoreUnknown = true) + data class ItemGroup( + @JsonProperty(value = "PackageReference") + @JacksonXmlElementWrapper(useWrapping = false) + val packageReference: List? + ) + + @JsonIgnoreProperties(ignoreUnknown = true) + data class PackageReference( + @JsonProperty(value = "Include") + @JacksonXmlProperty(isAttribute = true) + val Include: String?, + @JsonProperty(value = "Version") + @JacksonXmlProperty(isAttribute = true) + val Version: String? + ) + } + + class Factory : AbstractPackageManagerFactory("DotNet") { + override val globsForDefinitionFiles = listOf("*.csproj", "dotnet.xml") + + override fun create(analyzerConfig: AnalyzerConfiguration, repoConfig: RepositoryConfiguration) = + DotNet(managerName, analyzerConfig, repoConfig) + } + + override fun resolveDependencies(definitionFile: File): ProjectAnalyzerResult? { + val workingDir = definitionFile.parentFile + + val dotnet = DotNetSupport(mapPackageReferences(definitionFile), workingDir) + + val vcsInfo = VersionControlSystem.getPathInfo(workingDir) + + val project = Project( + id = Identifier( + type = "nuget", + namespace = "", + name = workingDir.name, + version = "" + ), + definitionFilePath = VersionControlSystem.getPathInfo(definitionFile).path, + declaredLicenses = sortedSetOf(), + vcs = vcsInfo, + vcsProcessed = vcsInfo.normalize(), + homepageUrl = "", + scopes = sortedSetOf(dotnet.scope) + ) + + return ProjectAnalyzerResult( + project, + packages = dotnet.packages.values.map { it.toCuratedPackage() }.toSortedSet(), + errors = dotnet.errors + ) + } +} + diff --git a/analyzer/src/main/kotlin/managers/Nuget.kt b/analyzer/src/main/kotlin/managers/Nuget.kt new file mode 100644 index 0000000000000..77867e18d69c3 --- /dev/null +++ b/analyzer/src/main/kotlin/managers/Nuget.kt @@ -0,0 +1,123 @@ +/* + * Copyright (C) 2019 Bosch Software Innovations + * Based on: + * Copyright (C) 2017-2019 HERE Europe B.V. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * License-Filename: LICENSE + */ + +package com.here.ort.analyzer.managers + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties +import com.fasterxml.jackson.annotation.JsonProperty +import com.fasterxml.jackson.dataformat.xml.XmlMapper +import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlElementWrapper +import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlProperty +import com.fasterxml.jackson.module.kotlin.readValue +import com.fasterxml.jackson.module.kotlin.registerKotlinModule + +import com.here.ort.analyzer.AbstractPackageManagerFactory +import com.here.ort.analyzer.DotNetSupport +import com.here.ort.analyzer.PackageManager +import com.here.ort.downloader.VersionControlSystem +import com.here.ort.model.Identifier +import com.here.ort.model.Project +import com.here.ort.model.ProjectAnalyzerResult +import com.here.ort.model.config.AnalyzerConfiguration +import com.here.ort.model.config.RepositoryConfiguration + +import java.io.File + +/** + * The Nuget package mangager for .NET, see https://www.nuget.org/ + */ +open class Nuget(name: String, analyzerConfig: AnalyzerConfiguration, repoConfig: RepositoryConfiguration) : + PackageManager(name, analyzerConfig, repoConfig) { + companion object { + fun mapPackageReferences(workingDir: File): Map { + val map = mutableMapOf() + + val mapper = XmlMapper().registerKotlinModule() + val mappedFile: Packages = mapper.readValue(workingDir) + + mappedFile.packages?.forEach { it -> + map[it.id] = it.version + } + + return map + } + + @JsonIgnoreProperties(ignoreUnknown = true) + data class Packages( + @JsonProperty(value = "package") + @JacksonXmlElementWrapper(useWrapping = false) + val packages: List? + ) + + @JsonIgnoreProperties(ignoreUnknown = true) + data class Package( + @JacksonXmlProperty(isAttribute = true) + val id: String, + @JacksonXmlProperty(isAttribute = true) + val version: String) + } + + class Factory : AbstractPackageManagerFactory("Nuget") { + override val globsForDefinitionFiles = listOf("nuget.exe", "package.config", "nuget.xml") + + override fun create(analyzerConfig: AnalyzerConfiguration, repoConfig: RepositoryConfiguration) = + Nuget(managerName, analyzerConfig, repoConfig) + } + + override fun resolveDependencies(definitionFile: File): ProjectAnalyzerResult? { + val workingDir = definitionFile.parentFile + + val nuget = DotNetSupport(makeMapOfPackageFileReferences(workingDir), workingDir) + + val vcsInfo = VersionControlSystem.getPathInfo(workingDir) + + val project = Project( + id = Identifier( + type = "nuget", + namespace = "", + name = workingDir.name, + version = "" + ), + definitionFilePath = VersionControlSystem.getPathInfo(definitionFile).path, + declaredLicenses = sortedSetOf(), + vcs = vcsInfo, + vcsProcessed = vcsInfo.normalize(), + homepageUrl = "", + scopes = sortedSetOf(nuget.scope) + ) + + return ProjectAnalyzerResult( + project = project, + packages = nuget.packages.values.map { it.toCuratedPackage() }.toSortedSet(), + errors = nuget.errors + ) + } + + private fun makeMapOfPackageFileReferences(workingDir: File): Map { + val mapOfPackageFileReferences = mutableMapOf() + workingDir.walkTopDown().filter { + it.name == "packages.config" + }.forEach { + mapOfPackageFileReferences.putAll(mapPackageReferences(it)) + } + return mapOfPackageFileReferences + } +} diff --git a/analyzer/src/main/resources/META-INF/services/com.here.ort.analyzer.PackageManagerFactory b/analyzer/src/main/resources/META-INF/services/com.here.ort.analyzer.PackageManagerFactory index 62b623dddf245..f0e5c0e348eb3 100644 --- a/analyzer/src/main/resources/META-INF/services/com.here.ort.analyzer.PackageManagerFactory +++ b/analyzer/src/main/resources/META-INF/services/com.here.ort.analyzer.PackageManagerFactory @@ -1,9 +1,11 @@ com.here.ort.analyzer.managers.Bower$Factory com.here.ort.analyzer.managers.Bundler$Factory +com.here.ort.analyzer.managers.DotNet$Factory com.here.ort.analyzer.managers.GoDep$Factory com.here.ort.analyzer.managers.Gradle$Factory com.here.ort.analyzer.managers.Maven$Factory com.here.ort.analyzer.managers.NPM$Factory +com.here.ort.analyzer.managers.Nuget$Factory com.here.ort.analyzer.managers.PhpComposer$Factory com.here.ort.analyzer.managers.PIP$Factory com.here.ort.analyzer.managers.SBT$Factory