diff --git a/analyzer/src/funTest/assets/projects/synthetic/all-managers/dotnet.xml b/analyzer/src/funTest/assets/projects/synthetic/all-managers/dotnet.xml new file mode 100644 index 0000000000000..d953968befcce --- /dev/null +++ b/analyzer/src/funTest/assets/projects/synthetic/all-managers/dotnet.xml @@ -0,0 +1 @@ +this is an empty and meaningless file to have ORT realise we are using dotnet as package manager diff --git a/analyzer/src/funTest/assets/projects/synthetic/all-managers/nuget.xml b/analyzer/src/funTest/assets/projects/synthetic/all-managers/nuget.xml new file mode 100644 index 0000000000000..3bfcab4cb57ae --- /dev/null +++ b/analyzer/src/funTest/assets/projects/synthetic/all-managers/nuget.xml @@ -0,0 +1 @@ +this is an empty and meaningless file to have ORT realise we are using nuget as package manager diff --git a/analyzer/src/funTest/assets/projects/synthetic/dotnet-expected-output.yml b/analyzer/src/funTest/assets/projects/synthetic/dotnet-expected-output.yml new file mode 100644 index 0000000000000..5e33c0ba98dcd --- /dev/null +++ b/analyzer/src/funTest/assets/projects/synthetic/dotnet-expected-output.yml @@ -0,0 +1,132 @@ +--- +project: + id: "nuget::dotnet:" + definition_file_path: "analyzer/src/funTest/assets/projects/synthetic/dotnet/dotnet.xml" + declared_licenses: [] + vcs: + type: "Git" + url: "https://github.com/heremaps/oss-review-toolkit.git" + revision: "" + path: "" + vcs_processed: + type: "git" + url: "" + revision: "" + path: "" + homepage_url: "" + scopes: + - name: "dependencies" + dependencies: + - id: "nuget::WebGrease:1.5.2" + dependencies: + - id: "nuget::Antlr:3.4.1.9004" + - id: "nuget::Newtonsoft.Json:5.0.4" + - id: "nuget::jQuery:3.3.1" +packages: +- package: + id: "nuget::Antlr:3.4.1.9004" + declared_licenses: [] + description: "ANother Tool for Language Recognition, is a language tool that provides\ + \ a framework for constructing recognizers, interpreters, compilers, and translators\ + \ from grammatical descriptions containing actions in a variety of target languages." + homepage_url: "" + binary_artifact: + url: "https://api.nuget.org/v3-flatcontainer/antlr/3.4.1.9004/antlr.3.4.1.9004.nupkg" + hash: "t4RqqB/yvSHU8okrySS1L2KaaPsAOCDM8NPbiOy8OJw/oiNIjjUzS5igIVJds0m5k5AmYyGWV9jBVpFQcq830w==" + hash_algorithm: "SHA-512" + source_artifact: + url: "" + hash: "" + hash_algorithm: "" + vcs: + type: "" + url: "" + revision: "" + path: "" + vcs_processed: + type: "" + url: "" + revision: "" + path: "" + curations: [] +- package: + id: "nuget::Newtonsoft.Json:5.0.4" + declared_licenses: + - "http://json.codeplex.com/license" + description: "Json.NET is a popular high-performance JSON framework for .NET" + homepage_url: "http://james.newtonking.com/projects/json-net.aspx" + binary_artifact: + url: "https://api.nuget.org/v3-flatcontainer/newtonsoft.json/5.0.4/newtonsoft.json.5.0.4.nupkg" + hash: "dg2TSL5YEyhbitOHlaKlr+eIvN2rT3GQ7VgcbqUkwNZ6fdW91ThPGX47WljAcKtb23BEk8yOCtY6s191O21hvw==" + hash_algorithm: "SHA-512" + source_artifact: + url: "" + hash: "" + hash_algorithm: "" + vcs: + type: "" + url: "" + revision: "" + path: "" + vcs_processed: + type: "" + url: "" + revision: "" + path: "" + curations: [] +- package: + id: "nuget::WebGrease:1.5.2" + declared_licenses: + - "http://www.microsoft.com/web/webpi/eula/msn_webgrease_eula.htm" + description: "Web Grease is a suite of tools for optimizing javascript, css files\ + \ and images." + homepage_url: "" + binary_artifact: + url: "https://api.nuget.org/v3-flatcontainer/webgrease/1.5.2/webgrease.1.5.2.nupkg" + hash: "RT+no0g7Qk28J/XVwHgn5lJ/qm4hyWFp7wt0I71Uw0j3SfTfOmNqPMxfedHanAjtf3dB2/0KjQ3fQkvVewsvsA==" + hash_algorithm: "SHA-512" + source_artifact: + url: "" + hash: "" + hash_algorithm: "" + vcs: + type: "" + url: "" + revision: "" + path: "" + vcs_processed: + type: "" + url: "" + revision: "" + path: "" + curations: [] +- package: + id: "nuget::jQuery:3.3.1" + declared_licenses: + - "http://jquery.org/license" + description: "jQuery is a new kind of JavaScript Library.\n jQuery is a\ + \ fast and concise JavaScript Library that simplifies HTML document traversing,\ + \ event handling, animating, and Ajax interactions for rapid web development.\ + \ jQuery is designed to change the way that you write JavaScript.\n NOTE:\ + \ This package is maintained on behalf of the library owners by the NuGet Community\ + \ Packages project at http://nugetpackages.codeplex.com/" + homepage_url: "http://jquery.com/" + binary_artifact: + url: "https://api.nuget.org/v3-flatcontainer/jquery/3.3.1/jquery.3.3.1.nupkg" + hash: "Emsts1D6E2AgHVBm4cTD2dxoRyWVG+3zK1x8mYbf+k6aAArPTXHR2Ip++o1utlNbmYIzaPFnhGw3YDexp8PjlQ==" + hash_algorithm: "SHA-512" + source_artifact: + url: "" + hash: "" + hash_algorithm: "" + vcs: + type: "" + url: "" + revision: "" + path: "" + vcs_processed: + type: "" + url: "" + revision: "" + path: "" + curations: [] diff --git a/analyzer/src/funTest/assets/projects/synthetic/dotnet/dotnet.xml b/analyzer/src/funTest/assets/projects/synthetic/dotnet/dotnet.xml new file mode 100644 index 0000000000000..d953968befcce --- /dev/null +++ b/analyzer/src/funTest/assets/projects/synthetic/dotnet/dotnet.xml @@ -0,0 +1 @@ +this is an empty and meaningless file to have ORT realise we are using dotnet as package manager diff --git a/analyzer/src/funTest/assets/projects/synthetic/dotnet/test.csproj b/analyzer/src/funTest/assets/projects/synthetic/dotnet/test.csproj new file mode 100644 index 0000000000000..13c0abf33dc35 --- /dev/null +++ b/analyzer/src/funTest/assets/projects/synthetic/dotnet/test.csproj @@ -0,0 +1,6 @@ + + + + + + diff --git a/analyzer/src/funTest/assets/projects/synthetic/nuget-expected-output.yml b/analyzer/src/funTest/assets/projects/synthetic/nuget-expected-output.yml new file mode 100644 index 0000000000000..2c590ecf79b5b --- /dev/null +++ b/analyzer/src/funTest/assets/projects/synthetic/nuget-expected-output.yml @@ -0,0 +1,132 @@ +--- +project: + id: "nuget::nuget:" + definition_file_path: "analyzer/src/funTest/assets/projects/synthetic/nuget/nuget.xml" + declared_licenses: [] + vcs: + type: "Git" + url: "https://github.com/heremaps/oss-review-toolkit.git" + revision: "" + path: "" + vcs_processed: + type: "git" + url: "" + revision: "" + path: "" + homepage_url: "" + scopes: + - name: "dependencies" + dependencies: + - id: "nuget::WebGrease:1.5.2" + dependencies: + - id: "nuget::Antlr:3.4.1.9004" + - id: "nuget::Newtonsoft.Json:5.0.4" + - id: "nuget::jQuery:3.3.1" +packages: +- package: + id: "nuget::Antlr:3.4.1.9004" + declared_licenses: [] + description: "ANother Tool for Language Recognition, is a language tool that provides\ + \ a framework for constructing recognizers, interpreters, compilers, and translators\ + \ from grammatical descriptions containing actions in a variety of target languages." + homepage_url: "" + binary_artifact: + url: "https://api.nuget.org/v3-flatcontainer/antlr/3.4.1.9004/antlr.3.4.1.9004.nupkg" + hash: "t4RqqB/yvSHU8okrySS1L2KaaPsAOCDM8NPbiOy8OJw/oiNIjjUzS5igIVJds0m5k5AmYyGWV9jBVpFQcq830w==" + hash_algorithm: "SHA-512" + source_artifact: + url: "" + hash: "" + hash_algorithm: "" + vcs: + type: "" + url: "" + revision: "" + path: "" + vcs_processed: + type: "" + url: "" + revision: "" + path: "" + curations: [] +- package: + id: "nuget::Newtonsoft.Json:5.0.4" + declared_licenses: + - "http://json.codeplex.com/license" + description: "Json.NET is a popular high-performance JSON framework for .NET" + homepage_url: "http://james.newtonking.com/projects/json-net.aspx" + binary_artifact: + url: "https://api.nuget.org/v3-flatcontainer/newtonsoft.json/5.0.4/newtonsoft.json.5.0.4.nupkg" + hash: "dg2TSL5YEyhbitOHlaKlr+eIvN2rT3GQ7VgcbqUkwNZ6fdW91ThPGX47WljAcKtb23BEk8yOCtY6s191O21hvw==" + hash_algorithm: "SHA-512" + source_artifact: + url: "" + hash: "" + hash_algorithm: "" + vcs: + type: "" + url: "" + revision: "" + path: "" + vcs_processed: + type: "" + url: "" + revision: "" + path: "" + curations: [] +- package: + id: "nuget::WebGrease:1.5.2" + declared_licenses: + - "http://www.microsoft.com/web/webpi/eula/msn_webgrease_eula.htm" + description: "Web Grease is a suite of tools for optimizing javascript, css files\ + \ and images." + homepage_url: "" + binary_artifact: + url: "https://api.nuget.org/v3-flatcontainer/webgrease/1.5.2/webgrease.1.5.2.nupkg" + hash: "RT+no0g7Qk28J/XVwHgn5lJ/qm4hyWFp7wt0I71Uw0j3SfTfOmNqPMxfedHanAjtf3dB2/0KjQ3fQkvVewsvsA==" + hash_algorithm: "SHA-512" + source_artifact: + url: "" + hash: "" + hash_algorithm: "" + vcs: + type: "" + url: "" + revision: "" + path: "" + vcs_processed: + type: "" + url: "" + revision: "" + path: "" + curations: [] +- package: + id: "nuget::jQuery:3.3.1" + declared_licenses: + - "http://jquery.org/license" + description: "jQuery is a new kind of JavaScript Library.\n jQuery is a\ + \ fast and concise JavaScript Library that simplifies HTML document traversing,\ + \ event handling, animating, and Ajax interactions for rapid web development.\ + \ jQuery is designed to change the way that you write JavaScript.\n NOTE:\ + \ This package is maintained on behalf of the library owners by the NuGet Community\ + \ Packages project at http://nugetpackages.codeplex.com/" + homepage_url: "http://jquery.com/" + binary_artifact: + url: "https://api.nuget.org/v3-flatcontainer/jquery/3.3.1/jquery.3.3.1.nupkg" + hash: "Emsts1D6E2AgHVBm4cTD2dxoRyWVG+3zK1x8mYbf+k6aAArPTXHR2Ip++o1utlNbmYIzaPFnhGw3YDexp8PjlQ==" + hash_algorithm: "SHA-512" + source_artifact: + url: "" + hash: "" + hash_algorithm: "" + vcs: + type: "" + url: "" + revision: "" + path: "" + vcs_processed: + type: "" + url: "" + revision: "" + path: "" + curations: [] diff --git a/analyzer/src/funTest/assets/projects/synthetic/nuget/nuget.xml b/analyzer/src/funTest/assets/projects/synthetic/nuget/nuget.xml new file mode 100644 index 0000000000000..3bfcab4cb57ae --- /dev/null +++ b/analyzer/src/funTest/assets/projects/synthetic/nuget/nuget.xml @@ -0,0 +1 @@ +this is an empty and meaningless file to have ORT realise we are using nuget as package manager diff --git a/analyzer/src/funTest/assets/projects/synthetic/nuget/packages.config b/analyzer/src/funTest/assets/projects/synthetic/nuget/packages.config new file mode 100644 index 0000000000000..f7bc5e544973f --- /dev/null +++ b/analyzer/src/funTest/assets/projects/synthetic/nuget/packages.config @@ -0,0 +1,5 @@ + + + + + diff --git a/analyzer/src/funTest/kotlin/DotNetSupportTest.kt b/analyzer/src/funTest/kotlin/DotNetSupportTest.kt new file mode 100644 index 0000000000000..0ef8959f89dcc --- /dev/null +++ b/analyzer/src/funTest/kotlin/DotNetSupportTest.kt @@ -0,0 +1,113 @@ +/* + * Copyright (C) 2019 Bosch Software Innovations + * Based on: + * Copyright (C) 2017-2019 HERE Europe B.V. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * License-Filename: LICENSE + */ + +package com.here.ort.analyzer + +import com.here.ort.model.Identifier +import com.here.ort.model.OrtIssue +import com.here.ort.model.PackageReference +import com.here.ort.model.Scope +import java.io.File + +import io.kotlintest.specs.StringSpec +import io.kotlintest.shouldBe + +class DotNetSupportTest : StringSpec() { + private val projectDir = File("src/funTest/assets/projects/synthetic/dotnet") + + init { + "bad version is correctly fixed"{ + val testPackage = Pair("jQuery", "1.3.2") + + val dotNetSupport = DotNetSupport(mapOf(testPackage), projectDir) + + val resultScope = Scope("dependencies", sortedSetOf( + PackageReference( + Identifier(type = "nuget", + namespace = "", + name = "jQuery", + version = "3.3.1") + ))) + + dotNetSupport.scope.toString() shouldBe resultScope.toString() + + val resultErrors = listOf() + + errorsToStringWithoutTimestamp(dotNetSupport.errors) shouldBe errorsToStringWithoutTimestamp(resultErrors) + } + + "non-existing project gets registered as error and not in scope"{ + val testPackage = Pair("trifj", "2.0.0") + val testPackage2 = Pair("tffrifj", "2.0.0") + + val dotNetSupport = DotNetSupport(mapOf(testPackage, testPackage2), projectDir) + + val resultScope = Scope("dependencies", sortedSetOf()) + + dotNetSupport.scope.toString() shouldBe resultScope.toString() + + val resultErrors = listOf(OrtIssue(source = "nuget-API does not provide package", + message = "${testPackage.first}:${testPackage.second} can not be found on Nugets RestAPI"), + OrtIssue(source = "nuget-API does not provide package", + message = "${testPackage2.first}:${testPackage2.second} " + + "can not be found on Nugets RestAPI")) + + errorsToStringWithoutTimestamp(dotNetSupport.errors) shouldBe errorsToStringWithoutTimestamp(resultErrors) + } + + "dependencies are detected correctly"{ + val testPackage = Pair("WebGrease", "1.5.2") + + val dotNetSupport = DotNetSupport(mapOf(testPackage), projectDir) + + val resultScope = Scope("dependencies", sortedSetOf( + PackageReference( + Identifier(type = "nuget", + namespace = "", + name = "WebGrease", + version = "1.5.2"), + dependencies = sortedSetOf( + PackageReference( + Identifier(type = "nuget", + namespace = "", + name = "Antlr", + version = "3.4.1.9004") + ), + PackageReference( + Identifier(type = "nuget", + namespace = "", + name = "Newtonsoft.Json", + version = "5.0.4") + )) + ))) + dotNetSupport.scope.toString() shouldBe resultScope.toString() + } + } + + private fun errorsToStringWithoutTimestamp(errors: List): String { + var errorResult = "" + errors.forEach { issue: OrtIssue -> + if (errorResult != "") errorResult += ", " + errorResult += issue.toString().subSequence(26, issue.toString().length - 1) + } + return "[$errorResult]" + } +} diff --git a/analyzer/src/funTest/kotlin/DotNetTest.kt b/analyzer/src/funTest/kotlin/DotNetTest.kt new file mode 100644 index 0000000000000..87520f92c432c --- /dev/null +++ b/analyzer/src/funTest/kotlin/DotNetTest.kt @@ -0,0 +1,64 @@ +/* + * Copyright (C) 2019 Bosch Software Innovations + * Based on: + * Copyright (C) 2017-2019 HERE Europe B.V. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * License-Filename: LICENSE + */ + +package com.here.ort.analyzer + +import com.here.ort.downloader.VersionControlSystem +import com.here.ort.analyzer.managers.DotNet +import com.here.ort.model.yamlMapper +import com.here.ort.utils.normalizeVcsUrl +import com.here.ort.utils.test.DEFAULT_ANALYZER_CONFIGURATION +import com.here.ort.utils.test.DEFAULT_REPOSITORY_CONFIGURATION +import com.here.ort.utils.test.USER_DIR +import com.here.ort.utils.test.patchExpectedResult +import io.kotlintest.matchers.beEmpty +import io.kotlintest.should +import io.kotlintest.shouldBe +import io.kotlintest.shouldNotBe +import io.kotlintest.specs.StringSpec +import java.io.File + +class DotNetTest : StringSpec() { + private val projectDir = File("src/funTest/assets/projects/synthetic/dotnet") + private val vcsDir = VersionControlSystem.forDirectory(projectDir.absoluteFile)!! + private val vcsUrl = vcsDir.getRemoteUrl() + private val vcsRevision = vcsDir.getRevision() + + init { + "Project dependencies are detected correctly" { + val packageFile = File(projectDir, "dotnet.xml") + val vcsPath = vcsDir.getPathToRoot(projectDir) + val expectedResult = patchExpectedResult(File(projectDir.parentFile, + "dotnet-expected-output.yml"), + definitionFilePath = "$vcsPath/test.csproj", + path = vcsPath, + revision = vcsRevision, + url = normalizeVcsUrl(vcsUrl)) + + val result = DotNet(DEFAULT_ANALYZER_CONFIGURATION, DEFAULT_REPOSITORY_CONFIGURATION) + .resolveDependencies(USER_DIR, listOf(packageFile))[packageFile] + + result shouldNotBe null + result!!.errors should beEmpty() + yamlMapper.writeValueAsString(result) shouldBe expectedResult + } + } +} diff --git a/analyzer/src/funTest/kotlin/NugetTest.kt b/analyzer/src/funTest/kotlin/NugetTest.kt new file mode 100644 index 0000000000000..da9d4d608be75 --- /dev/null +++ b/analyzer/src/funTest/kotlin/NugetTest.kt @@ -0,0 +1,64 @@ +/* + * Copyright (C) 2019 Bosch Software Innovations + * Based on: + * Copyright (C) 2017-2019 HERE Europe B.V. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * License-Filename: LICENSE + */ + +package com.here.ort.analyzer + +import com.here.ort.analyzer.managers.Nuget +import com.here.ort.downloader.VersionControlSystem +import com.here.ort.model.yamlMapper +import com.here.ort.utils.normalizeVcsUrl +import com.here.ort.utils.test.DEFAULT_ANALYZER_CONFIGURATION +import com.here.ort.utils.test.DEFAULT_REPOSITORY_CONFIGURATION +import com.here.ort.utils.test.USER_DIR +import com.here.ort.utils.test.patchExpectedResult +import io.kotlintest.matchers.beEmpty +import io.kotlintest.should +import io.kotlintest.shouldBe +import io.kotlintest.shouldNotBe +import io.kotlintest.specs.StringSpec +import java.io.File + +class NugetTest : StringSpec() { + private val projectDir = File("src/funTest/assets/projects/synthetic/nuget") + private val vcsDir = VersionControlSystem.forDirectory(projectDir.absoluteFile)!! + private val vcsUrl = vcsDir.getRemoteUrl() + private val vcsRevision = vcsDir.getRevision() + + init { + "Project dependencies are detected correctly" { + val packageFile = File(projectDir, "nuget.xml") + val vcsPath = vcsDir.getPathToRoot(projectDir) + val expectedResult = patchExpectedResult(File(projectDir.parentFile, + "nuget-expected-output.yml"), + definitionFilePath = "$vcsPath/package.config", + path = vcsPath, + revision = vcsRevision, + url = normalizeVcsUrl(vcsUrl)) + + val result = Nuget(DEFAULT_ANALYZER_CONFIGURATION, DEFAULT_REPOSITORY_CONFIGURATION) + .resolveDependencies(USER_DIR, listOf(packageFile))[packageFile] + + result shouldNotBe null + result!!.errors should beEmpty() + yamlMapper.writeValueAsString(result) shouldBe expectedResult + } + } +} diff --git a/analyzer/src/main/kotlin/DotNetSupport.kt b/analyzer/src/main/kotlin/DotNetSupport.kt new file mode 100644 index 0000000000000..9abe28fad7f63 --- /dev/null +++ b/analyzer/src/main/kotlin/DotNetSupport.kt @@ -0,0 +1,303 @@ +/* + * Copyright (C) 2019 Bosch Software Innovations + * Based on: + * Copyright (C) 2017-2019 HERE Europe B.V. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * License-Filename: LICENSE + */ + +package com.here.ort.analyzer + +import com.fasterxml.jackson.databind.JsonNode +import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper +import com.here.ort.model.HashAlgorithm +import com.here.ort.model.Identifier +import com.here.ort.model.OrtIssue +import com.here.ort.model.Package +import com.here.ort.model.PackageReference +import com.here.ort.model.RemoteArtifact +import com.here.ort.model.Scope +import com.here.ort.model.VcsInfo +import com.here.ort.model.xmlMapper +import com.here.ort.utils.CommandLineTool +import com.here.ort.utils.textValueOrEmpty +import java.io.File + +class DotNetSupport( + packageReferencesMap: Map, + val workingDir: File +) : CommandLineTool { + companion object { + private const val SCOPE_NAME = "dependencies" + private const val PROVIDER_NAME = "nuget" + + private fun extractRepositoryType(node: JsonNode) = + node["repository"]?.get("type")?.textValue() + ?: "" + + private fun extractRepositoryUrl(node: JsonNode) = + node["repository"]?.get("url")?.textValue() + ?: node["projectURL"]?.textValue() + ?: "" + + private fun extractRepositoryRevision(node: JsonNode): String = + node["repository"]?.get("commit")?.textValue() + ?: "" + + private fun extractRepositoryPath(node: JsonNode): String = + node["repository"]?.get("branch")?.textValue() + ?: "" + + private fun extractVcsInfo(node: JsonNode) = + VcsInfo( + type = extractRepositoryType(node), + url = extractRepositoryUrl(node), + revision = extractRepositoryRevision(node), + path = extractRepositoryPath(node) + ) + + private fun extractPackageId(node: JsonNode) = Identifier( + type = PROVIDER_NAME, + namespace = "", + name = node["id"]?.textValue() ?: "", + version = node["version"]?.textValue() ?: "" + ) + + + private fun extractDeclaredLicenses(node: JsonNode) = sortedSetOf().apply { + val license = node["license"]?.textValue() ?: node["licenseUrl"]?.textValue() ?: "" + // most nuget packages only provide a licenseUrl, which counts as declared license, will be changed when scanner runs + if (license.isNotEmpty()) { + add(license) + } + } + + private fun extractRemoteArtifact(node: JsonNode, nupkgUrl: String) = RemoteArtifact( + url = nupkgUrl, + hash = node["packageHash"]?.textValue() ?: "", + hashAlgorithm = HashAlgorithm.fromString( + node["packageHashAlgorithm"]?.textValue() ?: "" + ) + ) + + private fun getCatalogURL(registrationNode: JsonNode): String = + registrationNode["catalogEntry"]?.textValue() ?: "" + + private fun getNuspecURL(registrationNode: JsonNode): String = + registrationNode["packageContent"]?.textValue() ?: "" + + private fun extractVersion(range: String): String { + if (range.isEmpty()) return "" + val rangeReplaces = range.replace("[", "") + .replace(" ", "") + .replace(")", "") + return rangeReplaces.split(",").elementAt(0) + } + + } + + override fun command(workingDir: File?) = "GET" + + var packages = mutableMapOf() + var errors = mutableListOf() + var scope: Scope = Scope(SCOPE_NAME, sortedSetOf()) + // consists of map key: id, version and pair map entry: nupkg url as string and catalogentry as jsonnode + private val packageReferencesAlreadyFound = mutableMapOf, Pair>() + + init { + for (entry: Map.Entry in packageReferencesMap) { + val scopeDependency = getPackageReferenceFromRestAPI(entry.key, entry.value) + if (scopeDependency != null) scope.dependencies += scopeDependency + } + + for (packageReference: PackageReference in scope.collectDependencies()) { + val pkg = packageReferenceToPackage(packageReference) + + if (pkg != Package.EMPTY) { + packages["${pkg.id.name}:${pkg.id.version}"] = pkg + } + } + } + + private fun packageReferenceToPackage(packageReference: PackageReference): Package { + return jsonNodeToPackage(getPackageReferenceJsonContent(packageReference)) + } + + private fun getPackageReferenceJsonContent(packageReference: PackageReference): Pair { + if (packageReferencesAlreadyFound.containsKey( + Pair(packageReference.id.name, packageReference.id.version) + )) { + return packageReferencesAlreadyFound[Pair( + packageReference.id.name, + packageReference.id.version)]!! + } + return try { + val informationUrl = getInformationURL(packageReference.id.name, + packageReference.id.version) ?: throw Exception() + Pair(informationUrl.first, jacksonObjectMapper().readTree(informationUrl.second.runCommand())) + } catch (e: Exception) { + Pair("", xmlMapper.readTree("")) + } + } + + private fun jsonNodeToPackage(packageContent: Pair): Package { + val jsonCatalogNode = packageContent.second + val jsonNuspecNode = try { + xmlMapper.readTree(packageContent.first.replace( + "${jsonCatalogNode["version"].textValueOrEmpty()}.nupkg", + "nuspec" + ).runCommand()) + } catch (e: Exception) { + xmlMapper.readTree("") + } + if (jsonCatalogNode["id"]?.textValue() == null) return Package.EMPTY + + val vcsInfo = extractVcsInfo(jsonNuspecNode["metadata"] ?: xmlMapper.readTree("")) + + return Package( + id = extractPackageId(jsonCatalogNode), + declaredLicenses = extractDeclaredLicenses(jsonCatalogNode), + description = jsonCatalogNode["description"]?.textValue() ?: "", + homepageUrl = jsonCatalogNode["projectUrl"]?.textValue() ?: "", + binaryArtifact = extractRemoteArtifact(jsonCatalogNode, packageContent.first), + sourceArtifact = RemoteArtifact.EMPTY, + vcs = vcsInfo, + vcsProcessed = vcsInfo.normalize() + ) + } + + private fun getPackageReferenceFromRestAPI(packageID: String, version: String): PackageReference? { + val packageJsonNode = preparePackageReference(packageID, version) + + if (packageJsonNode == null) { + errors.add(OrtIssue( + source = "nuget-API does not provide package", + message = "$packageID:$version can not be found on Nugets RestAPI" + )) + return null + } + + val packageReference = PackageReference( + Identifier( + type = PROVIDER_NAME, + namespace = "", + name = if (packageID == packageJsonNode["id"]?.textValue()) packageID + else packageJsonNode["id"]?.textValue() ?: "", + version = packageJsonNode["version"]?.textValue() ?: "" + )) + + val dependenciesIterator = packageJsonNode["dependencyGroups"]?.elements() + + dependenciesIterator?.forEach { + val dependencyIterator = it["dependencies"]?.elements() + + if (dependencyIterator != null) while (dependencyIterator.hasNext()) { + val node = dependencyIterator.next() + val nodeAsPair = Pair( + node["id"].textValueOrEmpty(), + extractVersion(node["range"].textValueOrEmpty())) + if (!hasPackageReferenceAlready(nodeAsPair)) { + val subPackageRef = getPackageReferenceFromRestAPI( + nodeAsPair.first, + nodeAsPair.second + ) + if (subPackageRef != null) packageReference.dependencies += subPackageRef + } + } + } + return packageReference + } + + private fun preparePackageReference(packageID: String, version: String): JsonNode? { + val (first, second) = getInformationURL(packageID, version) ?: return null + val packageJsonNode = try { + jacksonObjectMapper().readTree( + second.runCommand()) + ?: throw Exception("No configuration URL could be fetched") + } catch (e: Exception) { + xmlMapper.readTree("") + } + + packageReferencesAlreadyFound[Pair(first = packageID, second = version)] = + Pair(first, packageJsonNode) + + return packageJsonNode + } + + private fun hasPackageReferenceAlready(nodeAsPair: Pair): Boolean { + return packageReferencesAlreadyFound.containsKey(nodeAsPair) + } + + private fun getInformationURL(packageID: String, version: String): Pair? { + val registrationInfo: String? = try { + "https://api.nuget.org/v3/registration3/$packageID/$version.json".runCommand() + } catch (e: Exception) { + getIdUrl(packageID, version).runCommand() + } + val node = jacksonObjectMapper().readTree( + registrationInfo ?: "" + ) + return if (node != null) Pair(getNuspecURL(node), getCatalogURL(node)) + else null + } + + private fun getIdUrl(packageID: String, version: String): String { + val node = getCreateSearchRestAPIURL(packageID) + + return getRightVersionUrl(node["data"]?.elements(), packageID, version) + ?: getFirstMatchingIdUrl(node["data"]?.elements(), packageID) ?: "" + } + + private fun getRightVersionUrl(dataIterator: (MutableIterator)?, + packageID: String, version: String): String? { + while (dataIterator != null && dataIterator.hasNext()) { + val packageNode = dataIterator.next() + if (packageNode["id"].textValueOrEmpty() == packageID) { + packageNode["versions"].elements().forEach { + if (it["version"].textValueOrEmpty() == version) + return it["@id"].textValueOrEmpty() + else if (!dataIterator.hasNext() && version == "latest") + return it["@id"].textValueOrEmpty() + } + } + } + return null + } + + private fun getFirstMatchingIdUrl(dataIterator: MutableIterator?, packageID: String): String? { + while (dataIterator != null && dataIterator.hasNext()) { + val packageNode = dataIterator.next() + if (packageNode["id"].textValueOrEmpty() == packageID) { + return packageNode["versions"]?.last()?.get("@id").textValueOrEmpty() + } + } + return "" + } + + private fun getCreateSearchRestAPIURL(packageID: String): JsonNode { + return jacksonObjectMapper().readTree( + "https://api-v2v3search-0.nuget.org/query?q=\"$packageID\"&prerelease=false".runCommand()) + } + + private fun String.runCommand(): String { + val commandResult = run(workingDir, toLowerCase()).stdout + if (commandResult.contains("specified blob does not exist")) + throw Exception("GET with URL ${this} could not be resolved") + else { + return commandResult + } + } +} diff --git a/analyzer/src/main/kotlin/managers/DotNet.kt b/analyzer/src/main/kotlin/managers/DotNet.kt new file mode 100644 index 0000000000000..d69375c2e1a45 --- /dev/null +++ b/analyzer/src/main/kotlin/managers/DotNet.kt @@ -0,0 +1,94 @@ +/* + * Copyright (C) 2019 Bosch Software Innovations + * Based on: + * Copyright (C) 2017-2019 HERE Europe B.V. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * License-Filename: LICENSE + */ + +package com.here.ort.analyzer.managers + +import com.here.ort.analyzer.AbstractPackageManagerFactory +import com.here.ort.analyzer.DotNetSupport +import com.here.ort.analyzer.PackageManager +import com.here.ort.downloader.VersionControlSystem +import com.here.ort.model.Identifier +import com.here.ort.model.Project +import com.here.ort.model.ProjectAnalyzerResult +import com.here.ort.model.config.AnalyzerConfiguration +import com.here.ort.model.config.RepositoryConfiguration +import com.here.ort.model.xmlMapper +import java.io.File + +/** + * Dotnet package manager + */ +open class DotNet(analyzerConfig: AnalyzerConfiguration, repoConfig: RepositoryConfiguration) : + PackageManager(analyzerConfig, repoConfig)/*, CommandLineTool */ { + + class Factory : AbstractPackageManagerFactory() { + override val globsForDefinitionFiles = listOf("dotnet.xml") + + override fun create(analyzerConfig: AnalyzerConfiguration, repoConfig: RepositoryConfiguration) = + DotNet(analyzerConfig, repoConfig) + } + + override fun resolveDependencies(definitionFile: File): ProjectAnalyzerResult? { + val workingDir = definitionFile.parentFile + + val dotnet = DotNetSupport(makeMapOfCsprojPackageReference(workingDir), workingDir) + + val vcsInfo = VersionControlSystem.getPathInfo(workingDir) + + val project = Project( + id = Identifier( + type = "nuget", + namespace = "", + name = workingDir.name, + version = "" + ), + definitionFilePath = VersionControlSystem.getPathInfo(definitionFile).path, + declaredLicenses = sortedSetOf(), + vcs = vcsInfo, + vcsProcessed = vcsInfo.normalize(), + homepageUrl = "", + scopes = sortedSetOf(dotnet.scope) + ) + + return ProjectAnalyzerResult( + project, + packages = dotnet.packages.values.map { it.toCuratedPackage() }.toSortedSet(), + errors = dotnet.errors + ) + } + + private fun makeMapOfCsprojPackageReference(workingDir: File): Map { + val mapOfCsProjectFileReferences = mutableMapOf() + + workingDir.walkTopDown().filter { + it.name.endsWith(".csproj") + }.forEach { + it.forEachLine { line: String -> + if (line.contains("PackageReference") || line.contains("Reference")) { + val lineNode = xmlMapper.readTree(line) + mapOfCsProjectFileReferences[lineNode["Include"].textValue() ?: ""] = + lineNode["Version"].textValue() ?: "" + } + } + } + return mapOfCsProjectFileReferences + } +} diff --git a/analyzer/src/main/kotlin/managers/Nuget.kt b/analyzer/src/main/kotlin/managers/Nuget.kt new file mode 100644 index 0000000000000..94cf422cb9c5b --- /dev/null +++ b/analyzer/src/main/kotlin/managers/Nuget.kt @@ -0,0 +1,95 @@ +/* + * Copyright (C) 2019 Bosch Software Innovations + * Based on: + * Copyright (C) 2017-2019 HERE Europe B.V. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * License-Filename: LICENSE + */ + +package com.here.ort.analyzer.managers + +import com.here.ort.analyzer.AbstractPackageManagerFactory +import com.here.ort.analyzer.DotNetSupport +import com.here.ort.analyzer.PackageManager +import com.here.ort.downloader.VersionControlSystem +import com.here.ort.model.Identifier +import com.here.ort.model.Project +import com.here.ort.model.ProjectAnalyzerResult +import com.here.ort.model.config.AnalyzerConfiguration +import com.here.ort.model.config.RepositoryConfiguration +import com.here.ort.model.xmlMapper +import java.io.File + +/** + * The Nuget package mangager for .NET, see https://www.nuget.org/ + */ +open class Nuget(analyzerConfig: AnalyzerConfiguration, repoConfig: RepositoryConfiguration) : + PackageManager(analyzerConfig, repoConfig) { + + class Factory : AbstractPackageManagerFactory() { + override val globsForDefinitionFiles = listOf("nuget.exe", "package.config", "nuget.xml") + + override fun create(analyzerConfig: AnalyzerConfiguration, repoConfig: RepositoryConfiguration) = + Nuget(analyzerConfig, repoConfig) + } + + override fun resolveDependencies(definitionFile: File): ProjectAnalyzerResult? { + val workingDir = definitionFile.parentFile + + val nuget = DotNetSupport(makeMapOfPackageFileReferences(workingDir), workingDir) + + val vcsInfo = VersionControlSystem.getPathInfo(workingDir) + + val project = Project( + id = Identifier( + type = "nuget", + namespace = "", + name = workingDir.name, + version = "" + ), + definitionFilePath = VersionControlSystem.getPathInfo(definitionFile).path, + declaredLicenses = sortedSetOf(), + vcs = vcsInfo, + vcsProcessed = vcsInfo.normalize(), + homepageUrl = "", + scopes = sortedSetOf(nuget.scope) + ) + + return ProjectAnalyzerResult( + project = project, + packages = nuget.packages.values.map { it.toCuratedPackage() }.toSortedSet(), + errors = nuget.errors + ) + } + + private fun makeMapOfPackageFileReferences(workingDir: File): Map { + val mapOfPackageFileReferences = mutableMapOf() + workingDir.walkTopDown().filter { + it.name == "packages.config" + }.forEach { + it.forEachLine { line: String -> + if (line.contains("package") && !(line.contains("packages"))) { + val lineNode = xmlMapper.readTree(line) + mapOfPackageFileReferences.put( + lineNode["id"].textValue() ?: "", + lineNode["version"].textValue() ?: "" + ) + } + } + } + return mapOfPackageFileReferences + } +} diff --git a/analyzer/src/main/resources/META-INF/services/com.here.ort.analyzer.PackageManagerFactory b/analyzer/src/main/resources/META-INF/services/com.here.ort.analyzer.PackageManagerFactory index 0cf90f0776c88..89b917f38eb9b 100644 --- a/analyzer/src/main/resources/META-INF/services/com.here.ort.analyzer.PackageManagerFactory +++ b/analyzer/src/main/resources/META-INF/services/com.here.ort.analyzer.PackageManagerFactory @@ -9,3 +9,5 @@ com.here.ort.analyzer.managers.PIP$Factory com.here.ort.analyzer.managers.Bundler$Factory com.here.ort.analyzer.managers.PhpComposer$Factory com.here.ort.analyzer.managers.Stack$Factory +com.here.ort.analyzer.managers.Nuget$Factory +com.here.ort.analyzer.managers.DotNet$Factory \ No newline at end of file