[Xamarin.Android.Build.Tasks] Better support for netstandard librarie…

…s. (#1356)

Fixes: #1154
Fixes: #1162

NetStandard packages sometimes ship with both reference and
implementation assemblies. The NuGet build task
`<ResolveNuGetPackageAssets/>` only resolves the `ref` version of the
assemblies. There does not seem to be away way to *force* NuGet to
resolve the lib one.

This commit reworks the `<ResolveAssemblies/>` task to attempt to map
the `ref` to a `lib` if we find a Referenece Assembly. Historically
we just issue a warning (which will probably be ignored), but
now we will use the `project.assets.json` file to find the
implementation version of the `ref` assembly, by using the
`NuGet.ProjectModel` package to find the library which is associated
with a reference assembly.

We thus "ignore" reference assemblies, using the "referenced"/"real"
assemblies instead, to ensure that our existing build process
continues to generate usable packages.

*Note*: An alternative approach would be to "embrace" reference
assemblies, allowing them to be used in the build. This doesn't
currently work with our build system, as we assume assemblies will
contain native libraries, Android resources, and other artifacts
which must be extracted at build time, and reference assemblies will
not contain these artifacts.

We would like to explore the "embrace" strategy, but this will
require far more effort to support.

src/Xamarin.Android.Build.Tasks.tpnitems

@@ -78,5 +78,47 @@
       </LicenseText>
       <SourceUrl>https://github.com/IronyProject/Irony</SourceUrl>
     </ThirdPartyNotice>
+    <ThirdPartyNotice Include="JamesNK/Newtonsoft.Json">
+      <LicenseText>
+      The MIT License (MIT)
+
+      Copyright (c) 2007 James Newton-King
+
+      Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
+      associated documentation files (the "Software"), to deal in the Software without restriction, including
+      without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+      copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the
+      following conditions:
+
+      The above copyright notice and this permission notice shall be included in all copies or substantial
+      portions of the Software.
+
+      THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT
+      LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+      IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+      WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+      SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+      </LicenseText>
+      <SourceUrl>https://github.com/JamesNK/Newtonsoft.Json</SourceUrl>
+    </ThirdPartyNotice>
+    <ThirdPartyNotice Include="NuGet/NuGet.Client">
+      <LicenseText>
+      Copyright (c) .NET Foundation and Contributors.
+
+      All rights reserved.
+
+      Licensed under the Apache License, Version 2.0 (the "License"); you may not use
+      these files except in compliance with the License. You may obtain a copy of the
+      License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+      Unless required by applicable law or agreed to in writing, software distributed
+      under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+      CONDITIONS OF ANY KIND, either express or implied. See the License for the
+      specific language governing permissions and limitations under the License.
+      </LicenseText>
+      <SourceUrl>https://github.com/NuGet/NuGet.Client</SourceUrl>
+    </ThirdPartyNotice>
   </ItemGroup>
 </Project>

src/Xamarin.Android.Build.Tasks/Tasks/ResolveAssemblies.cs

@@ -10,12 +10,15 @@
 using MonoDroid.Tuner;
 using System.IO;
 using Xamarin.Android.Tools;
+using NuGet.Common;
+using NuGet.Frameworks;
+using NuGet.ProjectModel;
 
 using Java.Interop.Tools.Cecil;
 
 namespace Xamarin.Android.Tasks
 {
-	public class ResolveAssemblies : Task
+	public class ResolveAssemblies : AsyncTask
 	{
 		// The user's assemblies to package
 		[Required]
@@ -24,6 +27,12 @@ public class ResolveAssemblies : Task
 		[Required]
 		public string ReferenceAssembliesDirectory { get; set; }
 
+		public string ProjectAssetFile { get; set; }
+
+		public string TargetMoniker { get; set; }
+
+		public string NuGetPackageRoot { get; set; }
+
 		public string I18nAssemblies { get; set; }
 		public string LinkMode { get; set; }
 
@@ -45,25 +54,45 @@ public class ResolveAssemblies : Task
 
 		public override bool Execute ()
 		{
-			using (var resolver = new DirectoryAssemblyResolver (this.CreateTaskLogger (), loadDebugSymbols: false)) {
-				return Execute (resolver);
-			}
+			System.Threading.Tasks.Task.Run (() => {
+				using (var resolver = new DirectoryAssemblyResolver (this.CreateTaskLogger (), loadDebugSymbols: false)) {
+					return Execute (resolver);
+				}
+			}, Token).ContinueWith ((t) => {
+				if (t.Exception != null) {
+					var ex = t.Exception.GetBaseException ();
+					LogError (ex.Message + Environment.NewLine + ex.StackTrace);
+				}
+				Complete ();
+			});
+			return base.Execute ();
 		}
 
 		bool Execute (DirectoryAssemblyResolver resolver)
 		{
-			Log.LogDebugMessage ("ResolveAssemblies Task");
-			Log.LogDebugMessage ("  ReferenceAssembliesDirectory: {0}", ReferenceAssembliesDirectory);
-			Log.LogDebugMessage ("  I18nAssemblies: {0}", I18nAssemblies);
-			Log.LogDebugMessage ("  LinkMode: {0}", LinkMode);
-			Log.LogDebugTaskItems ("  Assemblies:", Assemblies);
+			LogDebugMessage ("ResolveAssemblies Task");
+			LogDebugMessage ("  ReferenceAssembliesDirectory: {0}", ReferenceAssembliesDirectory);
+			LogDebugMessage ("  I18nAssemblies: {0}", I18nAssemblies);
+			LogDebugMessage ("  LinkMode: {0}", LinkMode);
+			LogDebugTaskItems ("  Assemblies:", Assemblies);
+			LogDebugMessage ("  ProjectAssetFile: {0}", ProjectAssetFile);
+			LogDebugMessage ("  NuGetPackageRoot: {0}", NuGetPackageRoot);
+			LogDebugMessage ("  TargetMoniker: {0}", TargetMoniker);
 
 			foreach (var dir in ReferenceAssembliesDirectory.Split (new char[] { ';' }, StringSplitOptions.RemoveEmptyEntries))
 				resolver.SearchDirectories.Add (dir);
 
 			var assemblies = new HashSet<string> ();
 
 			var topAssemblyReferences = new List<AssemblyDefinition> ();
+			var logger = new NuGetLogger((s) => {
+				LogDebugMessage ("{0}", s);
+			});
+
+			LockFile lockFile = null;
+			if (!string.IsNullOrEmpty (ProjectAssetFile) && File.Exists (ProjectAssetFile)) {
+				lockFile = LockFileUtilities.GetLockFile (ProjectAssetFile, logger);
+			}
 
 			try {
 				foreach (var assembly in Assemblies) {
@@ -77,21 +106,26 @@ bool Execute (DirectoryAssemblyResolver resolver)
 					if (assemblyDef == null)
 						throw new InvalidOperationException ("Failed to load assembly " + assembly.ItemSpec);
 					if (MonoAndroidHelper.IsReferenceAssembly (assemblyDef)) {
-						Log.LogWarning ($"Ignoring {assembly_path} as it is a Reference Assembly");
-						continue;
+						// Resolve "runtime" library
+						if (lockFile != null)
+							assemblyDef = ResolveRuntimeAssemblyForReferenceAssembly (lockFile, resolver, assemblyDef.Name);
+						if (lockFile == null || assemblyDef == null) {
+							LogWarning ($"Ignoring {assembly_path} as it is a Reference Assembly");
+							continue;
+						}
 					}
 					topAssemblyReferences.Add (assemblyDef);
 					assemblies.Add (Path.GetFullPath (assemblyDef.MainModule.FullyQualifiedName));
 				}
 			} catch (Exception ex) {
-				Log.LogError ("Exception while loading assemblies: {0}", ex);
+				LogError ("Exception while loading assemblies: {0}", ex);
 				return false;
 			}
 			try {
 				foreach (var assembly in topAssemblyReferences)
 					AddAssemblyReferences (resolver, assemblies, assembly, true);
 			} catch (Exception ex) {
-				Log.LogError ("Exception while loading assemblies: {0}", ex);
+				LogError ("Exception while loading assemblies: {0}", ex);
 				return false;
 			}
 
@@ -109,17 +143,46 @@ bool Execute (DirectoryAssemblyResolver resolver)
 			ResolvedUserAssemblies = ResolvedAssemblies.Where (p => !MonoAndroidHelper.IsFrameworkAssembly (p.ItemSpec, true)).ToArray ();
 			ResolvedDoNotPackageAttributes = do_not_package_atts.ToArray ();
 
-			Log.LogDebugTaskItems ("  [Output] ResolvedAssemblies:", ResolvedAssemblies);
-			Log.LogDebugTaskItems ("  [Output] ResolvedUserAssemblies:", ResolvedUserAssemblies);
-			Log.LogDebugTaskItems ("  [Output] ResolvedFrameworkAssemblies:", ResolvedFrameworkAssemblies);
-			Log.LogDebugTaskItems ("  [Output] ResolvedDoNotPackageAttributes:", ResolvedDoNotPackageAttributes);
+			LogDebugTaskItems ("  [Output] ResolvedAssemblies:", ResolvedAssemblies);
+			LogDebugTaskItems ("  [Output] ResolvedUserAssemblies:", ResolvedUserAssemblies);
+			LogDebugTaskItems ("  [Output] ResolvedFrameworkAssemblies:", ResolvedFrameworkAssemblies);
+			LogDebugTaskItems ("  [Output] ResolvedDoNotPackageAttributes:", ResolvedDoNotPackageAttributes);
 
 			return !Log.HasLoggedErrors;
 		}
 
 		readonly List<string> do_not_package_atts = new List<string> ();
 		int indent = 2;
 
+		AssemblyDefinition ResolveRuntimeAssemblyForReferenceAssembly (LockFile lockFile, DirectoryAssemblyResolver resolver, AssemblyNameDefinition assemblyNameDefinition)
+		{
+			if (string.IsNullOrEmpty(TargetMoniker) || string.IsNullOrEmpty (NuGetPackageRoot) || !Directory.Exists (NuGetPackageRoot)) 
+				return null;
+
+			var framework = NuGetFramework.Parse (TargetMoniker);
+			if (framework == null) {
+				LogWarning ($"Could not parse '{TargetMoniker}'");
+				return null;
+			}
+			var target = lockFile.GetTarget (framework, string.Empty);
+			if (target == null) {
+				LogWarning ($"Could not resolve target for '{TargetMoniker}'");
+				return null;
+			}
+			var libraryPath = lockFile.Libraries.FirstOrDefault (x => x.Name == assemblyNameDefinition.Name);
+			if (libraryPath == null)
+				return null;
+			var library = target.Libraries.FirstOrDefault (x => x.Name == assemblyNameDefinition.Name);
+			if (library == null)
+				return null;
+			var runtime = library.RuntimeAssemblies.FirstOrDefault ();
+			if (runtime == null)
+				return null;
+			var path = Path.Combine (NuGetPackageRoot, libraryPath.Path, runtime.Path);
+			LogDebugMessage ($"Attempting to load {path}");
+			return resolver.Load (path, forceLoad: true);
+		}
+
 		void AddAssemblyReferences (DirectoryAssemblyResolver resolver, ICollection<string> assemblies, AssemblyDefinition assembly, bool topLevel)
 		{
 			var fqname = assembly.MainModule.FullyQualifiedName;
@@ -132,11 +195,11 @@ void AddAssemblyReferences (DirectoryAssemblyResolver resolver, ICollection<stri
 			foreach (var att in assembly.CustomAttributes.Where (a => a.AttributeType.FullName == "Java.Interop.DoNotPackageAttribute")) {
 				string file = (string) att.ConstructorArguments.First ().Value;
 				if (string.IsNullOrWhiteSpace (file))
-					Log.LogError ("In referenced assembly {0}, Java.Interop.DoNotPackageAttribute requires non-null file name.", assembly.FullName);
+					LogError ("In referenced assembly {0}, Java.Interop.DoNotPackageAttribute requires non-null file name.", assembly.FullName);
 				do_not_package_atts.Add (Path.GetFileName (file));
 			}
 
-			Log.LogMessage (MessageImportance.Low, "{0}Adding assembly reference for {1}, recursively...", new string (' ', indent), assembly.Name);
+			LogMessage ("{0}Adding assembly reference for {1}, recursively...", new string (' ', indent), assembly.Name);
 			indent += 2;
 			// Add this assembly
 			if (!topLevel && assemblies.All (a => new AssemblyNameDefinition (a, null).Name != assembly.Name.Name))

src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/PackagingTest.cs

@@ -280,9 +280,14 @@ public partial class App : Application
 
         public App()
         {
-            JsonConvert.DeserializeObject<string>(""test"");
-            package = Package.Open ("""");
+            try {
+                JsonConvert.DeserializeObject<string>(""test"");
+                package = Package.Open ("""");
+            } catch {
+            }
             InitializeComponent();
+
+            MainPage = new ContentPage ();
         }
 
         protected override void OnStart()
@@ -337,6 +342,29 @@ protected override void OnResume()
 					KnownPackages.XamarinForms_2_3_4_231,
 				}
 			};
+			app.MainActivity = @"using System;
+using Android.App;
+using Android.Content;
+using Android.Runtime;
+using Android.Views;
+using Android.Widget;
+using Android.OS;
+using XamFormsSample;
+
+namespace App1
+{
+	[Activity (Label = ""App1"", MainLauncher = true, Icon = ""@drawable/icon"")]
+	public class MainActivity : global::Xamarin.Forms.Platform.Android.FormsAppCompatActivity {
+			protected override void OnCreate (Bundle bundle)
+			{
+				base.OnCreate (bundle);
+
+				global::Xamarin.Forms.Forms.Init (this, bundle);
+
+				LoadApplication (new App ());
+			}
+		}
+	}";
 			app.SetProperty (KnownProperties.AndroidSupportedAbis, "x86;armeabi-v7a");
 			var expectedFiles = new string [] {
 				"Java.Interop.dll",
@@ -346,6 +374,7 @@ protected override void OnResume()
 				"System.dll",
 				"System.Runtime.Serialization.dll",
 				"System.IO.Packaging.dll",
+				"System.IO.Compression.dll",
 				"Mono.Android.Export.dll",
 				"App1.dll",
 				"FormsViewGroup.dll",

src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Xamarin.Android.Build.Tests.csproj

@@ -7,7 +7,7 @@
     <OutputType>Library</OutputType>
     <RootNamespace>Xamarin.Android.Build.Tests</RootNamespace>
     <AssemblyName>Xamarin.Android.Build.Tests</AssemblyName>
-    <TargetFrameworkVersion>v4.5.1</TargetFrameworkVersion>
+    <TargetFrameworkVersion>v4.6.2</TargetFrameworkVersion>
   </PropertyGroup>
   <Import Project="Xamarin.Android.Build.Tests.Shared.projitems" Label="Shared" Condition="Exists('Xamarin.Android.Build.Tests.Shared.projitems')" />
   <Import Project="..\..\..\..\Configuration.props" />

src/Xamarin.Android.Build.Tasks/Utilities/NuGetLogger.cs

@@ -0,0 +1,34 @@
+using System;
+using NuGet.Common;
+using Microsoft.Build.Utilities;
+using TPL = System.Threading.Tasks;
+
+namespace Xamarin.Android.Tasks {
+
+	class NuGetLogger : LoggerBase {
+		Action<string> log;
+
+		public NuGetLogger (Action<string> log)
+		{
+			this.log = log;
+		}
+
+		public override void Log (ILogMessage message) {
+			log (message.Message);
+		}
+
+		public override void Log (LogLevel level, string data) {
+			log (data);
+		}
+
+		public override TPL.Task LogAsync (ILogMessage message) {
+			Log (message);
+			return TPL.Task.FromResult(0);
+		}
+
+		public override TPL.Task LogAsync (LogLevel level, string data) {
+			Log (level, data);
+			return TPL.Task.FromResult(0);
+		}
+	}
+}

src/Xamarin.Android.Build.Tasks/Xamarin.Android.Build.Tasks.csproj

@@ -9,7 +9,7 @@
     <RootNamespace>Xamarin.Android.Tasks</RootNamespace>
     <AssemblyName>Xamarin.Android.Build.Tasks</AssemblyName>
     <FileAlignment>512</FileAlignment>
-    <TargetFrameworkVersion>v4.5.1</TargetFrameworkVersion>
+    <TargetFrameworkVersion>v4.6.2</TargetFrameworkVersion>
     <SchemaVersion>2.0</SchemaVersion>
   </PropertyGroup>
   <Import Project="..\..\Configuration.props" />
@@ -51,6 +51,53 @@
     <Reference Include="FSharp.Core">
       <HintPath>..\..\packages\FSharp.Core.3.1.2.5\lib\net40\FSharp.Core.dll</HintPath>
     </Reference>
+    <Reference Include="Newtonsoft.Json">
+      <HintPath>..\..\packages\Newtonsoft.Json.11.0.1\lib\net45\Newtonsoft.Json.dll</HintPath>
+    </Reference>
+    <Reference Include="NuGet.Frameworks">
+      <HintPath>..\..\packages\NuGet.Frameworks.4.6.0\lib\net46\NuGet.Frameworks.dll</HintPath>
+    </Reference>
+    <Reference Include="NuGet.Common">
+      <HintPath>..\..\packages\NuGet.Common.4.6.0\lib\net46\NuGet.Common.dll</HintPath>
+    </Reference>
+    <Reference Include="System.IO.Compression" />
+    <Reference Include="NuGet.Configuration">
+      <HintPath>..\..\packages\NuGet.Configuration.4.6.0\lib\net46\NuGet.Configuration.dll</HintPath>
+    </Reference>
+    <Reference Include="System.Security" />
+    <Reference Include="NuGet.Versioning">
+      <HintPath>..\..\packages\NuGet.Versioning.4.6.0\lib\net46\NuGet.Versioning.dll</HintPath>
+    </Reference>
+    <Reference Include="NuGet.LibraryModel">
+      <HintPath>..\..\packages\NuGet.LibraryModel.4.6.0\lib\net46\NuGet.LibraryModel.dll</HintPath>
+    </Reference>
+    <Reference Include="NuGet.Packaging.Core">
+      <HintPath>..\..\packages\NuGet.Packaging.Core.4.6.0\lib\net46\NuGet.Packaging.Core.dll</HintPath>
+    </Reference>
+    <Reference Include="NuGet.Packaging">
+      <HintPath>..\..\packages\NuGet.Packaging.4.6.0\lib\net46\NuGet.Packaging.dll</HintPath>
+    </Reference>
+    <Reference Include="System.Runtime">
+      <HintPath>..\..\packages\System.Runtime.4.3.0\lib\net462\System.Runtime.dll</HintPath>
+    </Reference>
+    <Reference Include="mscorlib" />
+    <Reference Include="System.ComponentModel.Composition" />
+    <Reference Include="System.Runtime.InteropServices">
+      <HintPath>..\..\packages\System.Runtime.InteropServices.4.3.0\lib\net462\System.Runtime.InteropServices.dll</HintPath>
+    </Reference>
+    <Reference Include="NuGet.Protocol">
+      <HintPath>..\..\packages\NuGet.Protocol.4.6.0\lib\net46\NuGet.Protocol.dll</HintPath>
+    </Reference>
+    <Reference Include="System.IdentityModel" />
+    <Reference Include="System.Net.Http" />
+    <Reference Include="System.Net.Http.WebRequest" />
+    <Reference Include="System.ServiceModel" />
+    <Reference Include="NuGet.DependencyResolver.Core">
+      <HintPath>..\..\packages\NuGet.DependencyResolver.Core.4.6.0\lib\net46\NuGet.DependencyResolver.Core.dll</HintPath>
+    </Reference>
+    <Reference Include="NuGet.ProjectModel">
+      <HintPath>..\..\packages\NuGet.ProjectModel.4.6.0\lib\net46\NuGet.ProjectModel.dll</HintPath>
+    </Reference>
   </ItemGroup>
   <ItemGroup>
     <Compile Include="$(IntermediateOutputPath)Profile.g.cs" />
@@ -507,6 +554,7 @@
     <Compile Include="Utilities\SatelliteAssembly.cs" />
     <Compile Include="Tasks\AndroidApkSigner.cs" />
     <Compile Include="Tasks\Desugar.cs" />
+    <Compile Include="Utilities\NuGetLogger.cs" />
     <None Include="Resources\desugar_deploy.jar">
       <Link>desugar_deploy.jar</Link>
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>