[One .NET] simplify native library inclusion

jonathanpeppers · jonathanpeppers · commit 5b575fc71b28 · 2020-12-16T09:09:16.000-06:00
I've noticed the `_IncludeNativeSystemLibraries` MSBuild target can take 40ms because it is running this workaround:  <FrameworkNativeLibrary Remove="@(ResolvedFileToPublish)" Condition=" $([System.Text.RegularExpressions.Regex]::IsMatch('%(FullPath)', '.*linux-.*')) " /> The `_IncludeNativeSystemLibraries` also can be split up between "legacy" Xamarin.Android and .NET 6. The .NET 6 version can simply check the `@(ResolvedFileToPublish)` item group, as the other parts support the intepreter, profiler, etc., which are not available in .NET 6 yet. I created a new `<ProcessNativeLibraries/>` MSBuild task to simplify the work in C#: * We can iterate over all `.so` files from `@(ResolvedFileToPublish)`. * We can fix up `libmonodroid.so` as needed. * We can emit a warning if the ABI is unknown without using a `Regex`. It is possible for Linux `.so` files to be brought in from NuGet packages as discussed in: dotnet/runtime#45927 The warning is a new alternate message for XA4301, such as: warning XA4301: Cannot determine ABI of native library 'C:\src\xamarin-android\packages\sqlitepclraw.lib.e_sqlite3.linux\1.1.11\runtimes\linux-arm64\native\libe_sqlite3.so'. Remove the 'SQLitePCLRaw.lib.e_sqlite3.linux' NuGet package, or notify the library author. As part of this change, .NET 6 will no longer need to do: <PropertyGroup> <_Assemblies>@(_ResolvedFrameworkAssemblies)</_Assemblies> So we can avoid creating a ~14K character string. ~~ Results ~~ Timing the project fromm the `DotNetBuild(android.21-arm,false)` test, in a build with no changes: Before: 40 ms _IncludeNativeSystemLibraries 1 calls After: 4 ms _IncludeNativeSystemLibraries 1 calls This should save ~36ms from all .NET 6 builds.
diff --git a/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.AssemblyResolution.targets b/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.AssemblyResolution.targets
@@ -10,7 +10,8 @@ _ResolveAssemblies MSBuild target.
 
 <Project xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
 
-  <UsingTask TaskName="Xamarin.Android.Tasks.ProcessAssemblies" AssemblyFile="$(_XamarinAndroidBuildTasksAssembly)" />
+  <UsingTask TaskName="Xamarin.Android.Tasks.ProcessAssemblies"      AssemblyFile="$(_XamarinAndroidBuildTasksAssembly)" />
+  <UsingTask TaskName="Xamarin.Android.Tasks.ProcessNativeLibraries" AssemblyFile="$(_XamarinAndroidBuildTasksAssembly)" />
 
   <PropertyGroup Condition=" '$(_ComputeFilesToPublishForRuntimeIdentifiers)' == 'true' ">
     <OutputPath Condition=" '$(_OuterOutputPath)' != '' ">$(_OuterOutputPath)</OutputPath>
@@ -157,4 +158,15 @@ _ResolveAssemblies MSBuild target.
     </ItemGroup>
   </Target>
 
+  <Target Name="_IncludeNativeSystemLibraries">
+    <ItemGroup>
+      <_ResolvedNativeLibraries Include="@(ResolvedFileToPublish)" Condition=" '%(ResolvedFileToPublish.Extension)' == '.so' " />
+    </ItemGroup>
+    <ProcessNativeLibraries
+        InputLibraries="@(_ResolvedNativeLibraries)"
+        IncludeDebugSymbols="$(AndroidIncludeDebugSymbols)">
+      <Output TaskParameter="OutputLibraries" ItemName="FrameworkNativeLibrary" />
+    </ProcessNativeLibraries>
+  </Target>
+
 </Project>
diff --git a/src/Xamarin.Android.Build.Tasks/Properties/Resources.resx b/src/Xamarin.Android.Build.Tasks/Properties/Resources.resx
@@ -661,6 +661,12 @@ The following are literal names and should not be translated: ABI, 'libs/armeabi
     <value>Cannot determine ABI of native library '{0}'. Move this file to a directory with a valid Android ABI name such as 'libs/armeabi-v7a/'.</value>
     <comment>The following are literal names and should not be translated: ABI, 'libs/armeabi-v7a/'
 {0} - The file path</comment>
+  </data>
+  <data name="XA4301_ABI_NuGet" xml:space="preserve">
+    <value>Cannot determine ABI of native library '{0}'. Remove the '{1}' NuGet package, or notify the library author.</value>
+    <comment>The following are literal names and should not be translated: ABI, NuGet
+{0} - The native library file path
+{1} - Either the name of the NuGet package or a path to the native library</comment>
   </data>
   <data name="XA4301_ABI_Ignoring" xml:space="preserve">
     <value>Could not determine ABI of some native libraries. Ignoring those: {0}</value>
diff --git a/src/Xamarin.Android.Build.Tasks/Properties/xlf/Resources.cs.xlf b/src/Xamarin.Android.Build.Tasks/Properties/xlf/Resources.cs.xlf
@@ -64,6 +64,13 @@ The capitalized word "Portable" that appears earlier in the message is plain tex
         <note>In this message, the term "bundled" is a short way of saying "included into the final application package."
 The following are literal names and should not be translated: ABI, 'libs/armeabi-v7a/'
 {0} - The file path</note>
+      </trans-unit>
+      <trans-unit id="XA4301_ABI_NuGet">
+        <source>Cannot determine ABI of native library '{0}'. Remove the '{1}' NuGet package, or notify the library author.</source>
+        <target state="new">Cannot determine ABI of native library '{0}'. Remove the '{1}' NuGet package, or notify the library author.</target>
+        <note>The following are literal names and should not be translated: ABI, NuGet
+{0} - The native library file path
+{1} - Either the name of the NuGet package or a path to the native library</note>
       </trans-unit>
       <trans-unit id="XA_Manifest_AutoGenerated_Header">
         <source>
diff --git a/src/Xamarin.Android.Build.Tasks/Properties/xlf/Resources.de.xlf b/src/Xamarin.Android.Build.Tasks/Properties/xlf/Resources.de.xlf
@@ -64,6 +64,13 @@ The capitalized word "Portable" that appears earlier in the message is plain tex
         <note>In this message, the term "bundled" is a short way of saying "included into the final application package."
 The following are literal names and should not be translated: ABI, 'libs/armeabi-v7a/'
 {0} - The file path</note>
+      </trans-unit>
+      <trans-unit id="XA4301_ABI_NuGet">
+        <source>Cannot determine ABI of native library '{0}'. Remove the '{1}' NuGet package, or notify the library author.</source>
+        <target state="new">Cannot determine ABI of native library '{0}'. Remove the '{1}' NuGet package, or notify the library author.</target>
+        <note>The following are literal names and should not be translated: ABI, NuGet
+{0} - The native library file path
+{1} - Either the name of the NuGet package or a path to the native library</note>
       </trans-unit>
       <trans-unit id="XA_Manifest_AutoGenerated_Header">
         <source>
diff --git a/src/Xamarin.Android.Build.Tasks/Properties/xlf/Resources.es.xlf b/src/Xamarin.Android.Build.Tasks/Properties/xlf/Resources.es.xlf
@@ -64,6 +64,13 @@ The capitalized word "Portable" that appears earlier in the message is plain tex
         <note>In this message, the term "bundled" is a short way of saying "included into the final application package."
 The following are literal names and should not be translated: ABI, 'libs/armeabi-v7a/'
 {0} - The file path</note>
+      </trans-unit>
+      <trans-unit id="XA4301_ABI_NuGet">
+        <source>Cannot determine ABI of native library '{0}'. Remove the '{1}' NuGet package, or notify the library author.</source>
+        <target state="new">Cannot determine ABI of native library '{0}'. Remove the '{1}' NuGet package, or notify the library author.</target>
+        <note>The following are literal names and should not be translated: ABI, NuGet
+{0} - The native library file path
+{1} - Either the name of the NuGet package or a path to the native library</note>
       </trans-unit>
       <trans-unit id="XA_Manifest_AutoGenerated_Header">
         <source>
diff --git a/src/Xamarin.Android.Build.Tasks/Properties/xlf/Resources.fr.xlf b/src/Xamarin.Android.Build.Tasks/Properties/xlf/Resources.fr.xlf
@@ -64,6 +64,13 @@ The capitalized word "Portable" that appears earlier in the message is plain tex
         <note>In this message, the term "bundled" is a short way of saying "included into the final application package."
 The following are literal names and should not be translated: ABI, 'libs/armeabi-v7a/'
 {0} - The file path</note>
+      </trans-unit>
+      <trans-unit id="XA4301_ABI_NuGet">
+        <source>Cannot determine ABI of native library '{0}'. Remove the '{1}' NuGet package, or notify the library author.</source>
+        <target state="new">Cannot determine ABI of native library '{0}'. Remove the '{1}' NuGet package, or notify the library author.</target>
+        <note>The following are literal names and should not be translated: ABI, NuGet
+{0} - The native library file path
+{1} - Either the name of the NuGet package or a path to the native library</note>
       </trans-unit>
       <trans-unit id="XA_Manifest_AutoGenerated_Header">
         <source>
diff --git a/src/Xamarin.Android.Build.Tasks/Properties/xlf/Resources.it.xlf b/src/Xamarin.Android.Build.Tasks/Properties/xlf/Resources.it.xlf
@@ -64,6 +64,13 @@ The capitalized word "Portable" that appears earlier in the message is plain tex
         <note>In this message, the term "bundled" is a short way of saying "included into the final application package."
 The following are literal names and should not be translated: ABI, 'libs/armeabi-v7a/'
 {0} - The file path</note>
+      </trans-unit>
+      <trans-unit id="XA4301_ABI_NuGet">
+        <source>Cannot determine ABI of native library '{0}'. Remove the '{1}' NuGet package, or notify the library author.</source>
+        <target state="new">Cannot determine ABI of native library '{0}'. Remove the '{1}' NuGet package, or notify the library author.</target>
+        <note>The following are literal names and should not be translated: ABI, NuGet
+{0} - The native library file path
+{1} - Either the name of the NuGet package or a path to the native library</note>
       </trans-unit>
       <trans-unit id="XA_Manifest_AutoGenerated_Header">
         <source>
diff --git a/src/Xamarin.Android.Build.Tasks/Properties/xlf/Resources.ja.xlf b/src/Xamarin.Android.Build.Tasks/Properties/xlf/Resources.ja.xlf
@@ -64,6 +64,13 @@ The capitalized word "Portable" that appears earlier in the message is plain tex
         <note>In this message, the term "bundled" is a short way of saying "included into the final application package."
 The following are literal names and should not be translated: ABI, 'libs/armeabi-v7a/'
 {0} - The file path</note>
+      </trans-unit>
+      <trans-unit id="XA4301_ABI_NuGet">
+        <source>Cannot determine ABI of native library '{0}'. Remove the '{1}' NuGet package, or notify the library author.</source>
+        <target state="new">Cannot determine ABI of native library '{0}'. Remove the '{1}' NuGet package, or notify the library author.</target>
+        <note>The following are literal names and should not be translated: ABI, NuGet
+{0} - The native library file path
+{1} - Either the name of the NuGet package or a path to the native library</note>
       </trans-unit>
       <trans-unit id="XA_Manifest_AutoGenerated_Header">
         <source>
diff --git a/src/Xamarin.Android.Build.Tasks/Properties/xlf/Resources.ko.xlf b/src/Xamarin.Android.Build.Tasks/Properties/xlf/Resources.ko.xlf
@@ -64,6 +64,13 @@ The capitalized word "Portable" that appears earlier in the message is plain tex
         <note>In this message, the term "bundled" is a short way of saying "included into the final application package."
 The following are literal names and should not be translated: ABI, 'libs/armeabi-v7a/'
 {0} - The file path</note>
+      </trans-unit>
+      <trans-unit id="XA4301_ABI_NuGet">
+        <source>Cannot determine ABI of native library '{0}'. Remove the '{1}' NuGet package, or notify the library author.</source>
+        <target state="new">Cannot determine ABI of native library '{0}'. Remove the '{1}' NuGet package, or notify the library author.</target>
+        <note>The following are literal names and should not be translated: ABI, NuGet
+{0} - The native library file path
+{1} - Either the name of the NuGet package or a path to the native library</note>
       </trans-unit>
       <trans-unit id="XA_Manifest_AutoGenerated_Header">
         <source>
diff --git a/src/Xamarin.Android.Build.Tasks/Properties/xlf/Resources.pl.xlf b/src/Xamarin.Android.Build.Tasks/Properties/xlf/Resources.pl.xlf
@@ -64,6 +64,13 @@ The capitalized word "Portable" that appears earlier in the message is plain tex
         <note>In this message, the term "bundled" is a short way of saying "included into the final application package."
 The following are literal names and should not be translated: ABI, 'libs/armeabi-v7a/'
 {0} - The file path</note>
+      </trans-unit>
+      <trans-unit id="XA4301_ABI_NuGet">
+        <source>Cannot determine ABI of native library '{0}'. Remove the '{1}' NuGet package, or notify the library author.</source>
+        <target state="new">Cannot determine ABI of native library '{0}'. Remove the '{1}' NuGet package, or notify the library author.</target>
+        <note>The following are literal names and should not be translated: ABI, NuGet
+{0} - The native library file path
+{1} - Either the name of the NuGet package or a path to the native library</note>
       </trans-unit>
       <trans-unit id="XA_Manifest_AutoGenerated_Header">
         <source>
diff --git a/src/Xamarin.Android.Build.Tasks/Properties/xlf/Resources.pt-BR.xlf b/src/Xamarin.Android.Build.Tasks/Properties/xlf/Resources.pt-BR.xlf
@@ -64,6 +64,13 @@ The capitalized word "Portable" that appears earlier in the message is plain tex
         <note>In this message, the term "bundled" is a short way of saying "included into the final application package."
 The following are literal names and should not be translated: ABI, 'libs/armeabi-v7a/'
 {0} - The file path</note>
+      </trans-unit>
+      <trans-unit id="XA4301_ABI_NuGet">
+        <source>Cannot determine ABI of native library '{0}'. Remove the '{1}' NuGet package, or notify the library author.</source>
+        <target state="new">Cannot determine ABI of native library '{0}'. Remove the '{1}' NuGet package, or notify the library author.</target>
+        <note>The following are literal names and should not be translated: ABI, NuGet
+{0} - The native library file path
+{1} - Either the name of the NuGet package or a path to the native library</note>
       </trans-unit>
       <trans-unit id="XA_Manifest_AutoGenerated_Header">
         <source>
diff --git a/src/Xamarin.Android.Build.Tasks/Properties/xlf/Resources.ru.xlf b/src/Xamarin.Android.Build.Tasks/Properties/xlf/Resources.ru.xlf
@@ -64,6 +64,13 @@ The capitalized word "Portable" that appears earlier in the message is plain tex
         <note>In this message, the term "bundled" is a short way of saying "included into the final application package."
 The following are literal names and should not be translated: ABI, 'libs/armeabi-v7a/'
 {0} - The file path</note>
+      </trans-unit>
+      <trans-unit id="XA4301_ABI_NuGet">
+        <source>Cannot determine ABI of native library '{0}'. Remove the '{1}' NuGet package, or notify the library author.</source>
+        <target state="new">Cannot determine ABI of native library '{0}'. Remove the '{1}' NuGet package, or notify the library author.</target>
+        <note>The following are literal names and should not be translated: ABI, NuGet
+{0} - The native library file path
+{1} - Either the name of the NuGet package or a path to the native library</note>
       </trans-unit>
       <trans-unit id="XA_Manifest_AutoGenerated_Header">
         <source>
diff --git a/src/Xamarin.Android.Build.Tasks/Properties/xlf/Resources.tr.xlf b/src/Xamarin.Android.Build.Tasks/Properties/xlf/Resources.tr.xlf
@@ -64,6 +64,13 @@ The capitalized word "Portable" that appears earlier in the message is plain tex
         <note>In this message, the term "bundled" is a short way of saying "included into the final application package."
 The following are literal names and should not be translated: ABI, 'libs/armeabi-v7a/'
 {0} - The file path</note>
+      </trans-unit>
+      <trans-unit id="XA4301_ABI_NuGet">
+        <source>Cannot determine ABI of native library '{0}'. Remove the '{1}' NuGet package, or notify the library author.</source>
+        <target state="new">Cannot determine ABI of native library '{0}'. Remove the '{1}' NuGet package, or notify the library author.</target>
+        <note>The following are literal names and should not be translated: ABI, NuGet
+{0} - The native library file path
+{1} - Either the name of the NuGet package or a path to the native library</note>
       </trans-unit>
       <trans-unit id="XA_Manifest_AutoGenerated_Header">
         <source>
diff --git a/src/Xamarin.Android.Build.Tasks/Properties/xlf/Resources.zh-Hans.xlf b/src/Xamarin.Android.Build.Tasks/Properties/xlf/Resources.zh-Hans.xlf
@@ -64,6 +64,13 @@ The capitalized word "Portable" that appears earlier in the message is plain tex
         <note>In this message, the term "bundled" is a short way of saying "included into the final application package."
 The following are literal names and should not be translated: ABI, 'libs/armeabi-v7a/'
 {0} - The file path</note>
+      </trans-unit>
+      <trans-unit id="XA4301_ABI_NuGet">
+        <source>Cannot determine ABI of native library '{0}'. Remove the '{1}' NuGet package, or notify the library author.</source>
+        <target state="new">Cannot determine ABI of native library '{0}'. Remove the '{1}' NuGet package, or notify the library author.</target>
+        <note>The following are literal names and should not be translated: ABI, NuGet
+{0} - The native library file path
+{1} - Either the name of the NuGet package or a path to the native library</note>
       </trans-unit>
       <trans-unit id="XA_Manifest_AutoGenerated_Header">
         <source>
diff --git a/src/Xamarin.Android.Build.Tasks/Properties/xlf/Resources.zh-Hant.xlf b/src/Xamarin.Android.Build.Tasks/Properties/xlf/Resources.zh-Hant.xlf
@@ -64,6 +64,13 @@ The capitalized word "Portable" that appears earlier in the message is plain tex
         <note>In this message, the term "bundled" is a short way of saying "included into the final application package."
 The following are literal names and should not be translated: ABI, 'libs/armeabi-v7a/'
 {0} - The file path</note>
+      </trans-unit>
+      <trans-unit id="XA4301_ABI_NuGet">
+        <source>Cannot determine ABI of native library '{0}'. Remove the '{1}' NuGet package, or notify the library author.</source>
+        <target state="new">Cannot determine ABI of native library '{0}'. Remove the '{1}' NuGet package, or notify the library author.</target>
+        <note>The following are literal names and should not be translated: ABI, NuGet
+{0} - The native library file path
+{1} - Either the name of the NuGet package or a path to the native library</note>
       </trans-unit>
       <trans-unit id="XA_Manifest_AutoGenerated_Header">
         <source>
diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/ProcessNativeLibraries.cs b/src/Xamarin.Android.Build.Tasks/Tasks/ProcessNativeLibraries.cs
@@ -0,0 +1,66 @@
+using System.Collections.Generic;
+using System.IO;
+using Microsoft.Build.Framework;
+
+namespace Xamarin.Android.Tasks
+{
+	/// <summary>
+	/// Processes .so files coming from @(ResolvedFileToPublish).
+	/// * Checks if ABI is valid
+	/// * Fixes up libmonodroid.so based on $(AndroidIncludeDebugSymbols)
+	/// </summary>
+	public class ProcessNativeLibraries : AndroidTask
+	{
+		public override string TaskPrefix => "PRNL";
+
+		/// <summary>
+		/// Assumed to be .so files only
+		/// </summary>
+		public ITaskItem [] InputLibraries { get; set; }
+
+		public bool IncludeDebugSymbols { get; set; }
+
+		[Output]
+		public ITaskItem [] OutputLibraries { get; set; }
+
+		public override bool RunTask ()
+		{
+			if (InputLibraries == null || InputLibraries.Length == 0)
+				return true;
+
+			var output = new List<ITaskItem> (InputLibraries.Length);
+
+			foreach (var library in InputLibraries) {
+				var abi = MonoAndroidHelper.GetNativeLibraryAbi (library);
+				if (string.IsNullOrEmpty (abi)) {
+					var packageId = library.GetMetadata ("NuGetPackageId");
+					if (!string.IsNullOrEmpty (packageId)) {
+						Log.LogCodedWarning ("XA4301", library.ItemSpec, 0, Properties.Resources.XA4301_ABI_NuGet, library.ItemSpec, packageId);
+					} else {
+						Log.LogCodedWarning ("XA4301", library.ItemSpec, 0, Properties.Resources.XA4301_ABI, library.ItemSpec);
+					}
+					continue;
+				}
+				var fileName = Path.GetFileNameWithoutExtension (library.ItemSpec);
+				if (fileName == "libmono-android.debug") {
+					if (IncludeDebugSymbols) {
+						library.SetMetadata ("ArchiveFileName", "libmonodroid.so");
+					} else {
+						continue;
+					}
+				} else if (fileName == "libmono-android.release") {
+					if (IncludeDebugSymbols) {
+						continue;
+					} else {
+						library.SetMetadata ("ArchiveFileName", "libmonodroid.so");
+					}
+				}
+				output.Add (library);
+			}
+
+			OutputLibraries = output.ToArray ();
+
+			return !Log.HasLoggedErrors;
+		}
+	}
+}
diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/BuildTest.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/BuildTest.cs
@@ -3882,6 +3882,9 @@ public void AbiNameInIntermediateOutputPath ()
 			using (var b = CreateApkBuilder (Path.Combine ("temp", TestName))) {
 				Assert.IsTrue (b.Build (proj), "Build should have succeeded.");
 				Assert.IsFalse (StringAssertEx.ContainsText (b.LastBuildOutput, Path.Combine ("armeabi", "libe_sqlite3.so")), "Build should not use `armeabi`.");
+				if (Builder.UseDotNet) {
+					StringAssertEx.Contains ("warning XA4301", b.LastBuildOutput, "Should get a XA4301 warning");
+				}
 			}
 		}
 
diff --git a/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets b/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets
diff --git a/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Legacy.targets b/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Legacy.targets
