diff --git a/Configuration.props b/Configuration.props index bf26d82ab51..75fefeeca48 100644 --- a/Configuration.props +++ b/Configuration.props @@ -25,6 +25,7 @@ v4.4 19 + 21 21 $(AndroidFirstApiLevel) diff --git a/Documentation/README.md b/Documentation/README.md index fef9409c8d9..cb645dcbbe1 100644 --- a/Documentation/README.md +++ b/Documentation/README.md @@ -11,6 +11,7 @@ * [Submitting Bugs, Feature Requests, and Pull Requests][bugs] * [Directory Structure](project-docs/ExploringSources.md) + * [Assembly store format](project-docs/AssemblyStores.md) [bugs]: https://github.com/xamarin/xamarin-android/wiki/Submitting-Bugs,-Feature-Requests,-and-Pull-Requests diff --git a/Documentation/project-docs/AssemblyStores.md b/Documentation/project-docs/AssemblyStores.md new file mode 100644 index 00000000000..57be490322b --- /dev/null +++ b/Documentation/project-docs/AssemblyStores.md @@ -0,0 +1,224 @@ + +**Table of Contents** + +- [Assembly Store format and purpose](#assembly-store-format-and-purpose) + - [Rationale](#rationale) +- [Store kinds and locations](#store-kinds-and-locations) +- [Store format](#store-format) + - [Common header](#common-header) + - [Assembly descriptor table](#assembly-descriptor-table) + - [Index store](#index-store) + - [Hash table format](#hash-table-format) + + + +# Assembly Store format and purpose + +Assembly stores are binary files which contain the managed +assemblies, their debug data (optionally) and the associated config +file (optionally). They are placed inside the Android APK/AAB +archives, replacing individual assemblies/pdb/config files. + +Assembly stores are an optional form of assembly storage in the +archive, they can be used in all build configurations **except** when +Fast Deployment is in effect (in which case assemblies aren't placed +in the archives at all, they are instead synchronized from the host to +the device/emulator filesystem) + +## Rationale + +During native startup, the Xamarin.Android runtime looks inside the +application APK file for the managed assemblies (and their associated +pdb and config files, if applicable) in order to map them (using the +`mmap(2)` call) into memory so that they can be given to the Mono +runtime when it requests a given assembly is loaded. The reason for +the memory mapping is that, as far as Android is concerned, managed +assembly files are just data/resources and, thus, aren't extracted to +the filesystem. As a result, Mono wouldn't be able to find the +assemblies by scanning the filesystem - the host application +(Xamarin.Android) must give it a hand in finding them. + +Applications can contain hundreds of assemblies (for instance a Hello +World MAUI application currently contains over 120 assemblies) and +each of them would have to be mmapped at startup, together with its +pdb and config files, if found. This not only costs time (each `mmap` +invocation is a system call) but it also makes the assembly discovery +an O(n) algorithm, which takes more time as more assemblies are added +to the APK/AAB archive. + +An assembly store, however, needs to be mapped only once and any +further operations are merely pointer arithmetic, making the process +not only faster but also reducing the algorithm complexity to O(1). + +# Store kinds and locations + +Each application will contain at least a single assembly store, with +assemblies that are architecture-agnostics and any number of +architecture-specific stores. dotnet ships with a handful of +assemblies that **are** architecture-specific - those assemblies are +placed in an architecture specific store, one per architecture +supported by and enabled for the application. On the execution time, +the Xamarin.Android runtime will always map the architecture-agnostic +store and one, and **only** one, of the architecture-specific stores. + +Stores are placed in the same location in the APK/AAB archive where the +individual assemblies traditionally live, the `assemblies/` (for APK) +and `base/root/assemblies/` (for AAB) folders. + +The architecture agnostic store is always named `assemblies.blob` while +the architecture-specific one is called `assemblies.[ARCH].blob`. + +Each APK in the application (e.g. the future Feature APKs) **may** +contain the above two assembly store files (some APKs may contain only +resources, other may contain only native libraries etc) + +Currently, Xamarin.Android applications will produce only one set of +stores but when Xamarin.Android adds support for Android Features, each +feature APK will contain its own set of stores. All of the APKs will +follow the location, format and naming conventions described above. + +# Store format + +Each store is a structured binary file, using little-endian byte order +and aligned to a byte boundary. Each store consists of a header, an +assembly descriptor table and, optionally (see below), two tables with +assembly name hashes. All the stores are assigned a unique ID, with +the store having ID equal to `0` being the [Index store](#index-store) + +Assemblies are stored as adjacent byte streams: + + - **Image data** + Required to be present for all assemblies, contains the actual + assembly PE image. + - **Debug data** + Optional. Contains the assembly's PDB or MDB debug data. + - **Config data** + Optional. Contains the assembly's .config file. Config data + **must** be terminated with a `NUL` character (`0`), this is to + make runtime code slightly more efficient. + +All the structures described here are defined in the +[`xamarin-app.hh`](../../src/monodroid/jni/xamarin-app.hh) file. +Should there be any difference between this document and the +structures in the header file, the information from the header is the +one that should be trusted. + +## Common header + +All kinds of stores share the following header format: + + struct AssemblyStoreHeader + { + uint32_t magic; + uint32_t version; + uint32_t local_entry_count; + uint32_t global_entry_count; + uint32_t store_id; + ; + +Individual fields have the following meanings: + + - `magic`: has the value of 0x41424158 (`XABA`) + - `version`: a value increased every time assembly store format changes. + - `local_entry_count`: number of assemblies stored in this assembly + store (also the number of entries in the assembly descriptor + table, see below) + - `global_entry_count`: number of entries in the index store's (see + below) hash tables and, thus, the number of assemblies stored in + **all** of the assembly stores across **all** of the application's + APK files, all the other assembly stores have `0` in this field + since they do **not** have the hash tables. + - `store_id`: a unique ID of this store. + +## Assembly descriptor table + +Each store header is followed by a table of +`AssemblyStoreHeader.local_entry_count` entries, each entry +defined by the following structure: + + struct AssemblyStoreAssemblyDescriptor + { + uint32_t data_offset; + uint32_t data_size; + uint32_t debug_data_offset; + uint32_t debug_data_size; + uint32_t config_data_offset; + uint32_t config_data_size; + }; + +Only the `data_offset` and `data_size` fields must have a non-zero +value, other fields describe optional data and can be set to `0`. + +Individual fields have the following meanings: + + - `data_offset`: offset of the assembly image data from the + beginning of the store file + - `data_size`: number of bytes of the image data + - `debug_data_offset`: offset of the assembly's debug data from the + beginning of the store file. A value of `0` indicates there's no + debug data for this assembly. + - `debug_data_size`: number of bytes of debug data. Can be `0` only + if `debug_data_offset` is `0` + - `config_data_offset`: offset of the assembly's config file data + from the beginning of the store file. A value of `0` indicates + there's no config file data for this assembly. + - `config_data_size`: number of bytes of config file data. Can be + `0` only if `config_data_offset` is `0` + +## Index store + +Each application will contain exactly one store with a global index - +two tables with assembly name hashes. All the other stores **do not** +contain these tables. Two hash tables are necessary because hashes +for 32-bit and 64-bit devices are different. + +The hash tables follow the [Assembly descriptor +table](#assembly-descriptor-table) and precede the individual assembly +streams. + +Placing the hash tables in a single index store, while "wasting" a +certain amount of memory (since 32-bit devices won't use the 64-bit +table and vice versa), makes for simpler and faster runtime +implementation and the amount of memory wasted isn't big (1000 +two tables which are 8kb long each, this being the amount of memory +wasted) + +### Hash table format + +Both tables share the same format, despite the hashes themselves being +of different sizes. This is done to make handling of the tables +easier on the runtime. + +Each entry contains, among other fields, the assembly name hash. In +case of satellite assemblies, the assembly culture (e.g. `en/` or +`fr/`) is treated as part of the assembly name, thus resulting in a +unique hash. The hash value is obtained using the +[xxHash](https://cyan4973.github.io/xxHash/) algorithm and is +calculated **without** including the `.dll` extension. This is done +for runtime efficiency as the vast majority of Mono requests to load +an assembly does not include the `.dll` suffix, thus saving us time of +appending it in order to generate the hash for index lookup. + +Each entry is represented by the following structure: + + struct AssemblyStoreHashEntry + { + union { + uint64_t hash64; + uint32_t hash32; + }; + uint32_t mapping_index; + uint32_t local_store_index; + uint32_t store_id; + }; + +Individual fields have the following meanings: + + - `hash64`/`hash32`: the 32-bit or 64-bit hash of the assembly's name + **without** the `.dll` suffix + - `mapping_index`: index into a compile-time generated array of + assembly data pointers. This is a global index, unique across + **all** the APK files comprising the application. + - `local_store_index`: index into assembly store [Assembly descriptor table](#assembly-descriptor-table) + describing the assembly. + - `store_id`: ID of the assembly store containing the assembly diff --git a/Xamarin.Android.sln b/Xamarin.Android.sln index 3804cbfc365..608787bbea4 100644 --- a/Xamarin.Android.sln +++ b/Xamarin.Android.sln @@ -148,6 +148,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "decompress-assemblies", "to EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "tmt", "tools\tmt\tmt.csproj", "{1A273ED2-AE84-48E9-9C23-E978C2D0CB34}" EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "assembly-store-reader", "tools\assembly-store-reader\assembly-store-reader.csproj", "{DA50FC92-7FE7-48B5-BDB6-CDA57B37BB51}" +EndProject Global GlobalSection(SharedMSBuildProjectFiles) = preSolution src\Xamarin.Android.NamingCustomAttributes\Xamarin.Android.NamingCustomAttributes.projitems*{3f1f2f50-af1a-4a5a-bedb-193372f068d7}*SharedItemsImports = 5 @@ -408,6 +410,10 @@ Global {1FED3F23-1175-42AA-BE87-EF1E8DB52F8B}.Debug|AnyCPU.Build.0 = Debug|Any CPU {1FED3F23-1175-42AA-BE87-EF1E8DB52F8B}.Release|AnyCPU.ActiveCfg = Release|Any CPU {1FED3F23-1175-42AA-BE87-EF1E8DB52F8B}.Release|AnyCPU.Build.0 = Release|Any CPU + {DA50FC92-7FE7-48B5-BDB6-CDA57B37BB51}.Debug|AnyCPU.ActiveCfg = Debug|anycpu + {DA50FC92-7FE7-48B5-BDB6-CDA57B37BB51}.Debug|AnyCPU.Build.0 = Debug|anycpu + {DA50FC92-7FE7-48B5-BDB6-CDA57B37BB51}.Release|AnyCPU.ActiveCfg = Release|anycpu + {DA50FC92-7FE7-48B5-BDB6-CDA57B37BB51}.Release|AnyCPU.Build.0 = Release|anycpu EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -474,6 +480,7 @@ Global {37FCD325-1077-4603-98E7-4509CAD648D6} = {864062D3-A415-4A6F-9324-5820237BA058} {88B746FF-8D6E-464D-9D66-FF2ECCF148E0} = {864062D3-A415-4A6F-9324-5820237BA058} {1A273ED2-AE84-48E9-9C23-E978C2D0CB34} = {864062D3-A415-4A6F-9324-5820237BA058} + {DA50FC92-7FE7-48B5-BDB6-CDA57B37BB51} = {864062D3-A415-4A6F-9324-5820237BA058} {1FED3F23-1175-42AA-BE87-EF1E8DB52F8B} = {04E3E11E-B47D-4599-8AFC-50515A95E715} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution diff --git a/build-tools/automation/azure-pipelines.yaml b/build-tools/automation/azure-pipelines.yaml index 50558cc8f8c..92b936f8555 100644 --- a/build-tools/automation/azure-pipelines.yaml +++ b/build-tools/automation/azure-pipelines.yaml @@ -397,6 +397,9 @@ stages: cancelTimeoutInMinutes: 2 workspace: clean: all + variables: + CXX: g++-10 + CC: gcc-10 steps: - checkout: self clean: true diff --git a/build-tools/installers/create-installers.targets b/build-tools/installers/create-installers.targets index c74d6387bfc..cf289da4f98 100644 --- a/build-tools/installers/create-installers.targets +++ b/build-tools/installers/create-installers.targets @@ -290,6 +290,7 @@ <_MSBuildFiles Include="$(MSBuildSrcDir)\Xamarin.SourceWriter.dll" /> <_MSBuildFiles Include="$(MSBuildSrcDir)\Xamarin.SourceWriter.pdb" /> <_MSBuildFiles Include="$(MSBuildSrcDir)\K4os.Compression.LZ4.dll" /> + <_MSBuildFiles Include="$(MSBuildSrcDir)\K4os.Hash.xxHash.dll" /> <_MSBuildTargetsSrcFiles Include="$(MSBuildTargetsSrcDir)\Xamarin.Android.AvailableItems.targets" /> diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs b/src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs index f37a98b0a15..9672f853348 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs @@ -91,6 +91,8 @@ public class BuildApk : AndroidTask public string RuntimeConfigBinFilePath { get; set; } + public bool UseAssemblyStore { get; set; } + [Required] public string ProjectFullPath { get; set; } @@ -120,7 +122,7 @@ protected virtual void FixupArchive (ZipArchiveEx zip) { } List existingEntries = new List (); - void ExecuteWithAbi (string [] supportedAbis, string apkInputPath, string apkOutputPath, bool debug, bool compress, IDictionary compressedAssembliesInfo) + void ExecuteWithAbi (string [] supportedAbis, string apkInputPath, string apkOutputPath, bool debug, bool compress, IDictionary compressedAssembliesInfo, string assemblyStoreApkName) { ArchiveFileList files = new ArchiveFileList (); bool refresh = true; @@ -180,7 +182,7 @@ void ExecuteWithAbi (string [] supportedAbis, string apkInputPath, string apkOut } if (EmbedAssemblies && !BundleAssemblies) - AddAssemblies (apk, debug, compress, compressedAssembliesInfo); + AddAssemblies (apk, debug, compress, compressedAssembliesInfo, assemblyStoreApkName); AddRuntimeLibraries (apk, supportedAbis); apk.Flush(); @@ -301,7 +303,7 @@ public override bool RunTask () throw new InvalidOperationException ($"Assembly compression info not found for key '{key}'. Compression will not be performed."); } - ExecuteWithAbi (SupportedAbis, ApkInputPath, ApkOutputPath, debug, compress, compressedAssembliesInfo); + ExecuteWithAbi (SupportedAbis, ApkInputPath, ApkOutputPath, debug, compress, compressedAssembliesInfo, assemblyStoreApkName: null); outputFiles.Add (ApkOutputPath); if (CreatePackagePerAbi && SupportedAbis.Length > 1) { foreach (var abi in SupportedAbis) { @@ -310,7 +312,7 @@ public override bool RunTask () var apk = Path.GetFileNameWithoutExtension (ApkOutputPath); ExecuteWithAbi (new [] { abi }, String.Format ("{0}-{1}", ApkInputPath, abi), Path.Combine (path, String.Format ("{0}-{1}.apk", apk, abi)), - debug, compress, compressedAssembliesInfo); + debug, compress, compressedAssembliesInfo, assemblyStoreApkName: abi); outputFiles.Add (Path.Combine (path, String.Format ("{0}-{1}.apk", apk, abi))); } } @@ -322,84 +324,137 @@ public override bool RunTask () return !Log.HasLoggedErrors; } - void AddAssemblies (ZipArchiveEx apk, bool debug, bool compress, IDictionary compressedAssembliesInfo) + void AddAssemblies (ZipArchiveEx apk, bool debug, bool compress, IDictionary compressedAssembliesInfo, string assemblyStoreApkName) { string sourcePath; AssemblyCompression.AssemblyData compressedAssembly = null; string compressedOutputDir = Path.GetFullPath (Path.Combine (Path.GetDirectoryName (ApkOutputPath), "..", "lz4")); + AssemblyStoreGenerator storeGenerator; + + if (UseAssemblyStore) { + storeGenerator = new AssemblyStoreGenerator (AssembliesPath, Log); + } else { + storeGenerator = null; + } int count = 0; - foreach (ITaskItem assembly in ResolvedUserAssemblies) { - if (bool.TryParse (assembly.GetMetadata ("AndroidSkipAddToPackage"), out bool value) && value) { - Log.LogDebugMessage ($"Skipping {assembly.ItemSpec} due to 'AndroidSkipAddToPackage' == 'true' "); - continue; - } - if (MonoAndroidHelper.IsReferenceAssembly (assembly.ItemSpec)) { - Log.LogCodedWarning ("XA0107", assembly.ItemSpec, 0, Properties.Resources.XA0107, assembly.ItemSpec); - } + AssemblyStoreAssemblyInfo storeAssembly = null; + + // + // NOTE + // + // The very first store (ID 0) **must** contain an index of all the assemblies included in the application, even if they + // are included in other APKs than the base one. The ID 0 store **must** be placed in the base assembly + // + + // Currently, all the assembly stores end up in the "base" apk (the APK name is the key in the dictionary below) but the code is ready for the time when we + // partition assemblies into "feature" APKs + const string DefaultBaseApkName = "base"; + if (String.IsNullOrEmpty (assemblyStoreApkName)) { + assemblyStoreApkName = DefaultBaseApkName; + } - sourcePath = CompressAssembly (assembly); + // Add user assemblies + AddAssembliesFromCollection (ResolvedUserAssemblies); - // Add assembly - var assemblyPath = GetAssemblyPath (assembly, frameworkAssembly: false); - AddFileToArchiveIfNewer (apk, sourcePath, assemblyPath + Path.GetFileName (assembly.ItemSpec), compressionMethod: UncompressedMethod); + // Add framework assemblies + count = 0; + AddAssembliesFromCollection (ResolvedFrameworkAssemblies); - // Try to add config if exists - var config = Path.ChangeExtension (assembly.ItemSpec, "dll.config"); - AddAssemblyConfigEntry (apk, assemblyPath, config); + if (!UseAssemblyStore) { + return; + } - // Try to add symbols if Debug - if (debug) { - var symbols = Path.ChangeExtension (assembly.ItemSpec, "dll.mdb"); + Dictionary> assemblyStorePaths = storeGenerator.Generate (Path.GetDirectoryName (ApkOutputPath)); + if (assemblyStorePaths == null) { + throw new InvalidOperationException ("Assembly store generator did not generate any stores"); + } - if (File.Exists (symbols)) - AddFileToArchiveIfNewer (apk, symbols, assemblyPath + Path.GetFileName (symbols), compressionMethod: UncompressedMethod); + if (!assemblyStorePaths.TryGetValue (assemblyStoreApkName, out List baseAssemblyStores) || baseAssemblyStores == null || baseAssemblyStores.Count == 0) { + throw new InvalidOperationException ("Assembly store generator didn't generate the required base stores"); + } - symbols = Path.ChangeExtension (assembly.ItemSpec, "pdb"); + string assemblyStorePrefix = $"{assemblyStoreApkName}_"; + foreach (string assemblyStorePath in baseAssemblyStores) { + string inArchiveName = Path.GetFileName (assemblyStorePath); - if (File.Exists (symbols)) - AddFileToArchiveIfNewer (apk, symbols, assemblyPath + Path.GetFileName (symbols), compressionMethod: UncompressedMethod); + if (inArchiveName.StartsWith (assemblyStorePrefix, StringComparison.Ordinal)) { + inArchiveName = inArchiveName.Substring (assemblyStorePrefix.Length); } - count++; - if (count >= ZipArchiveEx.ZipFlushFilesLimit) { - apk.Flush(); - count = 0; + + CompressionMethod compressionMethod; + if (inArchiveName.EndsWith (".manifest", StringComparison.Ordinal)) { + compressionMethod = CompressionMethod.Default; + } else { + compressionMethod = UncompressedMethod; } + + AddFileToArchiveIfNewer (apk, assemblyStorePath, AssembliesPath + inArchiveName, compressionMethod); } - count = 0; - // Add framework assemblies - foreach (ITaskItem assembly in ResolvedFrameworkAssemblies) { - if (bool.TryParse (assembly.GetMetadata ("AndroidSkipAddToPackage"), out bool value) && value) { - Log.LogDebugMessage ($"Skipping {assembly.ItemSpec} due to 'AndroidSkipAddToPackage' == 'true' "); - continue; - } + void AddAssembliesFromCollection (ITaskItem[] assemblies) + { + foreach (ITaskItem assembly in assemblies) { + if (bool.TryParse (assembly.GetMetadata ("AndroidSkipAddToPackage"), out bool value) && value) { + Log.LogDebugMessage ($"Skipping {assembly.ItemSpec} due to 'AndroidSkipAddToPackage' == 'true' "); + continue; + } - if (MonoAndroidHelper.IsReferenceAssembly (assembly.ItemSpec)) { - Log.LogCodedWarning ("XA0107", assembly.ItemSpec, 0, Properties.Resources.XA0107, assembly.ItemSpec); - } + if (MonoAndroidHelper.IsReferenceAssembly (assembly.ItemSpec)) { + Log.LogCodedWarning ("XA0107", assembly.ItemSpec, 0, Properties.Resources.XA0107, assembly.ItemSpec); + } - sourcePath = CompressAssembly (assembly); - var assemblyPath = GetAssemblyPath (assembly, frameworkAssembly: true); - AddFileToArchiveIfNewer (apk, sourcePath, assemblyPath + Path.GetFileName (assembly.ItemSpec), compressionMethod: UncompressedMethod); - var config = Path.ChangeExtension (assembly.ItemSpec, "dll.config"); - AddAssemblyConfigEntry (apk, assemblyPath, config); - // Try to add symbols if Debug - if (debug) { - var symbols = Path.ChangeExtension (assembly.ItemSpec, "dll.mdb"); + sourcePath = CompressAssembly (assembly); - if (File.Exists (symbols)) - AddFileToArchiveIfNewer (apk, symbols, assemblyPath + Path.GetFileName (symbols), compressionMethod: UncompressedMethod); + // Add assembly + var assemblyPath = GetAssemblyPath (assembly, frameworkAssembly: false); + if (UseAssemblyStore) { + storeAssembly = new AssemblyStoreAssemblyInfo (sourcePath, assemblyPath, assembly.GetMetadata ("Abi")); + } else { + AddFileToArchiveIfNewer (apk, sourcePath, assemblyPath + Path.GetFileName (assembly.ItemSpec), compressionMethod: UncompressedMethod); + } - symbols = Path.ChangeExtension (assembly.ItemSpec, "pdb"); + // Try to add config if exists + var config = Path.ChangeExtension (assembly.ItemSpec, "dll.config"); + if (UseAssemblyStore) { + storeAssembly.SetConfigPath (config); + } else { + AddAssemblyConfigEntry (apk, assemblyPath, config); + } - if (File.Exists (symbols)) - AddFileToArchiveIfNewer (apk, symbols, assemblyPath + Path.GetFileName (symbols), compressionMethod: UncompressedMethod); - } - count++; - if (count >= ZipArchiveEx.ZipFlushFilesLimit) { - apk.Flush(); - count = 0; + // Try to add symbols if Debug + if (debug) { + var symbols = Path.ChangeExtension (assembly.ItemSpec, "dll.mdb"); + string symbolsPath = null; + + if (File.Exists (symbols)) { + symbolsPath = symbols; + } else { + symbols = Path.ChangeExtension (assembly.ItemSpec, "pdb"); + + if (File.Exists (symbols)) { + symbolsPath = symbols; + } + } + + if (!String.IsNullOrEmpty (symbolsPath)) { + if (UseAssemblyStore) { + storeAssembly.SetDebugInfoPath (symbolsPath); + } else { + AddFileToArchiveIfNewer (apk, symbolsPath, assemblyPath + Path.GetFileName (symbols), compressionMethod: UncompressedMethod); + } + } + } + + if (UseAssemblyStore) { + storeGenerator.Add (assemblyStoreApkName, storeAssembly); + } else { + count++; + if (count >= ZipArchiveEx.ZipFlushFilesLimit) { + apk.Flush(); + count = 0; + } + } } } diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/GeneratePackageManagerJava.cs b/src/Xamarin.Android.Build.Tasks/Tasks/GeneratePackageManagerJava.cs index a90c105c1c0..4125581f24f 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/GeneratePackageManagerJava.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/GeneratePackageManagerJava.cs @@ -30,6 +30,8 @@ public class GeneratePackageManagerJava : AndroidTask public ITaskItem[] SatelliteAssemblies { get; set; } + public bool UseAssemblyStore { get; set; } + [Required] public string OutputDirectory { get; set; } @@ -270,10 +272,13 @@ void AddEnvironment () } int assemblyNameWidth = 0; - int assemblyCount = ResolvedAssemblies.Length; Encoding assemblyNameEncoding = Encoding.UTF8; Action updateNameWidth = (ITaskItem assembly) => { + if (UseAssemblyStore) { + return; + } + string assemblyName = Path.GetFileName (assembly.ItemSpec); int nameBytes = assemblyNameEncoding.GetBytes (assemblyName).Length; if (nameBytes > assemblyNameWidth) { @@ -281,29 +286,55 @@ void AddEnvironment () } }; - if (SatelliteAssemblies != null) { - assemblyCount += SatelliteAssemblies.Length; + int assemblyCount = 0; + HashSet archAssemblyNames = null; + + Action updateAssemblyCount = (ITaskItem assembly) => { + if (!UseAssemblyStore) { + assemblyCount++; + return; + } + + string abi = assembly.GetMetadata ("Abi"); + if (String.IsNullOrEmpty (abi)) { + assemblyCount++; + } else { + archAssemblyNames ??= new HashSet (StringComparer.OrdinalIgnoreCase); + string assemblyName = Path.GetFileName (assembly.ItemSpec); + if (!archAssemblyNames.Contains (assemblyName)) { + assemblyCount++; + archAssemblyNames.Add (assemblyName); + } + } + }; + + if (SatelliteAssemblies != null) { foreach (ITaskItem assembly in SatelliteAssemblies) { updateNameWidth (assembly); + updateAssemblyCount (assembly); } } foreach (var assembly in ResolvedAssemblies) { updateNameWidth (assembly); + updateAssemblyCount (assembly); } - int abiNameLength = 0; - foreach (string abi in SupportedAbis) { - if (abi.Length <= abiNameLength) { - continue; + if (!UseAssemblyStore) { + int abiNameLength = 0; + foreach (string abi in SupportedAbis) { + if (abi.Length <= abiNameLength) { + continue; + } + abiNameLength = abi.Length; } - abiNameLength = abi.Length; + assemblyNameWidth += abiNameLength + 2; // room for '/' and the terminating NUL } - assemblyNameWidth += abiNameLength + 1; // room for '/' bool haveRuntimeConfigBlob = !String.IsNullOrEmpty (RuntimeConfigBinFilePath) && File.Exists (RuntimeConfigBinFilePath); var appConfState = BuildEngine4.GetRegisteredTaskObjectAssemblyLocal (ApplicationConfigTaskState.RegisterTaskObjectKey, RegisteredTaskObjectLifetime.Build); + foreach (string abi in SupportedAbis) { NativeAssemblerTargetProvider asmTargetProvider = GetAssemblyTargetProvider (abi); string baseAsmFilePath = Path.Combine (EnvironmentOutputDirectory, $"environment.{abi.ToLowerInvariant ()}"); @@ -323,7 +354,12 @@ void AddEnvironment () JniAddNativeMethodRegistrationAttributePresent = appConfState != null ? appConfState.JniAddNativeMethodRegistrationAttributePresent : false, HaveRuntimeConfigBlob = haveRuntimeConfigBlob, NumberOfAssembliesInApk = assemblyCount, - BundledAssemblyNameWidth = assemblyNameWidth + 1, + BundledAssemblyNameWidth = assemblyNameWidth, + NumberOfAssemblyStoresInApks = 2, // Until feature APKs are a thing, we're going to have just two stores in each app - one for arch-agnostic + // and up to 4 other for arch-specific assemblies. Only **one** arch-specific store is ever loaded on the app + // runtime, thus the number 2 here. All architecture specific stores contain assemblies with the same names + // and in the same order. + HaveAssemblyStore = UseAssemblyStore, }; using (var sw = MemoryStreamPool.Shared.CreateStreamWriter ()) { diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/ProcessAssemblies.cs b/src/Xamarin.Android.Build.Tasks/Tasks/ProcessAssemblies.cs index 379fe318c2c..29fb41f5d22 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/ProcessAssemblies.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/ProcessAssemblies.cs @@ -78,10 +78,35 @@ public override bool RunTask () return !Log.HasLoggedErrors; } + void SetAssemblyAbiMetadata (string abi, string assetType, ITaskItem assembly, ITaskItem? symbol, bool isDuplicate) + { + if (String.IsNullOrEmpty (abi) || (!isDuplicate && String.Compare ("native", assetType, StringComparison.OrdinalIgnoreCase) != 0)) { + return; + } + + assembly.SetMetadata ("Abi", abi); + if (symbol != null) { + symbol.SetMetadata ("Abi", abi); + } + } + + void SetAssemblyAbiMetadata (ITaskItem assembly, ITaskItem? symbol, bool isDuplicate) + { + string assetType = assembly.GetMetadata ("AssetType"); + string rid = assembly.GetMetadata ("RuntimeIdentifier"); + if (!String.IsNullOrEmpty (assembly.GetMetadata ("Culture")) || String.Compare ("resources", assetType, StringComparison.OrdinalIgnoreCase) == 0) { + // Satellite assemblies are abi-agnostic, they shouldn't have the Abi metadata set + return; + } + + SetAssemblyAbiMetadata (AndroidRidAbiHelper.RuntimeIdentifierToAbi (rid), assetType, assembly, symbol, isDuplicate); + } + void SetMetadataForAssemblies (List output, Dictionary symbols) { foreach (var assembly in InputAssemblies) { var symbol = GetOrCreateSymbolItem (symbols, assembly); + SetAssemblyAbiMetadata (assembly, symbol, isDuplicate: false); symbol?.SetDestinationSubPath (); assembly.SetDestinationSubPath (); assembly.SetMetadata ("FrameworkAssembly", IsFrameworkAssembly (assembly).ToString ()); @@ -119,7 +144,7 @@ void DeduplicateAssemblies (List output, Dictionary 1) { foreach (var assembly in group) { var symbol = GetOrCreateSymbolItem (symbols, assembly); - SetDestinationSubDirectory (assembly, group.Key, symbol); + SetDestinationSubDirectory (assembly, group.Key, symbol, isDuplicate: true); output.Add (assembly); } } else { @@ -133,6 +158,7 @@ void DeduplicateAssemblies (List output, Dictionary /// Sets %(DestinationSubDirectory) and %(DestinationSubPath) based on %(RuntimeIdentifier) /// - void SetDestinationSubDirectory (ITaskItem assembly, string fileName, ITaskItem? symbol) + void SetDestinationSubDirectory (ITaskItem assembly, string fileName, ITaskItem? symbol, bool isDuplicate) { var rid = assembly.GetMetadata ("RuntimeIdentifier"); + string assetType = assembly.GetMetadata ("AssetType"); + + // Satellite assemblies have `RuntimeIdentifier` set, but they shouldn't - they aren't specific to any architecture, so they should have none of the + // abi-specific metadata set + // + if (!String.IsNullOrEmpty (assembly.GetMetadata ("Culture")) || + String.Compare ("resources", assetType, StringComparison.OrdinalIgnoreCase) == 0) { + rid = String.Empty; + } + var abi = AndroidRidAbiHelper.RuntimeIdentifierToAbi (rid); if (!string.IsNullOrEmpty (abi)) { string destination = Path.Combine (assembly.GetMetadata ("DestinationSubDirectory"), abi); @@ -185,6 +221,8 @@ void SetDestinationSubDirectory (ITaskItem assembly, string fileName, ITaskItem? symbol.SetMetadata ("DestinationSubDirectory", destination + Path.DirectorySeparatorChar); symbol.SetMetadata ("DestinationSubPath", Path.Combine (destination, Path.GetFileName (symbol.ItemSpec))); } + + SetAssemblyAbiMetadata (abi, assetType, assembly, symbol, isDuplicate); } else { Log.LogDebugMessage ($"Android ABI not found for: {assembly.ItemSpec}"); assembly.SetDestinationSubPath (); diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/AotTests.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/AotTests.cs index bdc9d6faa81..f9a88389426 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/AotTests.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/AotTests.cs @@ -115,50 +115,64 @@ public void BuildBasicApplicationReleaseProfiledAotWithoutDefaultProfile () /* supportedAbis */ "armeabi-v7a", /* enableLLVM */ false, /* expectedResult */ true, + /* usesAssemblyBlobs */ false, + }, + new object[] { + /* supportedAbis */ "armeabi-v7a", + /* enableLLVM */ false, + /* expectedResult */ true, + /* usesAssemblyBlobs */ true, }, new object[] { /* supportedAbis */ "armeabi-v7a", /* enableLLVM */ true, /* expectedResult */ true, + /* usesAssemblyBlobs */ false, }, new object[] { /* supportedAbis */ "arm64-v8a", /* enableLLVM */ false, /* expectedResult */ true, + /* usesAssemblyBlobs */ false, }, new object[] { /* supportedAbis */ "arm64-v8a", /* enableLLVM */ true, /* expectedResult */ true, + /* usesAssemblyBlobs */ false, }, new object[] { /* supportedAbis */ "x86", /* enableLLVM */ false, /* expectedResult */ true, + /* usesAssemblyBlobs */ false, }, new object[] { /* supportedAbis */ "x86", /* enableLLVM */ true, /* expectedResult */ true, + /* usesAssemblyBlobs */ false, }, new object[] { /* supportedAbis */ "x86_64", /* enableLLVM */ false, /* expectedResult */ true, + /* usesAssemblyBlobs */ false, }, new object[] { /* supportedAbis */ "x86_64", /* enableLLVM */ true, /* expectedResult */ true, + /* usesAssemblyBlobs */ false, }, }; [Test] [TestCaseSource (nameof (AotChecks))] [Category ("DotNetIgnore")] // Not currently working, see: https://github.com/dotnet/runtime/issues/56163 - public void BuildAotApplicationAndÜmläüts (string supportedAbis, bool enableLLVM, bool expectedResult) + public void BuildAotApplicationAndÜmläüts (string supportedAbis, bool enableLLVM, bool expectedResult, bool usesAssemblyBlobs) { - var path = Path.Combine ("temp", string.Format ("BuildAotApplication AndÜmläüts_{0}_{1}_{2}", supportedAbis, enableLLVM, expectedResult)); + var path = Path.Combine ("temp", string.Format ("BuildAotApplication AndÜmläüts_{0}_{1}_{2}_{3}", supportedAbis, enableLLVM, expectedResult, usesAssemblyBlobs)); var proj = new XamarinAndroidApplicationProject () { IsRelease = true, BundleAssemblies = false, @@ -168,6 +182,7 @@ public void BuildAotApplicationAndÜmläüts (string supportedAbis, bool enableL proj.SetProperty (KnownProperties.TargetFrameworkVersion, "v5.1"); proj.SetAndroidSupportedAbis (supportedAbis); proj.SetProperty ("EnableLLVM", enableLLVM.ToString ()); + proj.SetProperty ("AndroidUseAssemblyStore", usesAssemblyBlobs.ToString ()); bool checkMinLlvmPath = enableLLVM && (supportedAbis == "armeabi-v7a" || supportedAbis == "x86"); if (checkMinLlvmPath) { // Set //uses-sdk/@android:minSdkVersion so that LLVM uses the right libc.so @@ -211,13 +226,13 @@ public void BuildAotApplicationAndÜmläüts (string supportedAbis, bool enableL Assert.IsTrue (File.Exists (assemblies), "{0} libaot-UnnamedProject.dll.so does not exist", abi); var apk = Path.Combine (Root, b.ProjectDirectory, proj.OutputPath, $"{proj.PackageName}-Signed.apk"); + + var helper = new ArchiveAssemblyHelper (apk, usesAssemblyBlobs); + Assert.IsTrue (helper.Exists ("assemblies/UnnamedProject.dll"), $"UnnamedProject.dll should be in the {proj.PackageName}-Signed.apk"); using (var zipFile = ZipHelper.OpenZip (apk)) { Assert.IsNotNull (ZipHelper.ReadFileFromZip (zipFile, string.Format ("lib/{0}/libaot-UnnamedProject.dll.so", abi)), $"lib/{0}/libaot-UnnamedProject.dll.so should be in the {proj.PackageName}-Signed.apk", abi); - Assert.IsNotNull (ZipHelper.ReadFileFromZip (zipFile, - "assemblies/UnnamedProject.dll"), - $"UnnamedProject.dll should be in the {proj.PackageName}-Signed.apk"); } } Assert.AreEqual (expectedResult, b.Build (proj), "Second Build should have {0}.", expectedResult ? "succeeded" : "failed"); @@ -234,9 +249,9 @@ public void BuildAotApplicationAndÜmläüts (string supportedAbis, bool enableL [TestCaseSource (nameof (AotChecks))] [Category ("Minor"), Category ("MkBundle")] [Category ("DotNetIgnore")] // Not currently working, see: https://github.com/dotnet/runtime/issues/56163 - public void BuildAotApplicationAndBundleAndÜmläüts (string supportedAbis, bool enableLLVM, bool expectedResult) + public void BuildAotApplicationAndBundleAndÜmläüts (string supportedAbis, bool enableLLVM, bool expectedResult, bool usesAssemblyBlobs) { - var path = Path.Combine ("temp", string.Format ("BuildAotApplicationAndBundle AndÜmläüts_{0}_{1}_{2}", supportedAbis, enableLLVM, expectedResult)); + var path = Path.Combine ("temp", string.Format ("BuildAotApplicationAndBundle AndÜmläüts_{0}_{1}_{2}_{3}", supportedAbis, enableLLVM, expectedResult, usesAssemblyBlobs)); var proj = new XamarinAndroidApplicationProject () { IsRelease = true, BundleAssemblies = true, @@ -246,6 +261,7 @@ public void BuildAotApplicationAndBundleAndÜmläüts (string supportedAbis, boo proj.SetProperty (KnownProperties.TargetFrameworkVersion, "v5.1"); proj.SetAndroidSupportedAbis (supportedAbis); proj.SetProperty ("EnableLLVM", enableLLVM.ToString ()); + proj.SetProperty ("AndroidUseAssemblyStore", usesAssemblyBlobs.ToString ()); using (var b = CreateApkBuilder (path)) { if (!b.CrossCompilerAvailable (supportedAbis)) Assert.Ignore ("Cross compiler was not available"); @@ -264,13 +280,12 @@ public void BuildAotApplicationAndBundleAndÜmläüts (string supportedAbis, boo Assert.IsTrue (File.Exists (assemblies), "{0} libaot-UnnamedProject.dll.so does not exist", abi); var apk = Path.Combine (Root, b.ProjectDirectory, proj.OutputPath, $"{proj.PackageName}-Signed.apk"); + var helper = new ArchiveAssemblyHelper (apk, usesAssemblyBlobs); + Assert.IsFalse (helper.Exists ("assemblies/UnnamedProject.dll"), $"UnnamedProject.dll should not be in the {proj.PackageName}-Signed.apk"); using (var zipFile = ZipHelper.OpenZip (apk)) { Assert.IsNotNull (ZipHelper.ReadFileFromZip (zipFile, string.Format ("lib/{0}/libaot-UnnamedProject.dll.so", abi)), $"lib/{0}/libaot-UnnamedProject.dll.so should be in the {proj.PackageName}-Signed.apk", abi); - Assert.IsNull (ZipHelper.ReadFileFromZip (zipFile, - "assemblies/UnnamedProject.dll"), - $"UnnamedProject.dll should not be in the {proj.PackageName}-Signed.apk"); } } Assert.AreEqual (expectedResult, b.Build (proj), "Second Build should have {0}.", expectedResult ? "succeeded" : "failed"); @@ -381,6 +396,8 @@ public static void Foo () { [Category ("HybridAOT")] public void HybridAOT ([Values ("armeabi-v7a;arm64-v8a", "armeabi-v7a", "arm64-v8a")] string abis) { + // There's no point in testing all of the ABIs with and without assembly blobs, let's test just one of them this way + bool usesAssemblyBlobs = String.Compare ("arm64-v8a", abis, StringComparison.Ordinal) == 0; var proj = new XamarinAndroidApplicationProject () { IsRelease = true, AotAssemblies = true, @@ -388,6 +405,7 @@ public static void Foo () { proj.SetProperty ("AndroidAotMode", "Hybrid"); // So we can use Mono.Cecil to open assemblies directly proj.SetProperty ("AndroidEnableAssemblyCompression", "False"); + proj.SetProperty ("AndroidUseAssemblyStore", usesAssemblyBlobs.ToString ()); proj.SetAndroidSupportedAbis (abis); using (var b = CreateApkBuilder ()) { @@ -412,17 +430,15 @@ public static void Foo () { var apk = Path.Combine (Root, b.ProjectDirectory, proj.OutputPath, $"{proj.PackageName}-Signed.apk"); FileAssert.Exists (apk); - using (var zip = ZipHelper.OpenZip (apk)) { - var entry = zip.ReadEntry ($"assemblies/{proj.ProjectName}.dll"); - Assert.IsNotNull (entry, $"{proj.ProjectName}.dll should exist in apk!"); - using (var stream = new MemoryStream ()) { - entry.Extract (stream); - stream.Position = 0; - using (var assembly = AssemblyDefinition.ReadAssembly (stream)) { - var type = assembly.MainModule.GetType ($"{proj.ProjectName}.MainActivity"); - var method = type.Methods.First (m => m.Name == "OnCreate"); - Assert.LessOrEqual (method.Body.Instructions.Count, 1, "OnCreate should have stripped method bodies!"); - } + var helper = new ArchiveAssemblyHelper (apk, usesAssemblyBlobs); + Assert.IsTrue (helper.Exists ($"assemblies/{proj.ProjectName}.dll"), $"{proj.ProjectName}.dll should exist in apk!"); + + using (var stream = helper.ReadEntry ($"assemblies/{proj.ProjectName}.dll")) { + stream.Position = 0; + using (var assembly = AssemblyDefinition.ReadAssembly (stream)) { + var type = assembly.MainModule.GetType ($"{proj.ProjectName}.MainActivity"); + var method = type.Methods.First (m => m.Name == "OnCreate"); + Assert.LessOrEqual (method.Body.Instructions.Count, 1, "OnCreate should have stripped method bodies!"); } } } diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/BuildTest.TestCaseSource.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/BuildTest.TestCaseSource.cs index ec519683685..e12617fa18a 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/BuildTest.TestCaseSource.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/BuildTest.TestCaseSource.cs @@ -96,6 +96,17 @@ public partial class BuildTest : BaseTest /* debugType */ "Full", /* embedMdb */ !CommercialBuildAvailable, // because we don't use FastDev in the OSS repo /* expectedRuntime */ "debug", + /* usesAssemblyBlobs */ false, + }, + new object[] { + /* isRelease */ false, + /* monoSymbolArchive */ false , + /* aotAssemblies */ false, + /* debugSymbols */ true, + /* debugType */ "Full", + /* embedMdb */ !CommercialBuildAvailable, // because we don't use FastDev in the OSS repo + /* expectedRuntime */ "debug", + /* usesAssemblyBlobs */ true, }, new object[] { /* isRelease */ true, @@ -105,6 +116,17 @@ public partial class BuildTest : BaseTest /* debugType */ "Full", /* embedMdb */ false, /* expectedRuntime */ "release", + /* usesAssemblyBlobs */ false, + }, + new object[] { + /* isRelease */ true, + /* monoSymbolArchive */ false, + /* aotAssemblies */ false, + /* debugSymbols */ true, + /* debugType */ "Full", + /* embedMdb */ false, + /* expectedRuntime */ "release", + /* usesAssemblyBlobs */ true, }, new object[] { /* isRelease */ true, @@ -114,6 +136,7 @@ public partial class BuildTest : BaseTest /* debugType */ "Full", /* embedMdb */ false, /* expectedRuntime */ "release", + /* usesAssemblyBlobs */ false, }, new object[] { /* isRelease */ true, @@ -123,6 +146,7 @@ public partial class BuildTest : BaseTest /* debugType */ "Portable", /* embedMdb */ false, /* expectedRuntime */ "release", + /* usesAssemblyBlobs */ false, }, new object[] { /* isRelease */ true, @@ -132,6 +156,7 @@ public partial class BuildTest : BaseTest /* debugType */ "Portable", /* embedMdb */ false, /* expectedRuntime */ "release", + /* usesAssemblyBlobs */ false, }, new object[] { /* isRelease */ true, @@ -141,15 +166,37 @@ public partial class BuildTest : BaseTest /* debugType */ "Portable", /* embedMdb */ false, /* expectedRuntime */ "release", + /* usesAssemblyBlobs */ false, }, new object[] { /* isRelease */ true, /* monoSymbolArchive */ false , + /* aotAssemblies */ false, + /* debugSymbols */ true, + /* debugType */ "Portable", + /* embedMdb */ false, + /* expectedRuntime */ "release", + /* usesAssemblyBlobs */ true, + }, + new object[] { + /* isRelease */ true, + /* monoSymbolArchive */ false , + /* aotAssemblies */ true, + /* debugSymbols */ false, + /* debugType */ "", + /* embedMdb */ false, + /* expectedRuntime */ "release", + /* usesAssemblyBlobs */ false, + }, + new object[] { + /* isRelease */ true, + /* monoSymbolArchive */ true , /* aotAssemblies */ true, /* debugSymbols */ false, /* debugType */ "", /* embedMdb */ false, /* expectedRuntime */ "release", + /* usesAssemblyBlobs */ false, }, new object[] { /* isRelease */ true, @@ -159,9 +206,9 @@ public partial class BuildTest : BaseTest /* debugType */ "", /* embedMdb */ false, /* expectedRuntime */ "release", + /* usesAssemblyBlobs */ true, }, }; #pragma warning restore 414 } } - diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/BuildTest.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/BuildTest.cs index 5e2c6c0d600..6beed1e2407 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/BuildTest.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/BuildTest.cs @@ -98,7 +98,7 @@ public void CheckWhichRuntimeIsIncluded (string supportedAbi, bool debugSymbols, [Category ("AOT"), Category ("MonoSymbolicate")] [TestCaseSource (nameof (SequencePointChecks))] public void CheckSequencePointGeneration (bool isRelease, bool monoSymbolArchive, bool aotAssemblies, - bool debugSymbols, string debugType, bool embedMdb, string expectedRuntime) + bool debugSymbols, string debugType, bool embedMdb, string expectedRuntime, bool usesAssemblyBlobs) { var proj = new XamarinAndroidApplicationProject () { IsRelease = isRelease, @@ -109,6 +109,7 @@ public void CheckSequencePointGeneration (bool isRelease, bool monoSymbolArchive proj.SetProperty (proj.ActiveConfigurationProperties, "MonoSymbolArchive", monoSymbolArchive); proj.SetProperty (proj.ActiveConfigurationProperties, "DebugSymbols", debugSymbols); proj.SetProperty (proj.ActiveConfigurationProperties, "DebugType", debugType); + proj.SetProperty (proj.ActiveConfigurationProperties, "AndroidUseAssemblyStore", usesAssemblyBlobs.ToString ()); using (var b = CreateApkBuilder ()) { if (aotAssemblies && !b.CrossCompilerAvailable (string.Join (";", abis))) Assert.Ignore ("Cross compiler was not available"); @@ -116,45 +117,42 @@ public void CheckSequencePointGeneration (bool isRelease, bool monoSymbolArchive var apk = Path.Combine (Root, b.ProjectDirectory, proj.OutputPath, $"{proj.PackageName}-Signed.apk"); var msymarchive = Path.Combine (Root, b.ProjectDirectory, proj.OutputPath, proj.PackageName + ".apk.mSYM"); - using (var zipFile = ZipHelper.OpenZip (apk)) { - var mdbExits = ZipHelper.ReadFileFromZip (zipFile, "assemblies/UnnamedProject.dll.mdb") != null || - ZipHelper.ReadFileFromZip (zipFile, "assemblies/UnnamedProject.pdb") != null; - Assert.AreEqual (embedMdb, mdbExits, - $"assemblies/UnnamedProject.dll.mdb or assemblies/UnnamedProject.pdb should{0}be in the {proj.PackageName}-Signed.apk", embedMdb ? " " : " not "); - if (aotAssemblies) { - foreach (var abi in abis) { - var assemblies = Path.Combine (Root, b.ProjectDirectory, proj.IntermediateOutputPath, - "aot", abi, "libaot-UnnamedProject.dll.so"); - var shouldExist = monoSymbolArchive && debugSymbols && (debugType == "PdbOnly" || debugType == "Portable"); - var symbolicateFile = Directory.GetFiles (Path.Combine (Root, b.ProjectDirectory, proj.IntermediateOutputPath, - "aot", abi), "UnnamedProject.dll.msym", SearchOption.AllDirectories).FirstOrDefault (); - if (shouldExist) - Assert.IsNotNull (symbolicateFile, "UnnamedProject.dll.msym should exist"); - else - Assert.IsNull (symbolicateFile, "{0} should not exist", symbolicateFile); - if (shouldExist) { - var foundMsyms = Directory.GetFiles (Path.Combine (msymarchive), "UnnamedProject.dll.msym", SearchOption.AllDirectories).Any (); - Assert.IsTrue (foundMsyms, "UnnamedProject.dll.msym should exist in the archive {0}", msymarchive); - } - Assert.IsTrue (File.Exists (assemblies), "{0} libaot-UnnamedProject.dll.so does not exist", abi); - Assert.IsNotNull (ZipHelper.ReadFileFromZip (zipFile, - string.Format ("lib/{0}/libaot-UnnamedProject.dll.so", abi)), - $"lib/{0}/libaot-UnnamedProject.dll.so should be in the {proj.PackageName}-Signed.apk", abi); - Assert.IsNotNull (ZipHelper.ReadFileFromZip (zipFile, - "assemblies/UnnamedProject.dll"), - $"UnnamedProject.dll should be in the {proj.PackageName}-Signed.apk"); - } - } - var runtimeInfo = b.GetSupportedRuntimes (); + var helper = new ArchiveAssemblyHelper (apk, usesAssemblyBlobs); + var mdbExits = helper.Exists ("assemblies/UnnamedProject.dll.mdb") || helper.Exists ("assemblies/UnnamedProject.pdb"); + Assert.AreEqual (embedMdb, mdbExits, + $"assemblies/UnnamedProject.dll.mdb or assemblies/UnnamedProject.pdb should{0}be in the {proj.PackageName}-Signed.apk", embedMdb ? " " : " not "); + if (aotAssemblies) { foreach (var abi in abis) { - var runtime = runtimeInfo.FirstOrDefault (x => x.Abi == abi && x.Runtime == expectedRuntime); - Assert.IsNotNull (runtime, "Could not find the expected runtime."); - var inApk = ZipHelper.ReadFileFromZip (apk, String.Format ("lib/{0}/{1}", abi, runtime.Name)); - var inApkRuntime = runtimeInfo.FirstOrDefault (x => x.Abi == abi && x.Size == inApk.Length); - Assert.IsNotNull (inApkRuntime, "Could not find the actual runtime used."); - Assert.AreEqual (runtime.Size, inApkRuntime.Size, "expected {0} got {1}", expectedRuntime, inApkRuntime.Runtime); + var assemblies = Path.Combine (Root, b.ProjectDirectory, proj.IntermediateOutputPath, + "aot", abi, "libaot-UnnamedProject.dll.so"); + var shouldExist = monoSymbolArchive && debugSymbols && (debugType == "PdbOnly" || debugType == "Portable"); + var symbolicateFile = Directory.GetFiles (Path.Combine (Root, b.ProjectDirectory, proj.IntermediateOutputPath, + "aot", abi), "UnnamedProject.dll.msym", SearchOption.AllDirectories).FirstOrDefault (); + if (shouldExist) + Assert.IsNotNull (symbolicateFile, "UnnamedProject.dll.msym should exist"); + else + Assert.IsNull (symbolicateFile, "{0} should not exist", symbolicateFile); + if (shouldExist) { + var foundMsyms = Directory.GetFiles (Path.Combine (msymarchive), "UnnamedProject.dll.msym", SearchOption.AllDirectories).Any (); + Assert.IsTrue (foundMsyms, "UnnamedProject.dll.msym should exist in the archive {0}", msymarchive); + } + Assert.IsTrue (File.Exists (assemblies), "{0} libaot-UnnamedProject.dll.so does not exist", abi); + Assert.IsTrue (helper.Exists ($"lib/{abi}/libaot-UnnamedProject.dll.so"), + $"lib/{0}/libaot-UnnamedProject.dll.so should be in the {proj.PackageName}-Signed.apk", abi); + Assert.IsTrue (helper.Exists ("assemblies/UnnamedProject.dll"), + $"UnnamedProject.dll should be in the {proj.PackageName}-Signed.apk"); } } + var runtimeInfo = b.GetSupportedRuntimes (); + foreach (var abi in abis) { + var runtime = runtimeInfo.FirstOrDefault (x => x.Abi == abi && x.Runtime == expectedRuntime); + Assert.IsNotNull (runtime, "Could not find the expected runtime."); + var inApk = ZipHelper.ReadFileFromZip (apk, String.Format ("lib/{0}/{1}", abi, runtime.Name)); + var inApkRuntime = runtimeInfo.FirstOrDefault (x => x.Abi == abi && x.Size == inApk.Length); + Assert.IsNotNull (inApkRuntime, "Could not find the actual runtime used."); + Assert.AreEqual (runtime.Size, inApkRuntime.Size, "expected {0} got {1}", expectedRuntime, inApkRuntime.Runtime); + } + b.Clean (proj); Assert.IsTrue (!Directory.Exists (msymarchive), "{0} should have been deleted on Clean", msymarchive); } @@ -942,6 +940,7 @@ public void BuildBasicApplicationCheckPdb () var proj = new XamarinAndroidApplicationProject { EmbedAssembliesIntoApk = true, }; + proj.SetProperty ("AndroidUseAssemblyStore", "False"); using (var b = CreateApkBuilder ()) { var reference = new BuildItem.Reference ("PdbTestLibrary.dll") { WebContentFileNameFromAzure = "PdbTestLibrary.dll" diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/BuildTest2.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/BuildTest2.cs index f098c481f7e..05af0db36b5 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/BuildTest2.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/BuildTest2.cs @@ -77,6 +77,7 @@ public void BuildReleaseArm64 ([Values (false, true)] bool forms) proj.IsRelease = true; proj.SetAndroidSupportedAbis ("arm64-v8a"); proj.SetProperty ("LinkerDumpDependencies", "True"); + proj.SetProperty ("AndroidUseAssemblyStore", "False"); if (forms) { proj.PackageReferences.Clear (); diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/PackagingTest.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/PackagingTest.cs index 1844149193c..7e096ecfb39 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/PackagingTest.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/PackagingTest.cs @@ -67,11 +67,12 @@ public void CheckProguardMappingFileExists () } [Test] - public void CheckIncludedAssemblies () + public void CheckIncludedAssemblies ([Values (false, true)] bool usesAssemblyStores) { var proj = new XamarinAndroidApplicationProject { IsRelease = true }; + proj.SetProperty ("AndroidUseAssemblyStore", usesAssemblyStores.ToString ()); proj.SetAndroidSupportedAbis ("armeabi-v7a"); if (!Builder.UseDotNet) { proj.PackageReferences.Add (new Package { @@ -107,17 +108,20 @@ public void CheckIncludedAssemblies () Assert.IsTrue (b.Build (proj), "build should have succeeded."); var apk = Path.Combine (Root, b.ProjectDirectory, proj.OutputPath, $"{proj.PackageName}-Signed.apk"); - using (var zip = ZipHelper.OpenZip (apk)) { - var existingFiles = zip.Where (a => a.FullName.StartsWith ("assemblies/", StringComparison.InvariantCultureIgnoreCase)); - var missingFiles = expectedFiles.Where (x => !zip.ContainsEntry ("assemblies/" + Path.GetFileName (x))); - Assert.IsFalse (missingFiles.Any (), - string.Format ("The following Expected files are missing. {0}", - string.Join (Environment.NewLine, missingFiles))); - var additionalFiles = existingFiles.Where (x => !expectedFiles.Contains (Path.GetFileName (x.FullName))); - Assert.IsTrue (!additionalFiles.Any (), - string.Format ("Unexpected Files found! {0}", - string.Join (Environment.NewLine, additionalFiles.Select (x => x.FullName)))); - } + var helper = new ArchiveAssemblyHelper (apk, usesAssemblyStores); + List existingFiles; + List missingFiles; + List additionalFiles; + + helper.Contains (expectedFiles, out existingFiles, out missingFiles, out additionalFiles); + + Assert.IsTrue (missingFiles == null || missingFiles.Count == 0, + string.Format ("The following Expected files are missing. {0}", + string.Join (Environment.NewLine, missingFiles))); + + Assert.IsTrue (additionalFiles == null || additionalFiles.Count == 0, + string.Format ("Unexpected Files found! {0}", + string.Join (Environment.NewLine, additionalFiles))); } } @@ -634,6 +638,7 @@ protected override void OnResume() }, } }; + app.SetProperty ("AndroidUseAssemblyStore", "False"); app.MainActivity = @"using System; using Android.App; using Android.Content; @@ -846,9 +851,8 @@ public void MissingSatelliteAssemblyInLibrary () var apk = Path.Combine (Root, appBuilder.ProjectDirectory, app.OutputPath, $"{app.PackageName}-Signed.apk"); - using (var zip = ZipHelper.OpenZip (apk)) { - Assert.IsTrue (zip.ContainsEntry ($"assemblies/es/{lib.ProjectName}.resources.dll"), "Apk should contain satellite assemblies!"); - } + var helper = new ArchiveAssemblyHelper (apk); + Assert.IsTrue (helper.Exists ($"assemblies/es/{lib.ProjectName}.resources.dll"), "Apk should contain satellite assemblies!"); } } @@ -875,9 +879,8 @@ public void MissingSatelliteAssemblyInApp () var apk = Path.Combine (Root, b.ProjectDirectory, proj.OutputPath, $"{proj.PackageName}-Signed.apk"); - using (var zip = ZipHelper.OpenZip (apk)) { - Assert.IsTrue (zip.ContainsEntry ($"assemblies/es/{proj.ProjectName}.resources.dll"), "Apk should contain satellite assemblies!"); - } + var helper = new ArchiveAssemblyHelper (apk); + Assert.IsTrue (helper.Exists ($"assemblies/es/{proj.ProjectName}.resources.dll"), "Apk should contain satellite assemblies!"); } } diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Tasks/LinkerTests.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Tasks/LinkerTests.cs index 217c5d2ebaf..c16466e6453 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Tasks/LinkerTests.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Tasks/LinkerTests.cs @@ -211,35 +211,33 @@ public void WarnAboutAppDomains ([Values (true, false)] bool isRelease) } [Test] - public void RemoveDesigner () + public void RemoveDesigner ([Values (true, false)] bool useAssemblyStore) { var proj = new XamarinAndroidApplicationProject { IsRelease = true, }; proj.SetProperty ("AndroidEnableAssemblyCompression", "False"); proj.SetProperty ("AndroidLinkResources", "True"); + proj.SetProperty ("AndroidUseAssemblyStore", useAssemblyStore.ToString ()); string assemblyName = proj.ProjectName; using (var b = CreateApkBuilder ()) { Assert.IsTrue (b.Build (proj), "build should have succeeded."); var apk = Path.Combine (Root, b.ProjectDirectory, proj.OutputPath, $"{proj.PackageName}-Signed.apk"); FileAssert.Exists (apk); - using (var zip = ZipHelper.OpenZip (apk)) { - Assert.IsTrue (zip.ContainsEntry ($"assemblies/{assemblyName}.dll"), $"{assemblyName}.dll should exist in apk!"); - var entry = zip.ReadEntry ($"assemblies/{assemblyName}.dll"); - using (var stream = new MemoryStream ()) { - entry.Extract (stream); - stream.Position = 0; - using (var assembly = AssemblyDefinition.ReadAssembly (stream)) { - var type = assembly.MainModule.GetType ($"{assemblyName}.Resource"); - Assert.AreEqual (0, type.NestedTypes.Count, "All Nested Resource Types should be removed."); - } + var helper = new ArchiveAssemblyHelper (apk, useAssemblyStore); + Assert.IsTrue (helper.Exists ($"assemblies/{assemblyName}.dll"), $"{assemblyName}.dll should exist in apk!"); + using (var stream = helper.ReadEntry ($"assemblies/{assemblyName}.dll")) { + stream.Position = 0; + using (var assembly = AssemblyDefinition.ReadAssembly (stream)) { + var type = assembly.MainModule.GetType ($"{assemblyName}.Resource"); + Assert.AreEqual (0, type.NestedTypes.Count, "All Nested Resource Types should be removed."); } } } } [Test] - public void LinkDescription () + public void LinkDescription ([Values (true, false)] bool useAssemblyStore) { string assembly_name = Builder.UseDotNet ? "System.Console" : "mscorlib"; string linker_xml = ""; @@ -254,6 +252,7 @@ public void LinkDescription () }; // So we can use Mono.Cecil to open assemblies directly proj.SetProperty ("AndroidEnableAssemblyCompression", "False"); + proj.SetProperty ("AndroidUseAssemblyStore", useAssemblyStore.ToString ()); using (var b = CreateApkBuilder ()) { Assert.IsTrue (b.Build (proj), "first build should have succeeded."); @@ -272,17 +271,14 @@ public void LinkDescription () var apk = Path.Combine (Root, b.ProjectDirectory, proj.OutputPath, $"{proj.PackageName}-Signed.apk"); FileAssert.Exists (apk); - using (var zip = ZipHelper.OpenZip (apk)) { - var entry = zip.ReadEntry ($"assemblies/{assembly_name}.dll"); - Assert.IsNotNull (entry, $"{assembly_name}.dll should exist in apk!"); - using (var stream = new MemoryStream ()) { - entry.Extract (stream); - stream.Position = 0; - using (var assembly = AssemblyDefinition.ReadAssembly (stream)) { - var type = assembly.MainModule.GetType ("System.Console"); - var method = type.Methods.FirstOrDefault (p => p.Name == "Beep"); - Assert.IsNotNull (method, "System.Console.Beep should exist!"); - } + var helper = new ArchiveAssemblyHelper (apk, useAssemblyStore); + Assert.IsTrue (helper.Exists ($"assemblies/{assembly_name}.dll"), $"{assembly_name}.dll should exist in apk!"); + using (var stream = helper.ReadEntry ($"assemblies/{assembly_name}.dll")) { + stream.Position = 0; + using (var assembly = AssemblyDefinition.ReadAssembly (stream)) { + var type = assembly.MainModule.GetType ("System.Console"); + var method = type.Methods.FirstOrDefault (p => p.Name == "Beep"); + Assert.IsNotNull (method, "System.Console.Beep should exist!"); } } } diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/ArchiveAssemblyHelper.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/ArchiveAssemblyHelper.cs new file mode 100644 index 00000000000..4dcf6aca41a --- /dev/null +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/ArchiveAssemblyHelper.cs @@ -0,0 +1,303 @@ +using System; +using System.Buffers; +using System.Collections.Generic; +using System.IO; +using System.Linq; + +using Xamarin.Android.AssemblyStore; +using Xamarin.ProjectTools; +using Xamarin.Tools.Zip; + +namespace Xamarin.Android.Build.Tests +{ + public class ArchiveAssemblyHelper + { + public const string DefaultAssemblyStoreEntryPrefix = "{storeReader}"; + const int AssemblyStoreReadBufferSize = 8192; + + static readonly HashSet SpecialExtensions = new HashSet (StringComparer.OrdinalIgnoreCase) { + ".dll", + ".config", + ".pdb", + ".mdb", + }; + + static readonly Dictionary ArchToAbi = new Dictionary (StringComparer.OrdinalIgnoreCase) { + {"x86", "x86"}, + {"x86_64", "x86_64"}, + {"armeabi_v7a", "armeabi-v7a"}, + {"arm64_v8a", "arm64-v8a"}, + }; + + static readonly ArrayPool buffers = ArrayPool.Shared; + + readonly string archivePath; + readonly string assembliesRootDir; + bool useAssemblyStores; + bool haveMultipleRids; + List archiveContents; + + public string ArchivePath => archivePath; + + public ArchiveAssemblyHelper (string archivePath, bool useAssemblyStores = true, string[] rids = null) + { + if (String.IsNullOrEmpty (archivePath)) { + throw new ArgumentException ("must not be null or empty", nameof (archivePath)); + } + + this.archivePath = archivePath; + this.useAssemblyStores = useAssemblyStores; + haveMultipleRids = rids != null && rids.Length > 1; + + string extension = Path.GetExtension (archivePath) ?? String.Empty; + if (String.Compare (".aab", extension, StringComparison.OrdinalIgnoreCase) == 0) { + assembliesRootDir = "base/root/assemblies"; + } else if (String.Compare (".apk", extension, StringComparison.OrdinalIgnoreCase) == 0) { + assembliesRootDir = "assemblies/"; + } else if (String.Compare (".zip", extension, StringComparison.OrdinalIgnoreCase) == 0) { + assembliesRootDir = "root/assemblies/"; + } else { + assembliesRootDir = String.Empty; + } + } + + public Stream ReadEntry (string path) + { + if (useAssemblyStores) { + return ReadStoreEntry (path); + } + + return ReadZipEntry (path); + } + + Stream ReadZipEntry (string path) + { + using (var zip = ZipHelper.OpenZip (archivePath)) { + ZipEntry entry = zip.ReadEntry (path); + var ret = new MemoryStream (); + entry.Extract (ret); + ret.Flush (); + return ret; + } + } + + Stream ReadStoreEntry (string path) + { + AssemblyStoreReader storeReader = null; + AssemblyStoreAssembly assembly = null; + string name = Path.GetFileNameWithoutExtension (path); + var explorer = new AssemblyStoreExplorer (archivePath); + + foreach (var asm in explorer.Assemblies) { + if (String.Compare (name, asm.Name, StringComparison.Ordinal) != 0) { + continue; + } + assembly = asm; + storeReader = asm.Store; + break; + } + + if (storeReader == null) { + Console.WriteLine ($"Store for entry {path} not found, will try a standard Zip read"); + return ReadZipEntry (path); + } + + string storeEntryName; + if (String.IsNullOrEmpty (storeReader.Arch)) { + storeEntryName = $"{assembliesRootDir}assemblies.blob"; + } else { + storeEntryName = $"{assembliesRootDir}assemblies_{storeReader.Arch}.blob"; + } + + Stream store = ReadZipEntry (storeEntryName); + if (store == null) { + Console.WriteLine ($"Store zip entry {storeEntryName} does not exist"); + return null; + } + + store.Seek (assembly.DataOffset, SeekOrigin.Begin); + var ret = new MemoryStream (); + byte[] buffer = buffers.Rent (AssemblyStoreReadBufferSize); + int toRead = (int)assembly.DataSize; + while (toRead > 0) { + int nread = store.Read (buffer, 0, AssemblyStoreReadBufferSize); + if (nread <= 0) { + break; + } + + ret.Write (buffer, 0, nread); + toRead -= nread; + } + ret.Flush (); + store.Dispose (); + buffers.Return (buffer); + + return ret; + } + + public List ListArchiveContents (string storeEntryPrefix = DefaultAssemblyStoreEntryPrefix, bool forceRefresh = false) + { + if (!forceRefresh && archiveContents != null) { + return archiveContents; + } + + if (String.IsNullOrEmpty (storeEntryPrefix)) { + throw new ArgumentException (nameof (storeEntryPrefix), "must not be null or empty"); + } + + var entries = new List (); + using (var zip = ZipArchive.Open (archivePath, FileMode.Open)) { + foreach (var entry in zip) { + entries.Add (entry.FullName); + } + } + + archiveContents = entries; + if (!useAssemblyStores) { + Console.WriteLine ("Not using assembly stores"); + return entries; + } + + Console.WriteLine ($"Creating AssemblyStoreExplorer for archive '{archivePath}'"); + var explorer = new AssemblyStoreExplorer (archivePath); + Console.WriteLine ($"Explorer found {explorer.Assemblies.Count} assemblies"); + foreach (var asm in explorer.Assemblies) { + string prefix = storeEntryPrefix; + + if (haveMultipleRids &&!String.IsNullOrEmpty (asm.Store.Arch)) { + string arch = ArchToAbi[asm.Store.Arch]; + prefix = $"{prefix}{arch}/"; + } + + entries.Add ($"{prefix}{asm.Name}.dll"); + if (asm.DebugDataOffset > 0) { + entries.Add ($"{prefix}{asm.Name}.pdb"); + } + + if (asm.ConfigDataOffset > 0) { + entries.Add ($"{prefix}{asm.Name}.dll.config"); + } + } + + Console.WriteLine ("Archive entries with synthetised assembly storeReader entries:"); + foreach (string e in entries) { + Console.WriteLine ($" {e}"); + } + + return entries; + } + + public bool Exists (string entryPath, bool forceRefresh = false) + { + List contents = ListArchiveContents (assembliesRootDir, forceRefresh); + if (contents.Count == 0) { + return false; + } + + return contents.Contains (entryPath); + } + + public void Contains (string[] fileNames, out List existingFiles, out List missingFiles, out List additionalFiles) + { + if (fileNames == null) { + throw new ArgumentNullException (nameof (fileNames)); + } + + if (fileNames.Length == 0) { + throw new ArgumentException ("must not be empty", nameof (fileNames)); + } + + if (useAssemblyStores) { + StoreContains (fileNames, out existingFiles, out missingFiles, out additionalFiles); + } else { + ArchiveContains (fileNames, out existingFiles, out missingFiles, out additionalFiles); + } + } + + void ArchiveContains (string[] fileNames, out List existingFiles, out List missingFiles, out List additionalFiles) + { + using (var zip = ZipHelper.OpenZip (archivePath)) { + existingFiles = zip.Where (a => a.FullName.StartsWith (assembliesRootDir, StringComparison.InvariantCultureIgnoreCase)).Select (a => a.FullName).ToList (); + missingFiles = fileNames.Where (x => !zip.ContainsEntry (assembliesRootDir + Path.GetFileName (x))).ToList (); + additionalFiles = existingFiles.Where (x => !fileNames.Contains (Path.GetFileName (x))).ToList (); + } + } + + void StoreContains (string[] fileNames, out List existingFiles, out List missingFiles, out List additionalFiles) + { + var assemblyNames = fileNames.Where (x => x.EndsWith (".dll", StringComparison.OrdinalIgnoreCase)).ToList (); + var configFiles = fileNames.Where (x => x.EndsWith (".config", StringComparison.OrdinalIgnoreCase)).ToList (); + var debugFiles = fileNames.Where (x => x.EndsWith (".pdb", StringComparison.OrdinalIgnoreCase) || x.EndsWith (".mdb", StringComparison.OrdinalIgnoreCase)).ToList (); + var otherFiles = fileNames.Where (x => !SpecialExtensions.Contains (Path.GetExtension (x))).ToList (); + + existingFiles = new List (); + missingFiles = new List (); + additionalFiles = new List (); + + if (otherFiles.Count > 0) { + using (var zip = ZipHelper.OpenZip (archivePath)) { + foreach (string file in otherFiles) { + string fullPath = assembliesRootDir + Path.GetFileName (file); + if (zip.ContainsEntry (fullPath)) { + existingFiles.Add (file); + } + } + } + } + + var explorer = new AssemblyStoreExplorer (archivePath); + + // Assembly stores don't store the assembly extension + var storeAssemblies = explorer.AssembliesByName.Keys.Select (x => $"{x}.dll"); + if (explorer.AssembliesByName.Count != 0) { + existingFiles.AddRange (storeAssemblies); + + // We need to fake config and debug files since they have no named entries in the storeReader + foreach (string file in configFiles) { + AssemblyStoreAssembly asm = GetStoreAssembly (file); + if (asm == null) { + continue; + } + + if (asm.ConfigDataOffset > 0) { + existingFiles.Add (file); + } + } + + foreach (string file in debugFiles) { + AssemblyStoreAssembly asm = GetStoreAssembly (file); + if (asm == null) { + continue; + } + + if (asm.DebugDataOffset > 0) { + existingFiles.Add (file); + } + } + } + + foreach (string file in fileNames) { + if (existingFiles.Contains (Path.GetFileName (file))) { + continue; + } + missingFiles.Add (file); + } + + additionalFiles = existingFiles.Where (x => !fileNames.Contains (x)).ToList (); + + AssemblyStoreAssembly GetStoreAssembly (string file) + { + string assemblyName = Path.GetFileNameWithoutExtension (file); + if (assemblyName.EndsWith (".dll", StringComparison.OrdinalIgnoreCase)) { + assemblyName = Path.GetFileNameWithoutExtension (assemblyName); + } + + if (!explorer.AssembliesByName.TryGetValue (assemblyName, out AssemblyStoreAssembly asm) || asm == null) { + return null; + } + + return asm; + } + } + } +} diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/AssertionExtensions.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/AssertionExtensions.cs index 740bb880c4e..8552a37a22b 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/AssertionExtensions.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/AssertionExtensions.cs @@ -60,12 +60,24 @@ public static void AssertContainsEntry (this ZipArchive zip, string zipPath, str Assert.IsTrue (zip.ContainsEntry (archivePath), $"{zipPath} should contain {archivePath}"); } + [DebuggerHidden] + public static void AssertContainsEntry (this ArchiveAssemblyHelper helper, string archivePath) + { + Assert.IsTrue (helper.Exists (archivePath), $"{helper.ArchivePath} should contain {archivePath}"); + } + [DebuggerHidden] public static void AssertDoesNotContainEntry (this ZipArchive zip, string zipPath, string archivePath) { Assert.IsFalse (zip.ContainsEntry (archivePath), $"{zipPath} should *not* contain {archivePath}"); } + [DebuggerHidden] + public static void AssertDoesNotContainEntry (this ArchiveAssemblyHelper helper, string archivePath) + { + Assert.IsFalse (helper.Exists (archivePath), $"{helper.ArchivePath} should *not* contain {archivePath}"); + } + [DebuggerHidden] public static void AssertContainsEntry (this ZipArchive zip, string zipPath, string archivePath, bool shouldContainEntry) { @@ -76,6 +88,16 @@ public static void AssertContainsEntry (this ZipArchive zip, string zipPath, str } } + [DebuggerHidden] + public static void AssertContainsEntry (this ArchiveAssemblyHelper helper, string archivePath, bool shouldContainEntry) + { + if (shouldContainEntry) { + helper.AssertContainsEntry (archivePath); + } else { + helper.AssertDoesNotContainEntry (archivePath); + } + } + [DebuggerHidden] public static void AssertEntryContents (this ZipArchive zip, string zipPath, string archivePath, string contents) { diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/EnvironmentHelper.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/EnvironmentHelper.cs index 272951341b7..5451c7d99e5 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/EnvironmentHelper.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/EnvironmentHelper.cs @@ -26,15 +26,17 @@ public sealed class ApplicationConfig public bool instant_run_enabled; public bool jni_add_native_method_registration_attribute_present; public bool have_runtime_config_blob; + public bool have_assemblies_blob; public byte bound_stream_io_exception_type; public uint package_naming_policy; public uint environment_variable_count; public uint system_property_count; public uint number_of_assemblies_in_apk; public uint bundled_assembly_name_width; + public uint number_of_assembly_blobs; public string android_package_name; }; - const uint ApplicationConfigFieldCount = 15; + const uint ApplicationConfigFieldCount = 17; static readonly object ndkInitLock = new object (); static readonly char[] readElfFieldSeparator = new [] { ' ', '\t' }; @@ -158,37 +160,47 @@ static ApplicationConfig ReadApplicationConfig (string envFile) ret.have_runtime_config_blob = ConvertFieldToBool ("have_runtime_config_blob", envFile, i, field [1]); break; - case 8: // bound_stream_io_exception_type: byte / .byte + case 8: // have_assemblies_blob: bool / .byte + AssertFieldType (envFile, ".byte", field [0], i); + ret.have_assemblies_blob = ConvertFieldToBool ("have_assemblies_blob", envFile, i, field [1]); + break; + + case 9: // bound_stream_io_exception_type: byte / .byte AssertFieldType (envFile, ".byte", field [0], i); ret.bound_stream_io_exception_type = ConvertFieldToByte ("bound_stream_io_exception_type", envFile, i, field [1]); break; - case 9: // package_naming_policy: uint32_t / .word | .long + case 10: // package_naming_policy: uint32_t / .word | .long Assert.IsTrue (expectedUInt32Types.Contains (field [0]), $"Unexpected uint32_t field type in '{envFile}:{i}': {field [0]}"); ret.package_naming_policy = ConvertFieldToUInt32 ("package_naming_policy", envFile, i, field [1]); break; - case 10: // environment_variable_count: uint32_t / .word | .long + case 11: // environment_variable_count: uint32_t / .word | .long Assert.IsTrue (expectedUInt32Types.Contains (field [0]), $"Unexpected uint32_t field type in '{envFile}:{i}': {field [0]}"); ret.environment_variable_count = ConvertFieldToUInt32 ("environment_variable_count", envFile, i, field [1]); break; - case 11: // system_property_count: uint32_t / .word | .long + case 12: // system_property_count: uint32_t / .word | .long Assert.IsTrue (expectedUInt32Types.Contains (field [0]), $"Unexpected uint32_t field type in '{envFile}:{i}': {field [0]}"); ret.system_property_count = ConvertFieldToUInt32 ("system_property_count", envFile, i, field [1]); break; - case 12: // number_of_assemblies_in_apk: uint32_t / .word | .long + case 13: // number_of_assemblies_in_apk: uint32_t / .word | .long Assert.IsTrue (expectedUInt32Types.Contains (field [0]), $"Unexpected uint32_t field type in '{envFile}:{i}': {field [0]}"); ret.number_of_assemblies_in_apk = ConvertFieldToUInt32 ("number_of_assemblies_in_apk", envFile, i, field [1]); break; - case 13: // bundled_assembly_name_width: uint32_t / .word | .long + case 14: // bundled_assembly_name_width: uint32_t / .word | .long Assert.IsTrue (expectedUInt32Types.Contains (field [0]), $"Unexpected uint32_t field type in '{envFile}:{i}': {field [0]}"); ret.bundled_assembly_name_width = ConvertFieldToUInt32 ("bundled_assembly_name_width", envFile, i, field [1]); break; - case 14: // android_package_name: string / [pointer type] + case 15: // number_of_assembly_blobs: uint32_t / .word | .long + Assert.IsTrue (expectedUInt32Types.Contains (field [0]), $"Unexpected uint32_t field type in '{envFile}:{i}': {field [0]}"); + ret.number_of_assembly_blobs = ConvertFieldToUInt32 ("number_of_assembly_blobs", envFile, i, field [1]); + break; + + case 16: // android_package_name: string / [pointer type] Assert.IsTrue (expectedPointerTypes.Contains (field [0]), $"Unexpected pointer field type in '{envFile}:{i}': {field [0]}"); pointers.Add (field [1].Trim ()); break; diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/XASdkTests.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/XASdkTests.cs index e8317d89e96..3324c25e31b 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/XASdkTests.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/XASdkTests.cs @@ -398,63 +398,104 @@ public void DotNetBuildBinding () /* runtimeIdentifiers */ "android-arm", /* isRelease */ false, /* aot */ false, + /* usesAssemblyStore */ false, + }, + new object [] { + /* runtimeIdentifiers */ "android-arm", + /* isRelease */ false, + /* aot */ false, + /* usesAssemblyStore */ true, }, new object [] { /* runtimeIdentifiers */ "android-arm64", /* isRelease */ false, /* aot */ false, + /* usesAssemblyStore */ false, }, new object [] { /* runtimeIdentifiers */ "android-x86", /* isRelease */ false, /* aot */ false, + /* usesAssemblyStore */ false, }, new object [] { /* runtimeIdentifiers */ "android-x64", /* isRelease */ false, /* aot */ false, + /* usesAssemblyStore */ false, + }, + new object [] { + /* runtimeIdentifiers */ "android-arm", + /* isRelease */ true, + /* aot */ false, + /* usesAssemblyStore */ false, }, new object [] { /* runtimeIdentifiers */ "android-arm", /* isRelease */ true, /* aot */ false, + /* usesAssemblyStore */ true, }, new object [] { /* runtimeIdentifiers */ "android-arm", /* isRelease */ true, /* aot */ true, + /* usesAssemblyStore */ false, + }, + new object [] { + /* runtimeIdentifiers */ "android-arm", + /* isRelease */ true, + /* aot */ true, + /* usesAssemblyStore */ true, }, new object [] { /* runtimeIdentifiers */ "android-arm64", /* isRelease */ true, /* aot */ false, + /* usesAssemblyStore */ false, }, new object [] { /* runtimeIdentifiers */ "android-arm;android-arm64;android-x86;android-x64", /* isRelease */ false, /* aot */ false, + /* usesAssemblyStore */ false, + }, + new object [] { + /* runtimeIdentifiers */ "android-arm;android-arm64;android-x86;android-x64", + /* isRelease */ false, + /* aot */ false, + /* usesAssemblyStore */ true, }, new object [] { /* runtimeIdentifiers */ "android-arm;android-arm64;android-x86", /* isRelease */ true, /* aot */ false, + /* usesAssemblyStore */ false, }, new object [] { /* runtimeIdentifiers */ "android-arm;android-arm64;android-x86;android-x64", /* isRelease */ true, /* aot */ false, + /* usesAssemblyStore */ false, + }, + new object [] { + /* runtimeIdentifiers */ "android-arm;android-arm64;android-x86;android-x64", + /* isRelease */ true, + /* aot */ false, + /* usesAssemblyStore */ true, }, new object [] { /* runtimeIdentifiers */ "android-arm;android-arm64;android-x86;android-x64", /* isRelease */ true, /* aot */ true, + /* usesAssemblyStore */ false, }, }; [Test] [Category ("SmokeTests")] [TestCaseSource (nameof (DotNetBuildSource))] - public void DotNetBuild (string runtimeIdentifiers, bool isRelease, bool aot) + public void DotNetBuild (string runtimeIdentifiers, bool isRelease, bool aot, bool usesAssemblyStore) { var proj = new XASdkProject { IsRelease = isRelease, @@ -489,6 +530,7 @@ public void DotNetBuild (string runtimeIdentifiers, bool isRelease, bool aot) } }; proj.MainActivity = proj.DefaultMainActivity.Replace (": Activity", ": AndroidX.AppCompat.App.AppCompatActivity"); + proj.SetProperty ("AndroidUseAssemblyStore", usesAssemblyStore.ToString ()); if (aot) { proj.SetProperty ("RunAOTCompilation", "true"); } @@ -577,19 +619,18 @@ public void DotNetBuild (string runtimeIdentifiers, bool isRelease, bool aot) bool expectEmbeddedAssembies = !(CommercialBuildAvailable && !isRelease); var apkPath = Path.Combine (outputPath, $"{proj.PackageName}-Signed.apk"); FileAssert.Exists (apkPath); - using (var apk = ZipHelper.OpenZip (apkPath)) { - apk.AssertContainsEntry (apkPath, $"assemblies/{proj.ProjectName}.dll", shouldContainEntry: expectEmbeddedAssembies); - apk.AssertContainsEntry (apkPath, $"assemblies/{proj.ProjectName}.pdb", shouldContainEntry: !CommercialBuildAvailable && !isRelease); - apk.AssertContainsEntry (apkPath, $"assemblies/System.Linq.dll", shouldContainEntry: expectEmbeddedAssembies); - apk.AssertContainsEntry (apkPath, $"assemblies/es/{proj.ProjectName}.resources.dll", shouldContainEntry: expectEmbeddedAssembies); - foreach (var abi in rids.Select (AndroidRidAbiHelper.RuntimeIdentifierToAbi)) { - apk.AssertContainsEntry (apkPath, $"lib/{abi}/libmonodroid.so"); - apk.AssertContainsEntry (apkPath, $"lib/{abi}/libmonosgen-2.0.so"); - if (rids.Length > 1) { - apk.AssertContainsEntry (apkPath, $"assemblies/{abi}/System.Private.CoreLib.dll", shouldContainEntry: expectEmbeddedAssembies); - } else { - apk.AssertContainsEntry (apkPath, "assemblies/System.Private.CoreLib.dll", shouldContainEntry: expectEmbeddedAssembies); - } + var helper = new ArchiveAssemblyHelper (apkPath, usesAssemblyStore, rids); + helper.AssertContainsEntry ($"assemblies/{proj.ProjectName}.dll", shouldContainEntry: expectEmbeddedAssembies); + helper.AssertContainsEntry ($"assemblies/{proj.ProjectName}.pdb", shouldContainEntry: !CommercialBuildAvailable && !isRelease); + helper.AssertContainsEntry ($"assemblies/System.Linq.dll", shouldContainEntry: expectEmbeddedAssembies); + helper.AssertContainsEntry ($"assemblies/es/{proj.ProjectName}.resources.dll", shouldContainEntry: expectEmbeddedAssembies); + foreach (var abi in rids.Select (AndroidRidAbiHelper.RuntimeIdentifierToAbi)) { + helper.AssertContainsEntry ($"lib/{abi}/libmonodroid.so"); + helper.AssertContainsEntry ($"lib/{abi}/libmonosgen-2.0.so"); + if (rids.Length > 1) { + helper.AssertContainsEntry ($"assemblies/{abi}/System.Private.CoreLib.dll", shouldContainEntry: expectEmbeddedAssembies); + } else { + helper.AssertContainsEntry ("assemblies/System.Private.CoreLib.dll", shouldContainEntry: expectEmbeddedAssembies); } } } diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Xamarin.Android.Build.Tests.csproj b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Xamarin.Android.Build.Tests.csproj index 34662f9ca3a..0ffc5865252 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Xamarin.Android.Build.Tests.csproj +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Xamarin.Android.Build.Tests.csproj @@ -32,6 +32,7 @@ + ..\Expected\GenerateDesignerFileExpected.cs PreserveNewest diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Common/DotNetCLI.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Common/DotNetCLI.cs index 0faef635c63..79baa6defb8 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Common/DotNetCLI.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Common/DotNetCLI.cs @@ -112,10 +112,11 @@ public bool Publish (string target = null, string [] parameters = null) public bool Run () { - string binlog = Path.Combine (Path.GetDirectoryName (projectOrSolution), "msbuild.binlog"); + string binlog = Path.Combine (Path.GetDirectoryName (projectOrSolution), "run.binlog"); var arguments = new List { "run", "--project", $"\"{projectOrSolution}\"", + "--no-build", $"/bl:\"{binlog}\"" }; return Execute (arguments.ToArray ()); @@ -146,7 +147,7 @@ List GetDefaultCommandLineArgs (string verb, string target = null, strin $"\"{projectOrSolution}\"", "/noconsolelogger", $"/flp1:LogFile=\"{BuildLogFile}\";Encoding=UTF-8;Verbosity={Verbosity}", - $"/bl:\"{Path.Combine (testDir, "msbuild.binlog")}\"" + $"/bl:\"{Path.Combine (testDir, $"{target}.binlog")}\"" }; if (!string.IsNullOrEmpty (target)) { arguments.Add ($"/t:{target}"); diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Resources/Base/BuildReleaseArm64XFormsLegacy.apkdesc b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Resources/Base/BuildReleaseArm64XFormsLegacy.apkdesc index e7a51b2bc49..2219d8765fe 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Resources/Base/BuildReleaseArm64XFormsLegacy.apkdesc +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Resources/Base/BuildReleaseArm64XFormsLegacy.apkdesc @@ -2,19 +2,19 @@ "Comment": null, "Entries": { "AndroidManifest.xml": { - "Size": 3120 + "Size": 3140 }, "assemblies/FormsViewGroup.dll": { "Size": 7200 }, "assemblies/Java.Interop.dll": { - "Size": 68710 + "Size": 68903 }, "assemblies/Mono.Android.dll": { - "Size": 562181 + "Size": 562556 }, "assemblies/Mono.Security.dll": { - "Size": 68422 + "Size": 68421 }, "assemblies/mscorlib.dll": { "Size": 915407 @@ -47,61 +47,61 @@ "Size": 116883 }, "assemblies/Xamarin.AndroidX.Activity.dll": { - "Size": 7686 + "Size": 7691 }, "assemblies/Xamarin.AndroidX.AppCompat.AppCompatResources.dll": { - "Size": 6635 + "Size": 6639 }, "assemblies/Xamarin.AndroidX.AppCompat.dll": { "Size": 125318 }, "assemblies/Xamarin.AndroidX.CardView.dll": { - "Size": 7354 + "Size": 7357 }, "assemblies/Xamarin.AndroidX.CoordinatorLayout.dll": { - "Size": 18259 + "Size": 18263 }, "assemblies/Xamarin.AndroidX.Core.dll": { - "Size": 131919 + "Size": 131923 }, "assemblies/Xamarin.AndroidX.DrawerLayout.dll": { - "Size": 15417 + "Size": 15413 }, "assemblies/Xamarin.AndroidX.Fragment.dll": { - "Size": 43119 + "Size": 43132 }, "assemblies/Xamarin.AndroidX.Legacy.Support.Core.UI.dll": { - "Size": 6704 + "Size": 6705 }, "assemblies/Xamarin.AndroidX.Lifecycle.Common.dll": { - "Size": 7050 + "Size": 7059 }, "assemblies/Xamarin.AndroidX.Lifecycle.LiveData.Core.dll": { - "Size": 7178 + "Size": 7183 }, "assemblies/Xamarin.AndroidX.Lifecycle.ViewModel.dll": { - "Size": 4860 + "Size": 4864 }, "assemblies/Xamarin.AndroidX.Loader.dll": { - "Size": 13574 + "Size": 13579 }, "assemblies/Xamarin.AndroidX.RecyclerView.dll": { - "Size": 102317 + "Size": 102321 }, "assemblies/Xamarin.AndroidX.SavedState.dll": { - "Size": 6262 + "Size": 6270 }, "assemblies/Xamarin.AndroidX.SwipeRefreshLayout.dll": { - "Size": 11261 + "Size": 11264 }, "assemblies/Xamarin.AndroidX.ViewPager.dll": { - "Size": 19409 + "Size": 19414 }, "assemblies/Xamarin.Forms.Core.dll": { "Size": 524723 }, "assemblies/Xamarin.Forms.Platform.Android.dll": { - "Size": 384855 + "Size": 384866 }, "assemblies/Xamarin.Forms.Platform.dll": { "Size": 56878 @@ -110,10 +110,10 @@ "Size": 55786 }, "assemblies/Xamarin.Google.Android.Material.dll": { - "Size": 43488 + "Size": 43494 }, "classes.dex": { - "Size": 3483748 + "Size": 3483824 }, "lib/arm64-v8a/libmono-btls-shared.so": { "Size": 1613872 @@ -122,7 +122,7 @@ "Size": 707024 }, "lib/arm64-v8a/libmonodroid.so": { - "Size": 281352 + "Size": 290768 }, "lib/arm64-v8a/libmonosgen-2.0.so": { "Size": 4037584 @@ -131,7 +131,7 @@ "Size": 65624 }, "lib/arm64-v8a/libxamarin-app.so": { - "Size": 140560 + "Size": 142176 }, "META-INF/android.support.design_material.version": { "Size": 12 @@ -1883,5 +1883,5 @@ "Size": 341040 } }, - "PackageSize": 9496734 + "PackageSize": 9504926 } \ No newline at end of file diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/ApplicationConfigNativeAssemblyGenerator.cs b/src/Xamarin.Android.Build.Tasks/Utilities/ApplicationConfigNativeAssemblyGenerator.cs index 8f7f69aad12..4d3a2806c05 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/ApplicationConfigNativeAssemblyGenerator.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/ApplicationConfigNativeAssemblyGenerator.cs @@ -24,7 +24,9 @@ class ApplicationConfigNativeAssemblyGenerator : NativeAssemblyGenerator public bool InstantRunEnabled { get; set; } public bool JniAddNativeMethodRegistrationAttributePresent { get; set; } public bool HaveRuntimeConfigBlob { get; set; } + public bool HaveAssemblyStore { get; set; } public int NumberOfAssembliesInApk { get; set; } + public int NumberOfAssemblyStoresInApks { get; set; } public int BundledAssemblyNameWidth { get; set; } // including the trailing NUL public PackageNamingPolicy PackageNamingPolicy { get; set; } @@ -52,7 +54,7 @@ protected override void WriteSymbols (StreamWriter output) WriteDataSection (output, "application_config"); WriteSymbol (output, "application_config", TargetProvider.GetStructureAlignment (true), packed: false, isGlobal: true, alwaysWriteSize: true, structureWriter: () => { // Order of fields and their type must correspond *exactly* to that in - // src/monodroid/jni/xamarin-app.h ApplicationConfig structure + // src/monodroid/jni/xamarin-app.hh ApplicationConfig structure WriteCommentLine (output, "uses_mono_llvm"); uint size = WriteData (output, UsesMonoLLVM); @@ -77,6 +79,9 @@ protected override void WriteSymbols (StreamWriter output) WriteCommentLine (output, "have_runtime_config_blob"); size += WriteData (output, HaveRuntimeConfigBlob); + WriteCommentLine (output, "have_assembly_store"); + size += WriteData (output, HaveAssemblyStore); + WriteCommentLine (output, "bound_exception_type"); size += WriteData (output, (byte)BoundExceptionType); @@ -95,6 +100,9 @@ protected override void WriteSymbols (StreamWriter output) WriteCommentLine (output, "bundled_assembly_name_width"); size += WriteData (output, BundledAssemblyNameWidth); + WriteCommentLine (output, "number_of_assembly_store_files"); + size += WriteData (output, NumberOfAssemblyStoresInApks); + WriteCommentLine (output, "android_package_name"); size += WritePointer (output, MakeLocalLabel (stringLabel)); @@ -110,34 +118,122 @@ protected override void WriteSymbols (StreamWriter output) WriteNameValueStringArray (output, "app_system_properties", systemProperties); WriteBundledAssemblies (output); + WriteAssemblyStoreAssemblies (output); + } + + void WriteAssemblyStoreAssemblies (StreamWriter output) + { + output.WriteLine (); + + string label = "assembly_store_bundled_assemblies"; + WriteCommentLine (output, "Assembly store individual assembly data"); + WriteDataSection (output, label); + WriteStructureSymbol (output, label, alignBits: TargetProvider.MapModulesAlignBits, isGlobal: true); + + uint size = 0; + if (HaveAssemblyStore) { + for (int i = 0; i < NumberOfAssembliesInApk; i++) { + size += WriteStructure (output, packed: false, structureWriter: () => WriteAssemblyStoreAssembly (output)); + } + } + WriteStructureSize (output, label, size); + + output.WriteLine (); + + label = "assembly_stores"; + WriteCommentLine (output, "Assembly store data"); + WriteDataSection (output, label); + WriteStructureSymbol (output, label, alignBits: TargetProvider.MapModulesAlignBits, isGlobal: true); + + size = 0; + if (HaveAssemblyStore) { + for (int i = 0; i < NumberOfAssemblyStoresInApks; i++) { + size += WriteStructure (output, packed: false, structureWriter: () => WriteAssemblyStore (output)); + } + } + WriteStructureSize (output, label, size); + } + + uint WriteAssemblyStoreAssembly (StreamWriter output) + { + // Order of fields and their type must correspond *exactly* to that in + // src/monodroid/jni/xamarin-app.hh AssemblyStoreSingleAssemblyRuntimeData structure + WriteCommentLine (output, "image_data"); + uint size = WritePointer (output); + + WriteCommentLine (output, "debug_info_data"); + size += WritePointer (output); + + WriteCommentLine (output, "config_data"); + size += WritePointer (output); + + WriteCommentLine (output, "descriptor"); + size += WritePointer (output); + + output.WriteLine (); + + return size; + } + + uint WriteAssemblyStore (StreamWriter output) + { + // Order of fields and their type must correspond *exactly* to that in + // src/monodroid/jni/xamarin-app.hh AssemblyStoreRuntimeData structure + WriteCommentLine (output, "data_start"); + uint size = WritePointer (output); + + WriteCommentLine (output, "assembly_count"); + size += WriteData (output, (uint)0); + + WriteCommentLine (output, "assemblies"); + size += WritePointer (output); + + output.WriteLine (); + + return size; } void WriteBundledAssemblies (StreamWriter output) { + output.WriteLine (); + WriteCommentLine (output, $"Bundled assembly name buffers, all {BundledAssemblyNameWidth} bytes long"); WriteSection (output, ".bss.bundled_assembly_names", hasStrings: false, writable: true, nobits: true); - var name_labels = new List (); - for (int i = 0; i < NumberOfAssembliesInApk; i++) { - string bufferLabel = GetBufferLabel (); - WriteBufferAllocation (output, bufferLabel, (uint)BundledAssemblyNameWidth); - name_labels.Add (bufferLabel); + List name_labels = null; + if (!HaveAssemblyStore) { + name_labels = new List (); + for (int i = 0; i < NumberOfAssembliesInApk; i++) { + string bufferLabel = GetBufferLabel (); + WriteBufferAllocation (output, bufferLabel, (uint)BundledAssemblyNameWidth); + name_labels.Add (bufferLabel); + } } + output.WriteLine (); + string label = "bundled_assemblies"; WriteCommentLine (output, "Bundled assemblies data"); WriteDataSection (output, label); WriteStructureSymbol (output, label, alignBits: TargetProvider.MapModulesAlignBits, isGlobal: true); uint size = 0; - for (int i = 0; i < NumberOfAssembliesInApk; i++) { - size += WriteStructure (output, packed: false, structureWriter: () => WriteBundledAssembly (output, MakeLocalLabel (name_labels[i]))); + if (!HaveAssemblyStore) { + for (int i = 0; i < NumberOfAssembliesInApk; i++) { + size += WriteStructure (output, packed: false, structureWriter: () => WriteBundledAssembly (output, MakeLocalLabel (name_labels[i]))); + } } WriteStructureSize (output, label, size); + + output.WriteLine (); } + uint WriteBundledAssembly (StreamWriter output, string nameLabel) { + // Order of fields and their type must correspond *exactly* to that in + // src/monodroid/jni/xamarin-app.hh XamarinAndroidBundledAssembly structure + WriteCommentLine (output, "apk_fd"); uint size = WriteData (output, (int)-1); diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/ArchAssemblyStore.cs b/src/Xamarin.Android.Build.Tasks/Utilities/ArchAssemblyStore.cs new file mode 100644 index 00000000000..a5b5811b7de --- /dev/null +++ b/src/Xamarin.Android.Build.Tasks/Utilities/ArchAssemblyStore.cs @@ -0,0 +1,112 @@ +using System; +using System.Collections.Generic; +using System.IO; + +using Microsoft.Build.Framework; +using Microsoft.Build.Utilities; + +namespace Xamarin.Android.Tasks +{ + class ArchAssemblyStore : AssemblyStore + { + readonly Dictionary> assemblies; + HashSet seenArchAssemblyNames; + + public ArchAssemblyStore (string apkName, string archiveAssembliesPrefix, TaskLoggingHelper log, uint id, AssemblyStoreGlobalIndex globalIndexCounter) + : base (apkName, archiveAssembliesPrefix, log, id, globalIndexCounter) + { + assemblies = new Dictionary> (StringComparer.OrdinalIgnoreCase); + } + + public override string WriteIndex (List globalIndex) + { + throw new InvalidOperationException ("Architecture-specific assembly blob cannot contain global assembly index"); + } + + public override void Add (AssemblyStoreAssemblyInfo blobAssembly) + { + if (String.IsNullOrEmpty (blobAssembly.Abi)) { + throw new InvalidOperationException ($"Architecture-agnostic assembly cannot be added to an architecture-specific blob ({blobAssembly.FilesystemAssemblyPath})"); + } + + if (!assemblies.ContainsKey (blobAssembly.Abi)) { + assemblies.Add (blobAssembly.Abi, new List ()); + } + + List blobAssemblies = assemblies[blobAssembly.Abi]; + blobAssemblies.Add (blobAssembly); + + if (seenArchAssemblyNames == null) { + seenArchAssemblyNames = new HashSet (StringComparer.Ordinal); + } + + string assemblyName = GetAssemblyName (blobAssembly); + if (seenArchAssemblyNames.Contains (assemblyName)) { + return; + } + + seenArchAssemblyNames.Add (assemblyName); + } + + public override void Generate (string outputDirectory, List globalIndex, List blobPaths) + { + if (assemblies.Count == 0) { + return; + } + + var assemblyNames = new Dictionary (); + foreach (var kvp in assemblies) { + string abi = kvp.Key; + List archAssemblies = kvp.Value; + + // All the architecture blobs must have assemblies in exactly the same order + archAssemblies.Sort ((AssemblyStoreAssemblyInfo a, AssemblyStoreAssemblyInfo b) => Path.GetFileName (a.FilesystemAssemblyPath).CompareTo (Path.GetFileName (b.FilesystemAssemblyPath))); + if (assemblyNames.Count == 0) { + for (int i = 0; i < archAssemblies.Count; i++) { + AssemblyStoreAssemblyInfo info = archAssemblies[i]; + assemblyNames.Add (i, Path.GetFileName (info.FilesystemAssemblyPath)); + } + continue; + } + + if (archAssemblies.Count != assemblyNames.Count) { + throw new InvalidOperationException ($"Assembly list for ABI '{abi}' has a different number of assemblies than other ABI lists (expected {assemblyNames.Count}, found {archAssemblies.Count}"); + } + + for (int i = 0; i < archAssemblies.Count; i++) { + AssemblyStoreAssemblyInfo info = archAssemblies[i]; + string fileName = Path.GetFileName (info.FilesystemAssemblyPath); + + if (assemblyNames[i] != fileName) { + throw new InvalidOperationException ($"Assembly list for ABI '{abi}' differs from other lists at index {i}. Expected '{assemblyNames[i]}', found '{fileName}'"); + } + } + } + + bool addToGlobalIndex = true; + foreach (var kvp in assemblies) { + string abi = kvp.Key; + List archAssemblies = kvp.Value; + + if (archAssemblies.Count == 0) { + continue; + } + + // Android uses underscores in place of dashes in ABI names, let's follow the convention + string androidAbi = abi.Replace ('-', '_'); + Generate (Path.Combine (outputDirectory, $"{ApkName}_{BlobPrefix}.{androidAbi}{BlobExtension}"), archAssemblies, globalIndex, blobPaths, addToGlobalIndex); + + // NOTE: not thread safe! The counter must grow monotonically but we also don't want to use different index values for the architecture-specific + // assemblies with the same names, that would only waste space in the generated `libxamarin-app.so`. To use the same index values for the same + // assemblies in different architectures we need to move the counter back here. + GlobalIndexCounter.Subtract ((uint)archAssemblies.Count); + + if (addToGlobalIndex) { + // We want the architecture-specific assemblies to be added to the global index only once + addToGlobalIndex = false; + } + } + + } + } +} diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStore.cs b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStore.cs new file mode 100644 index 00000000000..378a9ee8c46 --- /dev/null +++ b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStore.cs @@ -0,0 +1,377 @@ +using System; +using System.Buffers; +using System.Collections.Generic; +using System.IO; +using System.Text; + +using Microsoft.Build.Framework; +using Microsoft.Build.Utilities; + +namespace Xamarin.Android.Tasks +{ + abstract class AssemblyStore + { + // The two constants below must match their counterparts in src/monodroid/jni/xamarin-app.hh + const uint BlobMagic = 0x41424158; // 'XABA', little-endian, must match the BUNDLED_ASSEMBLIES_BLOB_MAGIC native constant + const uint BlobVersion = 1; // Must match the BUNDLED_ASSEMBLIES_BLOB_VERSION native constant + + // MUST be equal to the size of the BlobBundledAssembly struct in src/monodroid/jni/xamarin-app.hh + const uint BlobBundledAssemblyNativeStructSize = 6 * sizeof (uint); + + // MUST be equal to the size of the BlobHashEntry struct in src/monodroid/jni/xamarin-app.hh + const uint BlobHashEntryNativeStructSize = sizeof (ulong) + (3 * sizeof (uint)); + + // MUST be equal to the size of the BundledAssemblyBlobHeader struct in src/monodroid/jni/xamarin-app.hh + const uint BlobHeaderNativeStructSize = sizeof (uint) * 5; + + protected const string BlobPrefix = "assemblies"; + protected const string BlobExtension = ".blob"; + + static readonly ArrayPool bytePool = ArrayPool.Shared; + + string archiveAssembliesPrefix; + string indexBlobPath; + + protected string ApkName { get; } + protected TaskLoggingHelper Log { get; } + protected AssemblyStoreGlobalIndex GlobalIndexCounter { get; } + + public uint ID { get; } + public bool IsIndexStore => ID == 0; + + protected AssemblyStore (string apkName, string archiveAssembliesPrefix, TaskLoggingHelper log, uint id, AssemblyStoreGlobalIndex globalIndexCounter) + { + if (String.IsNullOrEmpty (archiveAssembliesPrefix)) { + throw new ArgumentException ("must not be null or empty", nameof (archiveAssembliesPrefix)); + } + + if (String.IsNullOrEmpty (apkName)) { + throw new ArgumentException ("must not be null or empty", nameof (apkName)); + } + + GlobalIndexCounter = globalIndexCounter ?? throw new ArgumentNullException (nameof (globalIndexCounter)); + ID = id; + + this.archiveAssembliesPrefix = archiveAssembliesPrefix; + ApkName = apkName; + Log = log; + } + + public abstract void Add (AssemblyStoreAssemblyInfo blobAssembly); + public abstract void Generate (string outputDirectory, List globalIndex, List blobPaths); + + public virtual string WriteIndex (List globalIndex) + { + if (!IsIndexStore) { + throw new InvalidOperationException ("Assembly index may be written only to blob with index 0"); + } + + if (String.IsNullOrEmpty (indexBlobPath)) { + throw new InvalidOperationException ("Index blob path not set, was Generate called properly?"); + } + + if (globalIndex == null) { + throw new ArgumentNullException (nameof (globalIndex)); + } + + string indexBlobHeaderPath = $"{indexBlobPath}.hdr"; + string indexBlobManifestPath = Path.ChangeExtension (indexBlobPath, "manifest"); + + using (var hfs = File.Open (indexBlobHeaderPath, FileMode.Create, FileAccess.Write, FileShare.None)) { + using (var writer = new BinaryWriter (hfs, Encoding.UTF8, leaveOpen: true)) { + WriteIndex (writer, indexBlobManifestPath, globalIndex); + writer.Flush (); + } + + using (var ifs = File.Open (indexBlobPath, FileMode.Open, FileAccess.Read, FileShare.Read)) { + ifs.CopyTo (hfs); + hfs.Flush (); + } + } + + File.Delete (indexBlobPath); + File.Move (indexBlobHeaderPath, indexBlobPath); + + return indexBlobManifestPath; + } + + void WriteIndex (BinaryWriter blobWriter, string manifestPath, List globalIndex) + { + using (var manifest = File.Open (manifestPath, FileMode.Create, FileAccess.Write)) { + using (var manifestWriter = new StreamWriter (manifest, new UTF8Encoding (false))) { + WriteIndex (blobWriter, manifestWriter, globalIndex); + manifestWriter.Flush (); + } + } + } + + void WriteIndex (BinaryWriter blobWriter, StreamWriter manifestWriter, List globalIndex) + { + uint localEntryCount = 0; + var localAssemblies = new List (); + + manifestWriter.WriteLine ("Hash 32 Hash 64 Blob ID Blob idx Name"); + + var seenHashes32 = new HashSet (); + var seenHashes64 = new HashSet (); + bool haveDuplicates = false; + foreach (AssemblyStoreIndexEntry assembly in globalIndex) { + if (assembly.StoreID == ID) { + localEntryCount++; + localAssemblies.Add (assembly); + } + + if (WarnAboutDuplicateHash ("32", assembly.Name, assembly.NameHash32, seenHashes32) || + WarnAboutDuplicateHash ("64", assembly.Name, assembly.NameHash64, seenHashes64)) { + haveDuplicates = true; + } + + manifestWriter.WriteLine ($"0x{assembly.NameHash32:x08} 0x{assembly.NameHash64:x016} {assembly.StoreID:d03} {assembly.LocalBlobIndex:d04} {assembly.Name}"); + } + + if (haveDuplicates) { + throw new InvalidOperationException ("Duplicate assemblies encountered"); + } + + uint globalAssemblyCount = (uint)globalIndex.Count; + + blobWriter.Seek (0, SeekOrigin.Begin); + WriteBlobHeader (blobWriter, localEntryCount, globalAssemblyCount); + + // Header and two tables of the same size, each for 32 and 64-bit hashes + uint offsetFixup = BlobHeaderNativeStructSize + (BlobHashEntryNativeStructSize * globalAssemblyCount * 2); + + WriteAssemblyDescriptors (blobWriter, localAssemblies, CalculateOffsetFixup ((uint)localAssemblies.Count, offsetFixup)); + + var sortedIndex = new List (globalIndex); + sortedIndex.Sort ((AssemblyStoreIndexEntry a, AssemblyStoreIndexEntry b) => a.NameHash32.CompareTo (b.NameHash32)); + foreach (AssemblyStoreIndexEntry entry in sortedIndex) { + WriteHash (entry, entry.NameHash32); + } + + sortedIndex.Sort ((AssemblyStoreIndexEntry a, AssemblyStoreIndexEntry b) => a.NameHash64.CompareTo (b.NameHash64)); + foreach (AssemblyStoreIndexEntry entry in sortedIndex) { + WriteHash (entry, entry.NameHash64); + } + + void WriteHash (AssemblyStoreIndexEntry entry, ulong hash) + { + blobWriter.Write (hash); + blobWriter.Write (entry.MappingIndex); + blobWriter.Write (entry.LocalBlobIndex); + blobWriter.Write (entry.StoreID); + } + + bool WarnAboutDuplicateHash (string bitness, string assemblyName, ulong hash, HashSet seenHashes) + { + if (seenHashes.Contains (hash)) { + Log.LogMessage (MessageImportance.High, $"Duplicate {bitness}-bit hash 0x{hash} encountered for assembly {assemblyName}"); + return true; + } + + seenHashes.Add (hash); + return false; + } + } + + protected string GetAssemblyName (AssemblyStoreAssemblyInfo assembly) + { + string assemblyName = Path.GetFileNameWithoutExtension (assembly.FilesystemAssemblyPath); + if (assemblyName.EndsWith (".dll", StringComparison.OrdinalIgnoreCase)) { + assemblyName = Path.GetFileNameWithoutExtension (assemblyName); + } + + return assemblyName; + } + + protected void Generate (string outputFilePath, List assemblies, List globalIndex, List blobPaths, bool addToGlobalIndex = true) + { + if (globalIndex == null) { + throw new ArgumentNullException (nameof (globalIndex)); + } + + if (blobPaths == null) { + throw new ArgumentNullException (nameof (blobPaths)); + } + + if (IsIndexStore) { + indexBlobPath = outputFilePath; + } + + blobPaths.Add (outputFilePath); + Log.LogMessage (MessageImportance.Low, $"AssemblyBlobGenerator: generating blob: {outputFilePath}"); + + using (var fs = File.Open (outputFilePath, FileMode.Create, FileAccess.Write, FileShare.Read)) { + using (var writer = new BinaryWriter (fs, Encoding.UTF8)) { + Generate (writer, assemblies, globalIndex, addToGlobalIndex); + writer.Flush (); + } + } + } + + void Generate (BinaryWriter writer, List assemblies, List globalIndex, bool addToGlobalIndex) + { + var localAssemblies = new List (); + + if (!IsIndexStore) { + // Index blob's header and data before the assemblies is handled in WriteIndex in a slightly different + // way. + uint nbytes = BlobHeaderNativeStructSize + (BlobBundledAssemblyNativeStructSize * (uint)assemblies.Count); + var zeros = bytePool.Rent ((int)nbytes); + writer.Write (zeros, 0, (int)nbytes); + bytePool.Return (zeros); + } + + foreach (AssemblyStoreAssemblyInfo assembly in assemblies) { + string assemblyName = GetAssemblyName (assembly); + string archivePath = assembly.ArchiveAssemblyPath; + if (archivePath.StartsWith (archiveAssembliesPrefix, StringComparison.OrdinalIgnoreCase)) { + archivePath = archivePath.Substring (archiveAssembliesPrefix.Length); + } + + if (!String.IsNullOrEmpty (assembly.Abi)) { + string abiPath = $"{assembly.Abi}/"; + if (archivePath.StartsWith (abiPath, StringComparison.Ordinal)) { + archivePath = archivePath.Substring (abiPath.Length); + } + } + + if (!String.IsNullOrEmpty (archivePath)) { + if (archivePath.EndsWith ("/", StringComparison.Ordinal)) { + assemblyName = $"{archivePath}{assemblyName}"; + } else { + assemblyName = $"{archivePath}/{assemblyName}"; + } + } + + AssemblyStoreIndexEntry entry = WriteAssembly (writer, assembly, assemblyName, (uint)localAssemblies.Count); + if (addToGlobalIndex) { + globalIndex.Add (entry); + } + localAssemblies.Add (entry); + } + + writer.Flush (); + + if (IsIndexStore) { + return; + } + + writer.Seek (0, SeekOrigin.Begin); + WriteBlobHeader (writer, (uint)localAssemblies.Count); + WriteAssemblyDescriptors (writer, localAssemblies); + } + + uint CalculateOffsetFixup (uint localAssemblyCount, uint extraOffset = 0) + { + return (BlobBundledAssemblyNativeStructSize * (uint)localAssemblyCount) + extraOffset; + } + + void WriteBlobHeader (BinaryWriter writer, uint localEntryCount, uint globalEntryCount = 0) + { + // Header, must be identical to the BundledAssemblyBlobHeader structure in src/monodroid/jni/xamarin-app.hh + writer.Write (BlobMagic); // magic + writer.Write (BlobVersion); // version + writer.Write (localEntryCount); // local_entry_count + writer.Write (globalEntryCount); // global_entry_count + writer.Write ((uint)ID); // blob_id + } + + void WriteAssemblyDescriptors (BinaryWriter writer, List assemblies, uint offsetFixup = 0) + { + // Each assembly must be identical to the BlobBundledAssembly structure in src/monodroid/jni/xamarin-app.hh + + foreach (AssemblyStoreIndexEntry assembly in assemblies) { + AdjustOffsets (assembly, offsetFixup); + + writer.Write (assembly.DataOffset); + writer.Write (assembly.DataSize); + + writer.Write (assembly.DebugDataOffset); + writer.Write (assembly.DebugDataSize); + + writer.Write (assembly.ConfigDataOffset); + writer.Write (assembly.ConfigDataSize); + } + } + + void AdjustOffsets (AssemblyStoreIndexEntry assembly, uint offsetFixup) + { + if (offsetFixup == 0) { + return; + } + + assembly.DataOffset += offsetFixup; + + if (assembly.DebugDataOffset > 0) { + assembly.DebugDataOffset += offsetFixup; + } + + if (assembly.ConfigDataOffset > 0) { + assembly.ConfigDataOffset += offsetFixup; + } + } + + AssemblyStoreIndexEntry WriteAssembly (BinaryWriter writer, AssemblyStoreAssemblyInfo assembly, string assemblyName, uint localBlobIndex) + { + uint offset; + uint size; + + (offset, size) = WriteFile (assembly.FilesystemAssemblyPath, true); + + // NOTE: globalAssemblIndex++ is not thread safe but it **must** increase monotonically (see also ArchAssemblyStore.Generate for a special case) + var ret = new AssemblyStoreIndexEntry (assemblyName, ID, GlobalIndexCounter.Increment (), localBlobIndex) { + DataOffset = offset, + DataSize = size, + }; + + (offset, size) = WriteFile (assembly.DebugInfoPath, required: false); + if (offset != 0 && size != 0) { + ret.DebugDataOffset = offset; + ret.DebugDataSize = size; + } + + // Config files must end with \0 (nul) + (offset, size) = WriteFile (assembly.ConfigPath, required: false, appendNul: true); + if (offset != 0 && size != 0) { + ret.ConfigDataOffset = offset; + ret.ConfigDataSize = size; + } + + return ret; + + (uint offset, uint size) WriteFile (string filePath, bool required, bool appendNul = false) + { + if (!File.Exists (filePath)) { + if (required) { + throw new InvalidOperationException ($"Required file '{filePath}' not found"); + } + + return (0, 0); + } + + var fi = new FileInfo (filePath); + if (fi.Length == 0) { + return (0, 0); + } + + if (fi.Length > UInt32.MaxValue || writer.BaseStream.Position + fi.Length > UInt32.MaxValue) { + throw new InvalidOperationException ($"Writing assembly '{filePath}' to assembly blob would exceed the maximum allowed data size."); + } + + uint offset = (uint)writer.BaseStream.Position; + using (var fs = File.Open (filePath, FileMode.Open, FileAccess.Read, FileShare.Read)) { + fs.CopyTo (writer.BaseStream); + } + + uint length = (uint)fi.Length; + if (appendNul) { + length++; + writer.Write ((byte)0); + } + + return (offset, length); + } + } + } +} diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreAssemblyInfo.cs b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreAssemblyInfo.cs new file mode 100644 index 00000000000..c5c166fb787 --- /dev/null +++ b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreAssemblyInfo.cs @@ -0,0 +1,48 @@ +using System; +using System.IO; + +namespace Xamarin.Android.Tasks +{ + class AssemblyStoreAssemblyInfo + { + public string FilesystemAssemblyPath { get; } + public string ArchiveAssemblyPath { get; } + public string DebugInfoPath { get; private set; } + public string ConfigPath { get; private set; } + public string Abi { get; } + + public AssemblyStoreAssemblyInfo (string filesystemAssemblyPath, string archiveAssemblyPath, string abi) + { + if (String.IsNullOrEmpty (filesystemAssemblyPath)) { + throw new ArgumentException ("must not be null or empty", nameof (filesystemAssemblyPath)); + } + + if (String.IsNullOrEmpty (archiveAssemblyPath)) { + throw new ArgumentException ("must not be null or empty", nameof (archiveAssemblyPath)); + } + + FilesystemAssemblyPath = filesystemAssemblyPath; + ArchiveAssemblyPath = archiveAssemblyPath; + Abi = abi; + } + + public void SetDebugInfoPath (string path) + { + DebugInfoPath = GetExistingPath (path); + } + + public void SetConfigPath (string path) + { + ConfigPath = GetExistingPath (path); + } + + string GetExistingPath (string path) + { + if (String.IsNullOrEmpty (path) || !File.Exists (path)) { + return String.Empty; + } + + return path; + } + } +} diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreGenerator.cs b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreGenerator.cs new file mode 100644 index 00000000000..d60af903bc9 --- /dev/null +++ b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreGenerator.cs @@ -0,0 +1,133 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Text; + +using Microsoft.Build.Framework; +using Microsoft.Build.Utilities; + +namespace Xamarin.Android.Tasks +{ + class AssemblyStoreGenerator + { + sealed class Store + { + public AssemblyStore Common; + public AssemblyStore Arch; + } + + readonly string archiveAssembliesPrefix; + readonly TaskLoggingHelper log; + + // NOTE: when/if we have parallel BuildApk these should become concurrent collections + readonly Dictionary stores = new Dictionary (StringComparer.Ordinal); + + AssemblyStore indexStore; + + // IDs must be counted per AssemblyStoreGenerator instance because it's possible that a single build will create more than one instance of the class and each time + // the stores must be assigned IDs starting from 0, or there will be errors due to "missing" index store + readonly Dictionary apkIds = new Dictionary (StringComparer.Ordinal); + + // Global assembly index must be restarted from 0 for the same reasons as apkIds above and at the same time it must be unique for each assembly added to **any** + // assembly store, thus we need to keep the state here + AssemblyStoreGlobalIndex globalIndexCounter = new AssemblyStoreGlobalIndex (); + + public AssemblyStoreGenerator (string archiveAssembliesPrefix, TaskLoggingHelper log) + { + if (String.IsNullOrEmpty (archiveAssembliesPrefix)) { + throw new ArgumentException ("must not be null or empty", nameof (archiveAssembliesPrefix)); + } + + this.archiveAssembliesPrefix = archiveAssembliesPrefix; + this.log = log; + } + + public void Add (string apkName, AssemblyStoreAssemblyInfo storeAssembly) + { + if (String.IsNullOrEmpty (apkName)) { + throw new ArgumentException ("must not be null or empty", nameof (apkName)); + } + + Store store; + if (!stores.ContainsKey (apkName)) { + store = new Store { + Common = new CommonAssemblyStore (apkName, archiveAssembliesPrefix, log, GetNextStoreID (apkName), globalIndexCounter), + Arch = new ArchAssemblyStore (apkName, archiveAssembliesPrefix, log, GetNextStoreID (apkName), globalIndexCounter) + }; + + stores.Add (apkName, store); + SetIndexStore (store.Common); + SetIndexStore (store.Arch); + } + + store = stores[apkName]; + if (String.IsNullOrEmpty (storeAssembly.Abi)) { + store.Common.Add (storeAssembly); + } else { + store.Arch.Add (storeAssembly); + } + + void SetIndexStore (AssemblyStore b) + { + if (!b.IsIndexStore) { + return; + } + + if (indexStore != null) { + throw new InvalidOperationException ("Index store already set!"); + } + + indexStore = b; + } + } + + uint GetNextStoreID (string apkName) + { + // NOTE: NOT thread safe, if we ever have parallel runs of BuildApk this operation must either be atomic or protected with a lock + if (!apkIds.ContainsKey (apkName)) { + apkIds.Add (apkName, 0); + } + return apkIds[apkName]++; + } + + public Dictionary> Generate (string outputDirectory) + { + if (stores.Count == 0) { + return null; + } + + if (indexStore == null) { + throw new InvalidOperationException ("Index store not found"); + } + + var globalIndex = new List (); + var ret = new Dictionary> (StringComparer.Ordinal); + string indexStoreApkName = null; + foreach (var kvp in stores) { + string apkName = kvp.Key; + Store store = kvp.Value; + + if (!ret.ContainsKey (apkName)) { + ret.Add (apkName, new List ()); + } + + if (store.Common == indexStore || store.Arch == indexStore) { + indexStoreApkName = apkName; + } + + GenerateStore (store.Common, apkName); + GenerateStore (store.Arch, apkName); + } + + string manifestPath = indexStore.WriteIndex (globalIndex); + ret[indexStoreApkName].Add (manifestPath); + + return ret; + + void GenerateStore (AssemblyStore store, string apkName) + { + store.Generate (outputDirectory, globalIndex, ret[apkName]); + } + } + } +} diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreGlobalIndex.cs b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreGlobalIndex.cs new file mode 100644 index 00000000000..6ce93f11f9d --- /dev/null +++ b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreGlobalIndex.cs @@ -0,0 +1,29 @@ +namespace Xamarin.Android.Tasks +{ + // This class may seem weird, but it's designed with the specific needs of AssemblyStore instances in mind and also prepared for thread-safe use in the future, should the + // need arise + sealed class AssemblyStoreGlobalIndex + { + uint value = 0; + + public uint Value => value; + + /// + /// Increments the counter and returns its previous value + /// + public uint Increment () + { + uint ret = value++; + return ret; + } + + public void Subtract (uint count) + { + if (value < count) { + return; + } + + value -= count; + } + } +} diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreIndexEntry.cs b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreIndexEntry.cs new file mode 100644 index 00000000000..59f012f5142 --- /dev/null +++ b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreIndexEntry.cs @@ -0,0 +1,44 @@ +using System; +using System.Text; + +using K4os.Hash.xxHash; + +namespace Xamarin.Android.Tasks +{ + class AssemblyStoreIndexEntry + { + public string Name { get; } + public uint StoreID { get; } + public uint MappingIndex { get; } + public uint LocalBlobIndex { get; } + + // Hash values must have the same type as they are inside a union in the native code + public ulong NameHash64 { get; } + public ulong NameHash32 { get; } + + public uint DataOffset { get; set; } + public uint DataSize { get; set; } + + public uint DebugDataOffset { get; set; } + public uint DebugDataSize { get; set; } + + public uint ConfigDataOffset { get; set; } + public uint ConfigDataSize { get; set; } + + public AssemblyStoreIndexEntry (string name, uint blobID, uint mappingIndex, uint localBlobIndex) + { + if (String.IsNullOrEmpty (name)) { + throw new ArgumentException ("must not be null or empty", nameof (name)); + } + + Name = name; + StoreID = blobID; + MappingIndex = mappingIndex; + LocalBlobIndex = localBlobIndex; + + byte[] nameBytes = Encoding.UTF8.GetBytes (name); + NameHash32 = XXH32.DigestOf (nameBytes, 0, nameBytes.Length); + NameHash64 = XXH64.DigestOf (nameBytes, 0, nameBytes.Length); + } + } +} diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/CommonAssemblyStore.cs b/src/Xamarin.Android.Build.Tasks/Utilities/CommonAssemblyStore.cs new file mode 100644 index 00000000000..9709a200e5a --- /dev/null +++ b/src/Xamarin.Android.Build.Tasks/Utilities/CommonAssemblyStore.cs @@ -0,0 +1,39 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Text; + +using Microsoft.Build.Framework; +using Microsoft.Build.Utilities; + +namespace Xamarin.Android.Tasks +{ + class CommonAssemblyStore : AssemblyStore + { + readonly List assemblies; + + public CommonAssemblyStore (string apkName, string archiveAssembliesPrefix, TaskLoggingHelper log, uint id, AssemblyStoreGlobalIndex globalIndexCounter) + : base (apkName, archiveAssembliesPrefix, log, id, globalIndexCounter) + { + assemblies = new List (); + } + + public override void Add (AssemblyStoreAssemblyInfo blobAssembly) + { + if (!String.IsNullOrEmpty (blobAssembly.Abi)) { + throw new InvalidOperationException ($"Architecture-specific assembly cannot be added to an architecture-agnostic blob ({blobAssembly.FilesystemAssemblyPath})"); + } + + assemblies.Add (blobAssembly); + } + + public override void Generate (string outputDirectory, List globalIndex, List blobPaths) + { + if (assemblies.Count == 0) { + return; + } + + Generate (Path.Combine (outputDirectory, $"{ApkName}_{BlobPrefix}{BlobExtension}"), assemblies, globalIndex, blobPaths); + } + } +} diff --git a/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Build.Tasks.csproj b/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Build.Tasks.csproj index 8e8d4fdacd7..02f03841016 100644 --- a/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Build.Tasks.csproj +++ b/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Build.Tasks.csproj @@ -58,6 +58,7 @@ + diff --git a/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets b/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets index ccf5537b1f5..e705656efd5 100644 --- a/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets +++ b/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets @@ -179,6 +179,8 @@ Copyright (C) 2011-2012 Xamarin. All rights reserved. Android.App.Fragment True False + False + True <_AndroidCheckedBuild Condition=" '$(_AndroidCheckedBuild)' == '' "> @@ -1510,6 +1512,7 @@ because xbuild doesn't support framework reference assemblies. .so;$(AndroidStoreUncompressedFileExtensions) .dex;$(AndroidStoreUncompressedFileExtensions) + .blob;$(AndroidStoreUncompressedFileExtensions) @@ -1596,6 +1599,7 @@ because xbuild doesn't support framework reference assemblies. InstantRunEnabled="$(_InstantRunEnabled)" RuntimeConfigBinFilePath="$(_BinaryRuntimeConfigPath)" UsingAndroidNETSdk="$(UsingAndroidNETSdk)" + UseAssemblyStore="$(AndroidUseAssemblyStore)" > @@ -2004,7 +2008,8 @@ because xbuild doesn't support framework reference assemblies. ProjectFullPath="$(MSBuildProjectFullPath)" IncludeWrapSh="$(AndroidIncludeWrapSh)" CheckedBuild="$(_AndroidCheckedBuild)" - RuntimeConfigBinFilePath="$(_BinaryRuntimeConfigPath)"> + RuntimeConfigBinFilePath="$(_BinaryRuntimeConfigPath)" + UseAssemblyStore="$(AndroidUseAssemblyStore)"> + RuntimeConfigBinFilePath="$(_BinaryRuntimeConfigPath)" + UseAssemblyStore="$(AndroidUseAssemblyStore)"> <_Target Condition="'$(JavacSourceVersion)' != ''">-source $(JavacSourceVersion) -target $(JavacTargetVersion) - <_AndroidJar>"$(AndroidToolchainDirectory)\sdk\platforms\android-$(AndroidFirstPlatformId)\android.jar" + <_AndroidJar>"$(AndroidToolchainDirectory)\sdk\platforms\android-$(AndroidJavaRuntimeApiLevel)\android.jar" = 21) { + if (runtimePackage.splitSourceDirs != null) { + haveSplitApks = runtimePackage.splitSourceDirs.length > 1; + } + } // // Preload DSOs libmonodroid.so depends on so that the dynamic @@ -73,7 +80,7 @@ public static void LoadApplication (Context context, ApplicationInfo runtimePack // if (BuildConfig.Debug) { System.loadLibrary ("xamarin-debug-app-helper"); - DebugRuntime.init (apks, runtimeDir, appDirs); + DebugRuntime.init (apks, runtimeDir, appDirs, haveSplitApks); } else { System.loadLibrary("monosgen-2.0"); } @@ -98,7 +105,8 @@ public static void LoadApplication (Context context, ApplicationInfo runtimePack loader, MonoPackageManager_Resources.Assemblies, Build.VERSION.SDK_INT, - isEmulator () + isEmulator (), + haveSplitApks ); mono.android.app.ApplicationRegistration.registerApplications (); diff --git a/src/java-runtime/java/mono/android/Runtime.java b/src/java-runtime/java/mono/android/Runtime.java index fb38ba728bb..62547d2b157 100644 --- a/src/java-runtime/java/mono/android/Runtime.java +++ b/src/java-runtime/java/mono/android/Runtime.java @@ -15,7 +15,7 @@ public class Runtime { } public static native void init (String lang, String[] runtimeApks, String runtimeDataDir, String[] appDirs, ClassLoader loader, String[] externalStorageDirs, String[] assemblies, String packageName, int apiLevel, String[] environmentVariables); - public static native void initInternal (String lang, String[] runtimeApks, String runtimeDataDir, String[] appDirs, ClassLoader loader, String[] assemblies, int apiLevel, boolean isEmulator); + public static native void initInternal (String lang, String[] runtimeApks, String runtimeDataDir, String[] appDirs, ClassLoader loader, String[] assemblies, int apiLevel, boolean isEmulator, boolean haveSplitApks); public static native void register (String managedType, java.lang.Class nativeClass, String methods); public static native void notifyTimeZoneChanged (); public static native int createNewContext (String[] runtimeApks, String[] assemblies, ClassLoader loader); diff --git a/src/monodroid/jni/application_dso_stub.cc b/src/monodroid/jni/application_dso_stub.cc index d88d57c8807..51c6e1eecea 100644 --- a/src/monodroid/jni/application_dso_stub.cc +++ b/src/monodroid/jni/application_dso_stub.cc @@ -34,6 +34,10 @@ CompressedAssemblies compressed_assemblies = { /*.descriptors = */ nullptr, }; +// +// Config settings below **must** be valid for Desktop builds as the default `libxamarin-app.{dll,dylib,so}` is used by +// the Designer +// ApplicationConfig application_config = { .uses_mono_llvm = false, .uses_mono_aot = false, @@ -43,12 +47,14 @@ ApplicationConfig application_config = { .instant_run_enabled = false, .jni_add_native_method_registration_attribute_present = false, .have_runtime_config_blob = false, + .have_assembly_store = false, .bound_exception_type = 0, // System .package_naming_policy = 0, .environment_variable_count = 0, .system_property_count = 0, - .number_of_assemblies_in_apk = 0, + .number_of_assemblies_in_apk = 2, .bundled_assembly_name_width = 0, + .number_of_assembly_store_files = 2, .android_package_name = "com.xamarin.test", }; @@ -80,3 +86,33 @@ XamarinAndroidBundledAssembly bundled_assemblies[] = { .name = second_assembly_name, }, }; + +AssemblyStoreSingleAssemblyRuntimeData assembly_store_bundled_assemblies[] = { + { + .image_data = nullptr, + .debug_info_data = nullptr, + .config_data = nullptr, + .descriptor = nullptr, + }, + + { + .image_data = nullptr, + .debug_info_data = nullptr, + .config_data = nullptr, + .descriptor = nullptr, + }, +}; + +AssemblyStoreRuntimeData assembly_stores[] = { + { + .data_start = nullptr, + .assembly_count = 0, + .assemblies = nullptr, + }, + + { + .data_start = nullptr, + .assembly_count = 0, + .assemblies = nullptr, + }, +}; diff --git a/src/monodroid/jni/basic-android-system.cc b/src/monodroid/jni/basic-android-system.cc index 12c5f7e5191..d27e7da7328 100644 --- a/src/monodroid/jni/basic-android-system.cc +++ b/src/monodroid/jni/basic-android-system.cc @@ -28,7 +28,7 @@ BasicAndroidSystem::detect_embedded_dso_mode (jstring_array_wrapper& appDirs) no } void -BasicAndroidSystem::setup_app_library_directories (jstring_array_wrapper& runtimeApks, jstring_array_wrapper& appDirs) +BasicAndroidSystem::setup_app_library_directories (jstring_array_wrapper& runtimeApks, jstring_array_wrapper& appDirs, bool have_split_apks) { if (!is_embedded_dso_mode_enabled ()) { log_info (LOG_DEFAULT, "Setting up for DSO lookup in app data directories"); @@ -44,7 +44,7 @@ BasicAndroidSystem::setup_app_library_directories (jstring_array_wrapper& runtim unsigned short built_for_cpu = 0, running_on_cpu = 0; unsigned char is64bit = 0; _monodroid_detect_cpu_and_architecture (&built_for_cpu, &running_on_cpu, &is64bit); - setup_apk_directories (running_on_cpu, runtimeApks); + setup_apk_directories (running_on_cpu, runtimeApks, have_split_apks); } } @@ -59,20 +59,36 @@ BasicAndroidSystem::for_each_apk (jstring_array_wrapper &runtimeApks, ForEachApk } } -void -BasicAndroidSystem::add_apk_libdir (const char *apk, size_t index, [[maybe_unused]] size_t apk_count, void *user_data) +force_inline void +BasicAndroidSystem::add_apk_libdir (const char *apk, size_t &index, const char *abi) noexcept { - abort_if_invalid_pointer_argument (user_data); abort_unless (index < app_lib_directories_size, "Index out of range"); - app_lib_directories [index] = utils.string_concat (apk, "!/lib/", static_cast(user_data)); + app_lib_directories [index] = utils.string_concat (apk, "!/lib/", abi); log_debug (LOG_ASSEMBLY, "Added APK DSO lookup location: %s", app_lib_directories[index]); + index++; } -void -BasicAndroidSystem::setup_apk_directories (unsigned short running_on_cpu, jstring_array_wrapper &runtimeApks) +force_inline void +BasicAndroidSystem::setup_apk_directories (unsigned short running_on_cpu, jstring_array_wrapper &runtimeApks, bool have_split_apks) noexcept { - // Man, the cast is ugly... - for_each_apk (runtimeApks, &BasicAndroidSystem::add_apk_libdir, const_cast (static_cast (android_abi_names [running_on_cpu]))); + const char *abi = android_abi_names [running_on_cpu]; + size_t number_of_added_directories = 0; + + for (size_t i = 0; i < runtimeApks.get_length (); ++i) { + jstring_wrapper &e = runtimeApks [i]; + const char *apk = e.get_cstr (); + + if (have_split_apks) { + if (utils.ends_with (apk, SharedConstants::split_config_abi_apk_name)) { + add_apk_libdir (apk, number_of_added_directories, abi); + break; + } + } else { + add_apk_libdir (apk, number_of_added_directories, abi); + } + } + + app_lib_directories_size = number_of_added_directories; } char* diff --git a/src/monodroid/jni/basic-android-system.hh b/src/monodroid/jni/basic-android-system.hh index 42f17181eb6..a12ba47fa1d 100644 --- a/src/monodroid/jni/basic-android-system.hh +++ b/src/monodroid/jni/basic-android-system.hh @@ -58,8 +58,7 @@ namespace xamarin::android::internal static const char* get_built_for_abi_name (); public: - void setup_app_library_directories (jstring_array_wrapper& runtimeApks, jstring_array_wrapper& appDirs); - void setup_apk_directories (unsigned short running_on_cpu, jstring_array_wrapper &runtimeApks); + void setup_app_library_directories (jstring_array_wrapper& runtimeApks, jstring_array_wrapper& appDirs, bool have_split_apks); const char* get_override_dir (size_t index) const { @@ -105,10 +104,11 @@ namespace xamarin::android::internal } protected: - void add_apk_libdir (const char *apk, size_t index, size_t apk_count, void *user_data); void for_each_apk (jstring_array_wrapper &runtimeApks, ForEachApkHandler handler, void *user_data); private: + void add_apk_libdir (const char *apk, size_t &index, const char *abi) noexcept; + void setup_apk_directories (unsigned short running_on_cpu, jstring_array_wrapper &runtimeApks, bool have_split_apks) noexcept; char* determine_primary_override_dir (jstring_wrapper &home); void set_embedded_dso_mode_enabled (bool yesno) noexcept diff --git a/src/monodroid/jni/basic-utilities.hh b/src/monodroid/jni/basic-utilities.hh index 7a4856aac61..630e7eba153 100644 --- a/src/monodroid/jni/basic-utilities.hh +++ b/src/monodroid/jni/basic-utilities.hh @@ -1,6 +1,7 @@ #ifndef __BASIC_UTILITIES_HH #define __BASIC_UTILITIES_HH +#include #include #include #include @@ -109,6 +110,20 @@ namespace xamarin::android return p != nullptr && p [N - 1] == '\0'; } + template + bool ends_with (const char *str, std::array const& end) const noexcept + { + char *p = const_cast (strstr (str, end.data ())); + return p != nullptr && p [N - 1] == '\0'; + } + + template + bool ends_with (const char *str, helper_char_array const& end) const noexcept + { + char *p = const_cast (strstr (str, end.data ())); + return p != nullptr && p [N - 1] == '\0'; + } + template bool ends_with (internal::string_base const& str, const char (&end)[N]) const noexcept { @@ -122,6 +137,32 @@ namespace xamarin::android return memcmp (str.get () + len - end_length, end, end_length) == 0; } + template + bool ends_with (internal::string_base const& str, std::array const& end) const noexcept + { + constexpr size_t end_length = N - 1; + + size_t len = str.length (); + if (XA_UNLIKELY (len < end_length)) { + return false; + } + + return memcmp (str.get () + len - end_length, end.data (), end_length) == 0; + } + + template + bool ends_with (internal::string_base const& str, helper_char_array const& end) const noexcept + { + constexpr size_t end_length = N - 1; + + size_t len = str.length (); + if (XA_UNLIKELY (len < end_length)) { + return false; + } + + return memcmp (str.get () + len - end_length, end.data (), end_length) == 0; + } + template const TChar* find_last (internal::string_base const& str, const char ch) const noexcept { diff --git a/src/monodroid/jni/cpp-util.hh b/src/monodroid/jni/cpp-util.hh index 2ca70bbe605..de1ac16b05f 100644 --- a/src/monodroid/jni/cpp-util.hh +++ b/src/monodroid/jni/cpp-util.hh @@ -1,6 +1,7 @@ #ifndef __CPP_UTIL_HH #define __CPP_UTIL_HH +#include #include #include #include @@ -54,5 +55,73 @@ namespace xamarin::android template using c_unique_ptr = std::unique_ptr>; + + template + struct helper_char_array final + { + constexpr char* data () noexcept + { + return _elems; + } + + constexpr const char* data () const noexcept + { + return _elems; + } + + constexpr char const& operator[] (size_t n) const noexcept + { + return _elems[n]; + } + + constexpr char& operator[] (size_t n) noexcept + { + return _elems[n]; + } + + char _elems[Size]{}; + }; + + // MinGW 9 on the CI build bots has a bug in the gcc compiler which causes builds to fail with: + // + // error G713F753E: ‘constexpr auto xamarin::android::concat_const(const char (&)[Length]...) [with long long unsigned int ...Length = {15, 7, 5}]’ called in a constant expression + // ... + // /usr/lib/gcc/x86_64-w64-mingw32/9.3-win32/include/c++/array:94:12: note: ‘struct std::array’ has no user-provided default constructor + // struct array + // ^~~~~ + // /usr/lib/gcc/x86_64-w64-mingw32/9.3-win32/include/c++/array:110:56: note: and the implicitly-defined constructor does not initialize ‘char std::array::_M_elems [17]’ + // typename _AT_Type::_Type _M_elems; + // ^~~~~~~~ + // + // thus we need to use this workaround here + // +#if defined (__MINGW32__) && __GNUC__ < 10 + template + using char_array = helper_char_array; +#else + template + using char_array = std::array; +#endif + + template + constexpr auto concat_const (const char (&...parts)[Length]) + { + // `parts` being constant string arrays, Length for each of them includes the trailing NUL byte, thus the + // `sizeof... (Length)` part which subtracts the number of template parameters - the amount of NUL bytes so that + // we don't waste space. + constexpr size_t total_length = (... + Length) - sizeof... (Length); + char_array ret; + ret[total_length] = 0; + + size_t i = 0; + for (char const* from : {parts...}) { + for (; *from != '\0'; i++) { + ret[i] = *from++; + } + } + + return ret; + }; + } #endif // !def __CPP_UTIL_HH diff --git a/src/monodroid/jni/debug-app-helper.cc b/src/monodroid/jni/debug-app-helper.cc index b3966e59361..e47daf8bdef 100644 --- a/src/monodroid/jni/debug-app-helper.cc +++ b/src/monodroid/jni/debug-app-helper.cc @@ -69,7 +69,7 @@ JNI_OnLoad ([[maybe_unused]] JavaVM *vm, [[maybe_unused]] void *reserved) JNIEXPORT void JNICALL Java_mono_android_DebugRuntime_init (JNIEnv *env, [[maybe_unused]] jclass klass, jobjectArray runtimeApksJava, - jstring runtimeNativeLibDir, jobjectArray appDirs) + jstring runtimeNativeLibDir, jobjectArray appDirs, jboolean haveSplitApks) { jstring_array_wrapper applicationDirs (env, appDirs); jstring_array_wrapper runtimeApks (env, runtimeApksJava); @@ -77,7 +77,7 @@ Java_mono_android_DebugRuntime_init (JNIEnv *env, [[maybe_unused]] jclass klass, androidSystem.detect_embedded_dso_mode (applicationDirs); androidSystem.set_primary_override_dir (applicationDirs [0]); androidSystem.set_override_dir (0, androidSystem.get_primary_override_dir ()); - androidSystem.setup_app_library_directories (runtimeApks, applicationDirs); + androidSystem.setup_app_library_directories (runtimeApks, applicationDirs, haveSplitApks); jstring_wrapper jstr (env); diff --git a/src/monodroid/jni/debug-app-helper.hh b/src/monodroid/jni/debug-app-helper.hh index 817c7636afd..209f1baba30 100644 --- a/src/monodroid/jni/debug-app-helper.hh +++ b/src/monodroid/jni/debug-app-helper.hh @@ -12,6 +12,6 @@ extern "C" { * Signature: ([Ljava/lang/String;Ljava/lang/String;[Ljava/lang/String);[Ljava/lang/String);Ljava/lang/String;IZ)V */ JNIEXPORT void JNICALL Java_mono_android_DebugRuntime_init - (JNIEnv *, jclass, jobjectArray, jstring, jobjectArray); + (JNIEnv *, jclass, jobjectArray, jstring, jobjectArray, jboolean); } #endif // _Included_mono_android_DebugRuntime diff --git a/src/monodroid/jni/embedded-assemblies-zip.cc b/src/monodroid/jni/embedded-assemblies-zip.cc index 4c58d4ad545..b0f581b5b3d 100644 --- a/src/monodroid/jni/embedded-assemblies-zip.cc +++ b/src/monodroid/jni/embedded-assemblies-zip.cc @@ -2,6 +2,7 @@ #include #include #include +#include #include #include @@ -34,48 +35,60 @@ EmbeddedAssemblies::is_debug_file (dynamic_local_string const ; } -void -EmbeddedAssemblies::zip_load_entries (int fd, const char *apk_name, [[maybe_unused]] monodroid_should_register should_register) +force_inline bool +EmbeddedAssemblies::zip_load_entry_common (size_t entry_index, std::vector const& buf, dynamic_local_string &entry_name, ZipEntryLoadState &state) noexcept { - uint32_t cd_offset; - uint32_t cd_size; - uint16_t cd_entries; + entry_name.clear (); + + bool result = zip_read_entry_info (buf, entry_name, state); - if (!zip_read_cd_info (fd, cd_offset, cd_size, cd_entries)) { - log_fatal (LOG_ASSEMBLY, "Failed to read the EOCD record from APK file %s", apk_name); - exit (FATAL_EXIT_NO_ASSEMBLIES); - } #ifdef DEBUG - log_info (LOG_ASSEMBLY, "Central directory offset: %u", cd_offset); - log_info (LOG_ASSEMBLY, "Central directory size: %u", cd_size); - log_info (LOG_ASSEMBLY, "Central directory entries: %u", cd_entries); + log_info (LOG_ASSEMBLY, "%s entry: %s", state.apk_name, entry_name.get () == nullptr ? "unknown" : entry_name.get ()); #endif - off_t retval = ::lseek (fd, static_cast(cd_offset), SEEK_SET); - if (retval < 0) { - log_fatal (LOG_ASSEMBLY, "Failed to seek to central directory position in the APK file %s. %s (result: %d; errno: %d)", apk_name, std::strerror (errno), retval, errno); + if (!result || entry_name.empty ()) { + log_fatal (LOG_ASSEMBLY, "Failed to read Central Directory info for entry %u in APK file %s", entry_index, state.apk_name); exit (FATAL_EXIT_NO_ASSEMBLIES); } - std::vector buf (cd_size); - const char *prefix = get_assemblies_prefix (); - uint32_t prefix_len = get_assemblies_prefix_length (); - size_t buf_offset = 0; - uint16_t compression_method; - uint32_t local_header_offset; - uint32_t data_offset; - uint32_t file_size; - - ssize_t nread = read (fd, buf.data (), static_cast(buf.size ())); - if (static_cast(nread) != cd_size) { - log_fatal (LOG_ASSEMBLY, "Failed to read Central Directory from the APK archive %s. %s (nread: %d; errno: %d)", apk_name, std::strerror (errno), nread, errno); + if (!zip_adjust_data_offset (state.apk_fd, state)) { + log_fatal (LOG_ASSEMBLY, "Failed to adjust data start offset for entry %u in APK file %s", entry_index, state.apk_name); exit (FATAL_EXIT_NO_ASSEMBLIES); } +#ifdef DEBUG + log_info (LOG_ASSEMBLY, " ZIP: local header offset: %u; data offset: %u; file size: %u", state.local_header_offset, state.data_offset, state.file_size); +#endif + if (state.compression_method != 0) { + return false; + } + + if (entry_name.get ()[0] != state.prefix[0] || strncmp (state.prefix, entry_name.get (), state.prefix_len) != 0) { + return false; + } - dynamic_local_string entry_name; #if defined (NET6) - bool runtime_config_blob_found = false; + if (application_config.have_runtime_config_blob && !runtime_config_blob_found) { + if (utils.ends_with (entry_name, SharedConstants::RUNTIME_CONFIG_BLOB_NAME)) { + runtime_config_blob_found = true; + runtime_config_blob_mmap = md_mmap_apk_file (state.apk_fd, state.data_offset, state.file_size, entry_name.get ()); + return false; + } + } #endif // def NET6 + // assemblies must be 4-byte aligned, or Bad Things happen + if ((state.data_offset & 0x3) != 0) { + log_fatal (LOG_ASSEMBLY, "Assembly '%s' is located at bad offset %lu within the .apk\n", entry_name.get (), state.data_offset); + log_fatal (LOG_ASSEMBLY, "You MUST run `zipalign` on %s\n", strrchr (state.apk_name, '/') + 1); + exit (FATAL_EXIT_MISSING_ZIPALIGN); + } + + return true; +} + +force_inline void +EmbeddedAssemblies::zip_load_individual_assembly_entries (std::vector const& buf, uint32_t num_entries, [[maybe_unused]] monodroid_should_register should_register, ZipEntryLoadState &state) noexcept +{ + dynamic_local_string entry_name; bool bundled_assemblies_slow_path = bundled_assembly_index >= application_config.number_of_assemblies_in_apk; uint32_t max_assembly_name_size = application_config.bundled_assembly_name_width - 1; @@ -87,47 +100,10 @@ EmbeddedAssemblies::zip_load_entries (int fd, const char *apk_name, [[maybe_unus // However, clang-tidy can't know that the value is owned by Mono and we must not free it, thus the suppression. // // NOLINTNEXTLINE(clang-analyzer-unix.Malloc) - for (size_t i = 0; i < cd_entries; i++) { - entry_name.clear (); - - bool result = zip_read_entry_info (buf, buf_offset, compression_method, local_header_offset, file_size, entry_name); - -#ifdef DEBUG - log_info (LOG_ASSEMBLY, "%s entry: %s", apk_name, entry_name.get () == nullptr ? "unknown" : entry_name.get ()); -#endif - if (!result || entry_name.empty ()) { - log_fatal (LOG_ASSEMBLY, "Failed to read Central Directory info for entry %u in APK file %s", i, apk_name); - exit (FATAL_EXIT_NO_ASSEMBLIES); - } - - if (!zip_adjust_data_offset (fd, local_header_offset, data_offset)) { - log_fatal (LOG_ASSEMBLY, "Failed to adjust data start offset for entry %u in APK file %s", i, apk_name); - exit (FATAL_EXIT_NO_ASSEMBLIES); - } -#ifdef DEBUG - log_info (LOG_ASSEMBLY, " ZIP: local header offset: %u; data offset: %u; file size: %u", local_header_offset, data_offset, file_size); -#endif - if (compression_method != 0) + for (size_t i = 0; i < num_entries; i++) { + bool interesting_entry = zip_load_entry_common (i, buf, entry_name, state); + if (!interesting_entry) { continue; - - if (entry_name.get ()[0] != prefix[0] || strncmp (prefix, entry_name.get (), prefix_len) != 0) - continue; - -#if defined (NET6) - if (application_config.have_runtime_config_blob && !runtime_config_blob_found) { - if (utils.ends_with (entry_name, SharedConstants::RUNTIME_CONFIG_BLOB_NAME)) { - runtime_config_blob_found = true; - runtime_config_blob_mmap = md_mmap_apk_file (fd, data_offset, file_size, entry_name.get ()); - continue; - } - } -#endif // def NET6 - - // assemblies must be 4-byte aligned, or Bad Things happen - if ((data_offset & 0x3) != 0) { - log_fatal (LOG_ASSEMBLY, "Assembly '%s' is located at bad offset %lu within the .apk\n", entry_name.get (), data_offset); - log_fatal (LOG_ASSEMBLY, "You MUST run `zipalign` on %s\n", strrchr (apk_name, '/') + 1); - exit (FATAL_EXIT_MISSING_ZIPALIGN); } #if defined (DEBUG) @@ -144,7 +120,7 @@ EmbeddedAssemblies::zip_load_entries (int fd, const char *apk_name, [[maybe_unus } bundled_debug_data->emplace_back (); - set_debug_entry_data (bundled_debug_data->back (), fd, data_offset, file_size, prefix_len, max_assembly_name_size, entry_name); + set_debug_entry_data (bundled_debug_data->back (), state.apk_fd, state.data_offset, state.file_size, state.prefix_len, max_assembly_name_size, entry_name); continue; } @@ -154,14 +130,14 @@ EmbeddedAssemblies::zip_load_entries (int fd, const char *apk_name, [[maybe_unus // Remove '.config' suffix *strrchr (assembly_name, '.') = '\0'; - md_mmap_info map_info = md_mmap_apk_file (fd, data_offset, file_size, entry_name.get ()); + md_mmap_info map_info = md_mmap_apk_file (state.apk_fd, state.data_offset, state.file_size, entry_name.get ()); mono_register_config_for_assembly (assembly_name, (const char*)map_info.area); continue; } #endif // ndef NET6 - if (!utils.ends_with (entry_name, ".dll")) + if (!utils.ends_with (entry_name, SharedConstants::DLL_EXTENSION)) continue; #if defined (DEBUG) @@ -181,17 +157,164 @@ EmbeddedAssemblies::zip_load_entries (int fd, const char *apk_name, [[maybe_unus extra_bundled_assemblies->emplace_back (); // means we need to allocate memory to store the entry name, only the entries pre-allocated during // build have valid pointer to the name storage area - set_entry_data (extra_bundled_assemblies->back (), fd, data_offset, file_size, prefix_len, max_assembly_name_size, entry_name); + set_entry_data (extra_bundled_assemblies->back (), state.apk_fd, state.data_offset, state.file_size, state.prefix_len, max_assembly_name_size, entry_name); continue; } - set_assembly_entry_data (bundled_assemblies [bundled_assembly_index], fd, data_offset, file_size, prefix_len, max_assembly_name_size, entry_name); + set_assembly_entry_data (bundled_assemblies [bundled_assembly_index], state.apk_fd, state.data_offset, state.file_size, state.prefix_len, max_assembly_name_size, entry_name); bundled_assembly_index++; + number_of_found_assemblies = bundled_assembly_index; } have_and_want_debug_symbols = register_debug_symbols && bundled_debug_data != nullptr; } +force_inline void +EmbeddedAssemblies::map_assembly_store (dynamic_local_string const& entry_name, ZipEntryLoadState &state) noexcept +{ + if (number_of_mapped_assembly_stores >= application_config.number_of_assembly_store_files) { + log_fatal (LOG_ASSEMBLY, "Too many assembly stores. Expected at most %u", application_config.number_of_assembly_store_files); + abort (); + } + + md_mmap_info assembly_store_map = md_mmap_apk_file (state.apk_fd, state.data_offset, state.file_size, entry_name.get ()); + auto header = static_cast(assembly_store_map.area); + + if (header->magic != ASSEMBLY_STORE_MAGIC) { + log_fatal (LOG_ASSEMBLY, "Assembly store '%s' is not a valid Xamarin.Android assembly store file", entry_name.get ()); + abort (); + } + + if (header->version > ASSEMBLY_STORE_FORMAT_VERSION) { + log_fatal (LOG_ASSEMBLY, "Assembly store '%s' uses format v%u which is not understood by this version of Xamarin.Android", entry_name.get (), header->version); + abort (); + } + + if (header->store_id >= application_config.number_of_assembly_store_files) { + log_fatal ( + LOG_ASSEMBLY, + "Assembly store '%s' index %u exceeds the number of stores known at application build time, %u", + entry_name.get (), + header->store_id, + application_config.number_of_assembly_store_files + ); + abort (); + } + + AssemblyStoreRuntimeData &rd = assembly_stores[header->store_id]; + if (rd.data_start != nullptr) { + log_fatal (LOG_ASSEMBLY, "Assembly store '%s' has a duplicate ID (%u)", entry_name.get (), header->store_id); + abort (); + } + + constexpr size_t header_size = sizeof(AssemblyStoreHeader); + + rd.data_start = static_cast(assembly_store_map.area); + rd.assembly_count = header->local_entry_count; + rd.assemblies = reinterpret_cast(rd.data_start + header_size); + + number_of_found_assemblies += rd.assembly_count; + + if (header->store_id == 0) { + constexpr size_t bundled_assembly_size = sizeof(AssemblyStoreAssemblyDescriptor); + constexpr size_t hash_entry_size = sizeof(AssemblyStoreHashEntry); + + index_assembly_store_header = header; + + size_t bytes_before_hashes = header_size + (bundled_assembly_size * header->local_entry_count); + if constexpr (std::is_same_v) { + assembly_store_hashes = reinterpret_cast(rd.data_start + bytes_before_hashes + (hash_entry_size * header->global_entry_count)); + } else { + assembly_store_hashes = reinterpret_cast(rd.data_start + bytes_before_hashes); + } + } + + number_of_mapped_assembly_stores++; + have_and_want_debug_symbols = register_debug_symbols; +} + +force_inline void +EmbeddedAssemblies::zip_load_assembly_store_entries (std::vector const& buf, uint32_t num_entries, ZipEntryLoadState &state) noexcept +{ + if (all_required_zip_entries_found ()) { + return; + } + + dynamic_local_string entry_name; + bool common_assembly_store_found = false; + bool arch_assembly_store_found = false; + + log_debug (LOG_ASSEMBLY, "Looking for assembly stores in APK (common: '%s'; arch-specific: '%s')", assembly_store_common_file_name.data (), assembly_store_arch_file_name.data ()); + for (size_t i = 0; i < num_entries; i++) { + if (all_required_zip_entries_found ()) { + need_to_scan_more_apks = false; + break; + } + + bool interesting_entry = zip_load_entry_common (i, buf, entry_name, state); + if (!interesting_entry) { + continue; + } + + if (!common_assembly_store_found && utils.ends_with (entry_name, assembly_store_common_file_name)) { + common_assembly_store_found = true; + map_assembly_store (entry_name, state); + } + + if (!arch_assembly_store_found && utils.ends_with (entry_name, assembly_store_arch_file_name)) { + arch_assembly_store_found = true; + map_assembly_store (entry_name, state); + } + } +} + +void +EmbeddedAssemblies::zip_load_entries (int fd, const char *apk_name, [[maybe_unused]] monodroid_should_register should_register) +{ + uint32_t cd_offset; + uint32_t cd_size; + uint16_t cd_entries; + + if (!zip_read_cd_info (fd, cd_offset, cd_size, cd_entries)) { + log_fatal (LOG_ASSEMBLY, "Failed to read the EOCD record from APK file %s", apk_name); + exit (FATAL_EXIT_NO_ASSEMBLIES); + } +#ifdef DEBUG + log_info (LOG_ASSEMBLY, "Central directory offset: %u", cd_offset); + log_info (LOG_ASSEMBLY, "Central directory size: %u", cd_size); + log_info (LOG_ASSEMBLY, "Central directory entries: %u", cd_entries); +#endif + off_t retval = ::lseek (fd, static_cast(cd_offset), SEEK_SET); + if (retval < 0) { + log_fatal (LOG_ASSEMBLY, "Failed to seek to central directory position in the APK file %s. %s (result: %d; errno: %d)", apk_name, std::strerror (errno), retval, errno); + exit (FATAL_EXIT_NO_ASSEMBLIES); + } + + std::vector buf (cd_size); + ZipEntryLoadState state { + .apk_fd = fd, + .apk_name = apk_name, + .prefix = get_assemblies_prefix (), + .prefix_len = get_assemblies_prefix_length (), + .buf_offset = 0, + .local_header_offset = 0, + .data_offset = 0, + .file_size = 0, + }; + + ssize_t nread = read (fd, buf.data (), static_cast(buf.size ())); + if (static_cast(nread) != cd_size) { + log_fatal (LOG_ASSEMBLY, "Failed to read Central Directory from the APK archive %s. %s (nread: %d; errno: %d)", apk_name, std::strerror (errno), nread, errno); + exit (FATAL_EXIT_NO_ASSEMBLIES); + } + + if (application_config.have_assembly_store) { + zip_load_assembly_store_entries (buf, cd_entries, state); + } else { + zip_load_individual_assembly_entries (buf, cd_entries, should_register, state); + } +} + template force_inline void EmbeddedAssemblies::set_entry_data (XamarinAndroidBundledAssembly &entry, int apk_fd, uint32_t data_offset, uint32_t data_size, uint32_t prefix_len, uint32_t max_name_size, dynamic_local_string const& entry_name) noexcept @@ -288,14 +411,14 @@ EmbeddedAssemblies::zip_read_cd_info (int fd, uint32_t& cd_offset, uint32_t& cd_ } bool -EmbeddedAssemblies::zip_adjust_data_offset (int fd, size_t local_header_offset, uint32_t &data_start_offset) +EmbeddedAssemblies::zip_adjust_data_offset (int fd, ZipEntryLoadState &state) { static constexpr size_t LH_FILE_NAME_LENGTH_OFFSET = 26; static constexpr size_t LH_EXTRA_LENGTH_OFFSET = 28; - off_t result = ::lseek (fd, static_cast(local_header_offset), SEEK_SET); + off_t result = ::lseek (fd, static_cast(state.local_header_offset), SEEK_SET); if (result < 0) { - log_error (LOG_ASSEMBLY, "Failed to seek to archive entry local header at offset %u. %s (result: %d; errno: %d)", local_header_offset, result, errno); + log_error (LOG_ASSEMBLY, "Failed to seek to archive entry local header at offset %u. %s (result: %d; errno: %d)", state.local_header_offset, result, errno); return false; } @@ -304,36 +427,36 @@ EmbeddedAssemblies::zip_adjust_data_offset (int fd, size_t local_header_offset, ssize_t nread = ::read (fd, local_header.data (), local_header.size ()); if (nread < 0 || nread != ZIP_LOCAL_LEN) { - log_error (LOG_ASSEMBLY, "Failed to read local header at offset %u: %s (nread: %d; errno: %d)", local_header_offset, std::strerror (errno), nread, errno); + log_error (LOG_ASSEMBLY, "Failed to read local header at offset %u: %s (nread: %d; errno: %d)", state.local_header_offset, std::strerror (errno), nread, errno); return false; } size_t index = 0; if (!zip_read_field (local_header, index, signature)) { - log_error (LOG_ASSEMBLY, "Failed to read Local Header entry signature at offset %u", local_header_offset); + log_error (LOG_ASSEMBLY, "Failed to read Local Header entry signature at offset %u", state.local_header_offset); return false; } if (memcmp (signature.data (), ZIP_LOCAL_MAGIC, signature.size ()) != 0) { - log_error (LOG_ASSEMBLY, "Invalid Local Header entry signature at offset %u", local_header_offset); + log_error (LOG_ASSEMBLY, "Invalid Local Header entry signature at offset %u", state.local_header_offset); return false; } uint16_t file_name_length; index = LH_FILE_NAME_LENGTH_OFFSET; if (!zip_read_field (local_header, index, file_name_length)) { - log_error (LOG_ASSEMBLY, "Failed to read Local Header 'file name length' field at offset %u", (local_header_offset + index)); + log_error (LOG_ASSEMBLY, "Failed to read Local Header 'file name length' field at offset %u", (state.local_header_offset + index)); return false; } uint16_t extra_field_length; index = LH_EXTRA_LENGTH_OFFSET; if (!zip_read_field (local_header, index, extra_field_length)) { - log_error (LOG_ASSEMBLY, "Failed to read Local Header 'extra field length' field at offset %u", (local_header_offset + index)); + log_error (LOG_ASSEMBLY, "Failed to read Local Header 'extra field length' field at offset %u", (state.local_header_offset + index)); return false; } - data_start_offset = static_cast(local_header_offset) + file_name_length + extra_field_length + local_header.size (); + state.data_offset = static_cast(state.local_header_offset) + file_name_length + extra_field_length + local_header.size (); return true; } @@ -433,7 +556,7 @@ EmbeddedAssemblies::zip_read_field (T const& buf, size_t index, size_t count, dy } bool -EmbeddedAssemblies::zip_read_entry_info (std::vector const& buf, size_t& buf_offset, uint16_t& compression_method, uint32_t& local_header_offset, uint32_t& file_size, dynamic_local_string& file_name) +EmbeddedAssemblies::zip_read_entry_info (std::vector const& buf, dynamic_local_string& file_name, ZipEntryLoadState &state) { constexpr size_t CD_COMPRESSION_METHOD_OFFSET = 10; constexpr size_t CD_UNCOMPRESSED_SIZE_OFFSET = 24; @@ -442,7 +565,7 @@ EmbeddedAssemblies::zip_read_entry_info (std::vector const& buf, size_t constexpr size_t CD_LOCAL_HEADER_POS_OFFSET = 42; constexpr size_t CD_COMMENT_LENGTH_OFFSET = 32; - size_t index = buf_offset; + size_t index = state.buf_offset; zip_ensure_valid_params (buf, index, ZIP_CENTRAL_LEN); std::array signature; @@ -456,45 +579,45 @@ EmbeddedAssemblies::zip_read_entry_info (std::vector const& buf, size_t return false; } - index = buf_offset + CD_COMPRESSION_METHOD_OFFSET; - if (!zip_read_field (buf, index, compression_method)) { + index = state.buf_offset + CD_COMPRESSION_METHOD_OFFSET; + if (!zip_read_field (buf, index, state.compression_method)) { log_error (LOG_ASSEMBLY, "Failed to read Central Directory entry 'compression method' field"); return false; } - index = buf_offset + CD_UNCOMPRESSED_SIZE_OFFSET;; - if (!zip_read_field (buf, index, file_size)) { + index = state.buf_offset + CD_UNCOMPRESSED_SIZE_OFFSET;; + if (!zip_read_field (buf, index, state.file_size)) { log_error (LOG_ASSEMBLY, "Failed to read Central Directory entry 'uncompressed size' field"); return false; } uint16_t file_name_length; - index = buf_offset + CD_FILENAME_LENGTH_OFFSET; + index = state.buf_offset + CD_FILENAME_LENGTH_OFFSET; if (!zip_read_field (buf, index, file_name_length)) { log_error (LOG_ASSEMBLY, "Failed to read Central Directory entry 'file name length' field"); return false; } uint16_t extra_field_length; - index = buf_offset + CD_EXTRA_LENGTH_OFFSET; + index = state.buf_offset + CD_EXTRA_LENGTH_OFFSET; if (!zip_read_field (buf, index, extra_field_length)) { log_error (LOG_ASSEMBLY, "Failed to read Central Directory entry 'extra field length' field"); return false; } uint16_t comment_length; - index = buf_offset + CD_COMMENT_LENGTH_OFFSET; + index = state.buf_offset + CD_COMMENT_LENGTH_OFFSET; if (!zip_read_field (buf, index, comment_length)) { log_error (LOG_ASSEMBLY, "Failed to read Central Directory entry 'file comment length' field"); return false; } - index = buf_offset + CD_LOCAL_HEADER_POS_OFFSET; - if (!zip_read_field (buf, index, local_header_offset)) { + index = state.buf_offset + CD_LOCAL_HEADER_POS_OFFSET; + if (!zip_read_field (buf, index, state.local_header_offset)) { log_error (LOG_ASSEMBLY, "Failed to read Central Directory entry 'relative offset of local header' field"); return false; } - index += sizeof(local_header_offset); + index += sizeof(state.local_header_offset); if (file_name_length == 0) { file_name.clear (); @@ -503,6 +626,6 @@ EmbeddedAssemblies::zip_read_entry_info (std::vector const& buf, size_t return false; } - buf_offset += ZIP_CENTRAL_LEN + file_name_length + extra_field_length + comment_length; + state.buf_offset += ZIP_CENTRAL_LEN + file_name_length + extra_field_length + comment_length; return true; } diff --git a/src/monodroid/jni/embedded-assemblies.cc b/src/monodroid/jni/embedded-assemblies.cc index f1c21207f33..c8099b2256a 100644 --- a/src/monodroid/jni/embedded-assemblies.cc +++ b/src/monodroid/jni/embedded-assemblies.cc @@ -1,5 +1,8 @@ #include +#if !defined (__MINGW32__) || (defined (__MINGW32__) && __GNUC__ >= 10) +#include +#endif #include #include #include @@ -66,10 +69,10 @@ void EmbeddedAssemblies::set_assemblies_prefix (const char *prefix) } force_inline void -EmbeddedAssemblies::get_assembly_data (XamarinAndroidBundledAssembly const& e, uint8_t*& assembly_data, uint32_t& assembly_data_size) +EmbeddedAssemblies::get_assembly_data (uint8_t *data, uint32_t data_size, [[maybe_unused]] const char *name, uint8_t*& assembly_data, uint32_t& assembly_data_size) noexcept { #if defined (ANDROID) && defined (HAVE_LZ4) && defined (RELEASE) - auto header = reinterpret_cast(e.data); + auto header = reinterpret_cast(data); if (header->magic == COMPRESSED_DATA_MAGIC) { if (XA_UNLIKELY (compressed_assemblies.descriptors == nullptr)) { log_fatal (LOG_ASSEMBLY, "Compressed assembly found but no descriptor defined"); @@ -81,7 +84,7 @@ EmbeddedAssemblies::get_assembly_data (XamarinAndroidBundledAssembly const& e, u } CompressedAssemblyDescriptor &cad = compressed_assemblies.descriptors[header->descriptor_index]; - assembly_data_size = e.data_size - sizeof(CompressedAssemblyHeader); + assembly_data_size = data_size - sizeof(CompressedAssemblyHeader); if (!cad.loaded) { if (XA_UNLIKELY (cad.data == nullptr)) { log_fatal (LOG_ASSEMBLY, "Invalid compressed assembly descriptor at %u: no data", header->descriptor_index); @@ -96,29 +99,29 @@ EmbeddedAssemblies::get_assembly_data (XamarinAndroidBundledAssembly const& e, u if (header->uncompressed_length != cad.uncompressed_file_size) { if (header->uncompressed_length > cad.uncompressed_file_size) { - log_fatal (LOG_ASSEMBLY, "Compressed assembly '%s' is larger than when the application was built (expected at most %u, got %u). Assemblies don't grow just like that!", e.name, cad.uncompressed_file_size, header->uncompressed_length); + log_fatal (LOG_ASSEMBLY, "Compressed assembly '%s' is larger than when the application was built (expected at most %u, got %u). Assemblies don't grow just like that!", name, cad.uncompressed_file_size, header->uncompressed_length); exit (FATAL_EXIT_MISSING_ASSEMBLY); } else { - log_debug (LOG_ASSEMBLY, "Compressed assembly '%s' is smaller than when the application was built. Adjusting accordingly.", e.name); + log_debug (LOG_ASSEMBLY, "Compressed assembly '%s' is smaller than when the application was built. Adjusting accordingly.", name); } cad.uncompressed_file_size = header->uncompressed_length; } - const char *data_start = reinterpret_cast(e.data + sizeof(CompressedAssemblyHeader)); + const char *data_start = reinterpret_cast(data + sizeof(CompressedAssemblyHeader)); int ret = LZ4_decompress_safe (data_start, reinterpret_cast(cad.data), static_cast(assembly_data_size), static_cast(cad.uncompressed_file_size)); if (XA_UNLIKELY (log_timing)) { decompress_time.mark_end (); - TIMING_LOG_INFO (decompress_time, "%s LZ4 decompression time", e.name); + TIMING_LOG_INFO (decompress_time, "%s LZ4 decompression time", name); } if (ret < 0) { - log_fatal (LOG_ASSEMBLY, "Decompression of assembly %s failed with code %d", e.name, ret); + log_fatal (LOG_ASSEMBLY, "Decompression of assembly %s failed with code %d", name, ret); exit (FATAL_EXIT_MISSING_ASSEMBLY); } if (static_cast(ret) != cad.uncompressed_file_size) { - log_debug (LOG_ASSEMBLY, "Decompression of assembly %s yielded a different size (expected %lu, got %u)", e.name, cad.uncompressed_file_size, static_cast(ret)); + log_debug (LOG_ASSEMBLY, "Decompression of assembly %s yielded a different size (expected %lu, got %u)", name, cad.uncompressed_file_size, static_cast(ret)); exit (FATAL_EXIT_MISSING_ASSEMBLY); } cad.loaded = true; @@ -128,11 +131,23 @@ EmbeddedAssemblies::get_assembly_data (XamarinAndroidBundledAssembly const& e, u } else #endif { - assembly_data = e.data; - assembly_data_size = e.data_size; + assembly_data = data; + assembly_data_size = data_size; } } +force_inline void +EmbeddedAssemblies::get_assembly_data (XamarinAndroidBundledAssembly const& e, uint8_t*& assembly_data, uint32_t& assembly_data_size) noexcept +{ + get_assembly_data (e.data, e.data_size, e.name, assembly_data, assembly_data_size); +} + +force_inline void +EmbeddedAssemblies::get_assembly_data (AssemblyStoreSingleAssemblyRuntimeData const& e, uint8_t*& assembly_data, uint32_t& assembly_data_size) noexcept +{ + get_assembly_data (e.image_data, e.descriptor->data_size, "", assembly_data, assembly_data_size); +} + #if defined (NET6) MonoAssembly* EmbeddedAssemblies::open_from_bundles (MonoAssemblyName* aname, MonoAssemblyLoadContextGCHandle alc_gchandle, [[maybe_unused]] MonoError *error) @@ -293,31 +308,19 @@ EmbeddedAssemblies::load_bundled_assembly ( return a; } -MonoAssembly* -EmbeddedAssemblies::open_from_bundles (MonoAssemblyName* aname, std::function loader, bool ref_only) +force_inline MonoAssembly* +EmbeddedAssemblies::individual_assemblies_open_from_bundles (dynamic_local_string& name, std::function loader, bool ref_only) noexcept { - const char *culture = mono_assembly_name_get_culture (aname); - const char *asmname = mono_assembly_name_get_name (aname); - - constexpr char path_separator[] = "/"; - dynamic_local_string name; - if (culture != nullptr && *culture != '\0') { - name.append_c (culture); - name.append (path_separator); + if (!utils.ends_with (name, SharedConstants::DLL_EXTENSION)) { + name.append (SharedConstants::DLL_EXTENSION); } - name.append_c (asmname); - constexpr char dll_extension[] = ".dll"; - if (!utils.ends_with (name, dll_extension)) { - name.append (dll_extension); - } - - log_debug (LOG_ASSEMBLY, "open_from_bundles: looking for bundled name: '%s'", name.get ()); + log_debug (LOG_ASSEMBLY, "individual_assemblies_open_from_bundles: looking for bundled name: '%s'", name.get ()); dynamic_local_string abi_name; abi_name .assign_c (BasicAndroidSystem::get_built_for_abi_name ()) - .append (path_separator) + .append (zip_path_separator) .append (name); MonoAssembly *a = nullptr; @@ -338,10 +341,172 @@ EmbeddedAssemblies::open_from_bundles (MonoAssemblyName* aname, std::function= 10) + hash_t entry_hash; + const AssemblyStoreHashEntry *ret; + + while (entry_count > 0) { + ret = entries + (entry_count / 2); + if constexpr (std::is_same_v) { + entry_hash = ret->hash64; + } else { + entry_hash = ret->hash32; + } + + std::strong_ordering result = hash <=> entry_hash; + + if (result < 0) { + entry_count /= 2; + } else if (result > 0) { + entries = ret + 1; + entry_count -= entry_count / 2 + 1; + } else { + return ret; + } + } +#endif // ndef __MINGW32__ || (def __MINGW32__ && __GNUC__ >= 10) + + return nullptr; +} + +force_inline MonoAssembly* +EmbeddedAssemblies::assembly_store_open_from_bundles (dynamic_local_string& name, std::function loader, bool ref_only) noexcept +{ + size_t len = name.length (); + bool have_dll_ext = utils.ends_with (name, SharedConstants::DLL_EXTENSION); + + if (have_dll_ext) { + len -= sizeof(SharedConstants::DLL_EXTENSION) - 1; + } + + hash_t name_hash = xxhash::hash (name.get (), len); + log_debug (LOG_ASSEMBLY, "assembly_store_open_from_bundles: looking for bundled name: '%s' (hash 0x%zx)", name.get (), name_hash); + + const AssemblyStoreHashEntry *hash_entry = find_assembly_store_entry (name_hash, assembly_store_hashes, application_config.number_of_assemblies_in_apk); + if (hash_entry == nullptr) { + log_warn (LOG_ASSEMBLY, "Assembly '%s' (hash 0x%zx) not found", name.get (), name_hash); + return nullptr; + } + + if (hash_entry->mapping_index >= application_config.number_of_assemblies_in_apk) { + log_fatal (LOG_ASSEMBLY, "Invalid assembly index %u, exceeds the maximum index of %u", hash_entry->mapping_index, application_config.number_of_assemblies_in_apk - 1); + abort (); + } + + AssemblyStoreSingleAssemblyRuntimeData &assembly_runtime_info = assembly_store_bundled_assemblies[hash_entry->mapping_index]; + if (assembly_runtime_info.image_data == nullptr) { + if (hash_entry->store_id >= application_config.number_of_assembly_store_files) { + log_fatal (LOG_ASSEMBLY, "Invalid assembly store ID %u, exceeds the maximum of %u", hash_entry->store_id, application_config.number_of_assembly_store_files - 1); + abort (); + } + + AssemblyStoreRuntimeData &rd = assembly_stores[hash_entry->store_id]; + if (hash_entry->local_store_index >= rd.assembly_count) { + log_fatal (LOG_ASSEMBLY, "Invalid index %u into local store assembly descriptor array", hash_entry->local_store_index); + } + + AssemblyStoreAssemblyDescriptor *bba = &rd.assemblies[hash_entry->local_store_index]; + + // The assignments here don't need to be atomic, the value will always be the same, so even if two threads + // arrive here at the same time, nothing bad will happen. + assembly_runtime_info.image_data = rd.data_start + bba->data_offset; + assembly_runtime_info.descriptor = bba; + + if (bba->debug_data_offset != 0) { + assembly_runtime_info.debug_info_data = rd.data_start + bba->debug_data_offset; + } +#if !defined (NET6) + if (bba->config_data_size != 0) { + assembly_runtime_info.config_data = rd.data_start + bba->config_data_offset; + + // Mono takes ownership of the pointers + mono_register_config_for_assembly ( + utils.string_concat (name.get (), ".dll"), + utils.strdup_new (reinterpret_cast(assembly_runtime_info.config_data)) + ); + } +#endif // NET6 + + log_debug ( + LOG_ASSEMBLY, + "Mapped: image_data == %p; debug_info_data == %p; config_data == %p; descriptor == %p; data size == %u; debug data size == %u; config data size == %u", + assembly_runtime_info.image_data, + assembly_runtime_info.debug_info_data, + assembly_runtime_info.config_data, + assembly_runtime_info.descriptor, + assembly_runtime_info.descriptor->data_size, + assembly_runtime_info.descriptor->debug_data_size, + assembly_runtime_info.descriptor->config_data_size + ); + } + + uint8_t *assembly_data; + uint32_t assembly_data_size; + + if (!have_dll_ext) { + // AOT needs this since Mono will form the DSO name by appending the .so suffix to the assembly name passed to + // functions below and `monodroid_dlopen` uses the `.dll.so` extension to check whether we're being asked to load + // the AOTd code for an assembly. + name.append (SharedConstants::DLL_EXTENSION); + } + + get_assembly_data (assembly_runtime_info, assembly_data, assembly_data_size); + MonoImage *image = loader (assembly_data, assembly_data_size, name.get ()); + if (image == nullptr) { + log_warn (LOG_ASSEMBLY, "Failed to load MonoImage of '%s'", name.get ()); + return nullptr; + } + + if (have_and_want_debug_symbols && assembly_runtime_info.debug_info_data != nullptr) { + log_debug (LOG_ASSEMBLY, "Registering debug data for assembly '%s'", name.get ()); + mono_debug_open_image_from_memory (image, reinterpret_cast (assembly_runtime_info.debug_info_data), static_cast(assembly_runtime_info.descriptor->debug_data_size)); + } + + MonoImageOpenStatus status; + MonoAssembly *a = mono_assembly_load_from_full (image, name.get (), &status, ref_only); + if (a == nullptr) { + return nullptr; + } + +#if !defined (NET6) + mono_config_for_assembly (image); +#endif + return a; +} + +MonoAssembly* +EmbeddedAssemblies::open_from_bundles (MonoAssemblyName* aname, std::function loader, bool ref_only) +{ + const char *culture = mono_assembly_name_get_culture (aname); + const char *asmname = mono_assembly_name_get_name (aname); + + dynamic_local_string name; + if (culture != nullptr && *culture != '\0') { + name.append_c (culture); + name.append (zip_path_separator); + } + name.append_c (asmname); + + MonoAssembly *a; + if (application_config.have_assembly_store) { + a = assembly_store_open_from_bundles (name, loader, ref_only); + } else { + a = individual_assemblies_open_from_bundles (name, loader, ref_only); + } + + if (a == nullptr) { + log_warn (LOG_ASSEMBLY, "open_from_bundles: failed to load assembly %s", name.get ()); + } + + return a; +} + #if defined (NET6) MonoAssembly* EmbeddedAssemblies::open_from_bundles (MonoAssemblyLoadContextGCHandle alc_gchandle, MonoAssemblyName *aname, [[maybe_unused]] char **assemblies_path, [[maybe_unused]] void *user_data, MonoError *error) @@ -1067,11 +1232,11 @@ EmbeddedAssemblies::try_load_typemaps_from_directory (const char *path) size_t EmbeddedAssemblies::register_from (const char *apk_file, monodroid_should_register should_register) { - size_t prev = bundled_assembly_index; + size_t prev = number_of_found_assemblies; gather_bundled_assemblies_from_apk (apk_file, should_register); - log_info (LOG_ASSEMBLY, "Package '%s' contains %i assemblies", apk_file, bundled_assembly_index - prev); + log_info (LOG_ASSEMBLY, "Package '%s' contains %i assemblies", apk_file, number_of_found_assemblies - prev); - return bundled_assembly_index; + return number_of_found_assemblies; } diff --git a/src/monodroid/jni/embedded-assemblies.hh b/src/monodroid/jni/embedded-assemblies.hh index 97f37aef34b..6d65d5aba18 100644 --- a/src/monodroid/jni/embedded-assemblies.hh +++ b/src/monodroid/jni/embedded-assemblies.hh @@ -29,6 +29,8 @@ #include "strings.hh" #include "xamarin-app.hh" #include "cpp-util.hh" +#include "shared-constants.hh" +#include "xxhash.hh" struct TypeMapHeader; @@ -55,6 +57,19 @@ namespace xamarin::android::internal { size_t size; }; + struct ZipEntryLoadState + { + int apk_fd; + const char * const apk_name; + const char * const prefix; + uint32_t prefix_len; + size_t buf_offset; + uint16_t compression_method; + uint32_t local_header_offset; + uint32_t data_offset; + uint32_t file_size; + }; + private: static constexpr char ZIP_CENTRAL_MAGIC[] = "PK\1\2"; static constexpr char ZIP_LOCAL_MAGIC[] = "PK\3\4"; @@ -62,7 +77,15 @@ namespace xamarin::android::internal { static constexpr off_t ZIP_EOCD_LEN = 22; static constexpr off_t ZIP_CENTRAL_LEN = 46; static constexpr off_t ZIP_LOCAL_LEN = 30; - static constexpr char assemblies_prefix[] = "assemblies/"; + static constexpr char assemblies_prefix[] = "assemblies/"; + static constexpr char zip_path_separator[] = "/"; + + static constexpr char assembly_store_prefix[] = "assemblies"; + static constexpr char assembly_store_extension[] = ".blob"; + static constexpr auto assembly_store_common_file_name = concat_const ("/", assembly_store_prefix, assembly_store_extension); + static constexpr auto assembly_store_arch_file_name = concat_const ("/", assembly_store_prefix, ".", SharedConstants::android_abi, assembly_store_extension); + + #if defined (DEBUG) || !defined (ANDROID) static constexpr char override_typemap_entry_name[] = ".__override__"; #endif @@ -112,12 +135,26 @@ namespace xamarin::android::internal { size = static_cast(runtime_config_blob_mmap.size); } - bool have_runtime_config_blob () const + bool have_runtime_config_blob () const noexcept { return application_config.have_runtime_config_blob && runtime_config_blob_mmap.area != nullptr; } #endif + bool keep_scanning () const noexcept + { + return need_to_scan_more_apks; + } + + void ensure_valid_assembly_stores () const noexcept + { + if (!application_config.have_assembly_store) { + return; + } + + abort_unless (index_assembly_store_header != nullptr && assembly_store_hashes != nullptr, "Invalid or incomplete assembly store data"); + } + private: const char* typemap_managed_to_java (MonoType *type, MonoClass *klass, const uint8_t *mvid); MonoReflectionType* typemap_java_to_managed (const char *java_type_name); @@ -127,6 +164,8 @@ namespace xamarin::android::internal { MonoAssembly* open_from_bundles (MonoAssemblyName* aname, MonoAssemblyLoadContextGCHandle alc_gchandle, MonoError *error); #endif // def NET6 MonoAssembly* open_from_bundles (MonoAssemblyName* aname, bool ref_only); + MonoAssembly* individual_assemblies_open_from_bundles (dynamic_local_string& name, std::function loader, bool ref_only) noexcept; + MonoAssembly* assembly_store_open_from_bundles (dynamic_local_string& name, std::function loader, bool ref_only) noexcept; MonoAssembly* open_from_bundles (MonoAssemblyName* aname, std::function loader, bool ref_only); template @@ -158,11 +197,16 @@ namespace xamarin::android::internal { #else // def NET6 static MonoAssembly* open_from_bundles_refonly (MonoAssemblyName *aname, char **assemblies_path, void *user_data); #endif // ndef NET6 - static void get_assembly_data (XamarinAndroidBundledAssembly const& e, uint8_t*& assembly_data, uint32_t& assembly_data_size); + static void get_assembly_data (uint8_t *data, uint32_t data_size, const char *name, uint8_t*& assembly_data, uint32_t& assembly_data_size) noexcept; + static void get_assembly_data (XamarinAndroidBundledAssembly const& e, uint8_t*& assembly_data, uint32_t& assembly_data_size) noexcept; + static void get_assembly_data (AssemblyStoreSingleAssemblyRuntimeData const& e, uint8_t*& assembly_data, uint32_t& assembly_data_size) noexcept; void zip_load_entries (int fd, const char *apk_name, monodroid_should_register should_register); + void zip_load_individual_assembly_entries (std::vector const& buf, uint32_t num_entries, monodroid_should_register should_register, ZipEntryLoadState &state) noexcept; + void zip_load_assembly_store_entries (std::vector const& buf, uint32_t num_entries, ZipEntryLoadState &state) noexcept; + bool zip_load_entry_common (size_t entry_index, std::vector const& buf, dynamic_local_string &entry_name, ZipEntryLoadState &state) noexcept; bool zip_read_cd_info (int fd, uint32_t& cd_offset, uint32_t& cd_size, uint16_t& cd_entries); - bool zip_adjust_data_offset (int fd, size_t local_header_offset, uint32_t &data_start_offset); + bool zip_adjust_data_offset (int fd, ZipEntryLoadState &state); template bool zip_extract_cd_info (std::array const& buf, uint32_t& cd_offset, uint32_t& cd_size, uint16_t& cd_entries); @@ -193,7 +237,7 @@ namespace xamarin::android::internal { template bool zip_read_field (T const& buf, size_t index, size_t count, dynamic_local_string& characters) const noexcept; - bool zip_read_entry_info (std::vector const& buf, size_t& buf_offset, uint16_t& compression_method, uint32_t& local_header_offset, uint32_t& file_size, dynamic_local_string& file_name); + bool zip_read_entry_info (std::vector const& buf, dynamic_local_string& file_name, ZipEntryLoadState &state); const char* get_assemblies_prefix () const { @@ -205,6 +249,16 @@ namespace xamarin::android::internal { return assemblies_prefix_override != nullptr ? static_cast(strlen (assemblies_prefix_override)) : sizeof(assemblies_prefix) - 1; } + bool all_required_zip_entries_found () const noexcept + { + return + number_of_mapped_assembly_stores == application_config.number_of_assembly_store_files +#if defined (NET6) + && ((application_config.have_runtime_config_blob && runtime_config_blob_found) || !application_config.have_runtime_config_blob) +#endif // NET6 + ; + } + bool is_debug_file (dynamic_local_string const& name) noexcept; template @@ -221,6 +275,8 @@ namespace xamarin::android::internal { void set_entry_data (XamarinAndroidBundledAssembly &entry, int apk_fd, uint32_t data_offset, uint32_t data_size, uint32_t prefix_len, uint32_t max_name_size, dynamic_local_string const& entry_name) noexcept; void set_assembly_entry_data (XamarinAndroidBundledAssembly &entry, int apk_fd, uint32_t data_offset, uint32_t data_size, uint32_t prefix_len, uint32_t max_name_size, dynamic_local_string const& entry_name) noexcept; void set_debug_entry_data (XamarinAndroidBundledAssembly &entry, int apk_fd, uint32_t data_offset, uint32_t data_size, uint32_t prefix_len, uint32_t max_name_size, dynamic_local_string const& entry_name) noexcept; + void map_assembly_store (dynamic_local_string const& entry_name, ZipEntryLoadState &state) noexcept; + const AssemblyStoreHashEntry* find_assembly_store_entry (hash_t hash, const AssemblyStoreHashEntry *entries, size_t entry_count) noexcept; private: std::vector *bundled_debug_data = nullptr; @@ -229,6 +285,8 @@ namespace xamarin::android::internal { bool register_debug_symbols; bool have_and_want_debug_symbols; size_t bundled_assembly_index = 0; + size_t number_of_found_assemblies = 0; + #if defined (DEBUG) || !defined (ANDROID) TypeMappingInfo *java_to_managed_maps; TypeMappingInfo *managed_to_java_maps; @@ -238,7 +296,13 @@ namespace xamarin::android::internal { const char *assemblies_prefix_override = nullptr; #if defined (NET6) md_mmap_info runtime_config_blob_mmap{}; + bool runtime_config_blob_found = false; #endif // def NET6 + uint32_t number_of_mapped_assembly_stores = 0; + bool need_to_scan_more_apks = true; + + AssemblyStoreHeader *index_assembly_store_header = nullptr; + AssemblyStoreHashEntry *assembly_store_hashes; }; } diff --git a/src/monodroid/jni/mono_android_Runtime.h b/src/monodroid/jni/mono_android_Runtime.h index 0e5aa37d095..d86951e0991 100644 --- a/src/monodroid/jni/mono_android_Runtime.h +++ b/src/monodroid/jni/mono_android_Runtime.h @@ -21,7 +21,7 @@ JNIEXPORT void JNICALL Java_mono_android_Runtime_init * Signature: (Ljava/lang/String;[Ljava/lang/String;Ljava/lang/String;[Ljava/lang/String;Ljava/lang/ClassLoader;[Ljava/lang/String;IZ)V */ JNIEXPORT void JNICALL Java_mono_android_Runtime_initInternal -(JNIEnv *, jclass, jstring, jobjectArray, jstring, jobjectArray, jobject, jobjectArray, jint, jboolean); +(JNIEnv *, jclass, jstring, jobjectArray, jstring, jobjectArray, jobject, jobjectArray, jint, jboolean, jboolean); /* * Class: mono_android_Runtime diff --git a/src/monodroid/jni/monodroid-glue-designer.cc b/src/monodroid/jni/monodroid-glue-designer.cc index 422e45e3c12..8e8d6ab4cba 100644 --- a/src/monodroid/jni/monodroid-glue-designer.cc +++ b/src/monodroid/jni/monodroid-glue-designer.cc @@ -42,7 +42,7 @@ MonodroidRuntime::Java_mono_android_Runtime_createNewContextWithData (JNIEnv *en jstring_array_wrapper runtimeApks (env, runtimeApksJava); jstring_array_wrapper assemblies (env, assembliesJava); jstring_array_wrapper assembliePaths (env, assembliesPaths); - MonoDomain *domain = create_and_initialize_domain (env, klass, runtimeApks, assemblies, assembliesBytes, assembliePaths, loader, /*is_root_domain:*/ false, force_preload_assemblies); + MonoDomain *domain = create_and_initialize_domain (env, klass, runtimeApks, assemblies, assembliesBytes, assembliePaths, loader, /*is_root_domain:*/ false, force_preload_assemblies, /* have_split_apks */ false); mono_domain_set (domain, FALSE); int domain_id = mono_domain_get_id (domain); current_context_id = domain_id; diff --git a/src/monodroid/jni/monodroid-glue-internal.hh b/src/monodroid/jni/monodroid-glue-internal.hh index caf8d3b1cd5..ce6b573bbb4 100644 --- a/src/monodroid/jni/monodroid-glue-internal.hh +++ b/src/monodroid/jni/monodroid-glue-internal.hh @@ -7,6 +7,7 @@ #include "android-system.hh" #include "osbridge.hh" #include "timing.hh" +#include "cpp-util.hh" #include "xxhash.hh" #include @@ -126,6 +127,7 @@ namespace xamarin::android::internal }; private: + static constexpr char base_apk_name[] = "/base.apk"; static constexpr size_t SMALL_STRING_PARSE_BUFFER_LEN = 50; static constexpr bool is_running_on_desktop = #if ANDROID @@ -151,7 +153,8 @@ namespace xamarin::android::internal void Java_mono_android_Runtime_register (JNIEnv *env, jstring managedType, jclass nativeClass, jstring methods); void Java_mono_android_Runtime_initInternal (JNIEnv *env, jclass klass, jstring lang, jobjectArray runtimeApksJava, jstring runtimeNativeLibDir, jobjectArray appDirs, jobject loader, - jobjectArray assembliesJava, jint apiLevel, jboolean isEmulator); + jobjectArray assembliesJava, jint apiLevel, jboolean isEmulator, + jboolean haveSplitApks); #if !defined (ANDROID) jint Java_mono_android_Runtime_createNewContextWithData (JNIEnv *env, jclass klass, jobjectArray runtimeApksJava, jobjectArray assembliesJava, jobjectArray assembliesBytes, jobjectArray assembliesPaths, jobject loader, jboolean force_preload_assemblies); @@ -294,12 +297,13 @@ namespace xamarin::android::internal #else // def NET6 MonoClass* get_android_runtime_class (MonoDomain *domain); #endif - MonoDomain* create_domain (JNIEnv *env, jstring_array_wrapper &runtimeApks, bool is_root_domain); + MonoDomain* create_domain (JNIEnv *env, jstring_array_wrapper &runtimeApks, bool is_root_domain, bool have_split_apks); MonoDomain* create_and_initialize_domain (JNIEnv* env, jclass runtimeClass, jstring_array_wrapper &runtimeApks, jstring_array_wrapper &assemblies, jobjectArray assembliesBytes, jstring_array_wrapper &assembliesPaths, - jobject loader, bool is_root_domain, bool force_preload_assemblies); + jobject loader, bool is_root_domain, bool force_preload_assemblies, + bool have_split_apks); - void gather_bundled_assemblies (jstring_array_wrapper &runtimeApks, size_t *out_user_assemblies_count); + void gather_bundled_assemblies (jstring_array_wrapper &runtimeApks, size_t *out_user_assemblies_count, bool have_split_apks); static bool should_register_file (const char *filename); void set_trace_options (); void set_profile_options (); diff --git a/src/monodroid/jni/monodroid-glue.cc b/src/monodroid/jni/monodroid-glue.cc index d2eed495b74..a4e1182f156 100644 --- a/src/monodroid/jni/monodroid-glue.cc +++ b/src/monodroid/jni/monodroid-glue.cc @@ -327,10 +327,9 @@ MonodroidRuntime::open_from_update_dir (MonoAssemblyName *aname, [[maybe_unused] } pname.append (name, name_len); - constexpr char dll_extension[] = ".dll"; - constexpr size_t dll_extension_len = sizeof(dll_extension) - 1; + constexpr size_t dll_extension_len = sizeof(SharedConstants::DLL_EXTENSION) - 1; - bool is_dll = utils.ends_with (name, dll_extension); + bool is_dll = utils.ends_with (name, SharedConstants::DLL_EXTENSION); size_t file_name_len = pname.length () + 1; if (!is_dll) file_name_len += dll_extension_len; @@ -344,7 +343,7 @@ MonodroidRuntime::open_from_update_dir (MonoAssemblyName *aname, [[maybe_unused] static_local_string fullpath (override_dir_len + file_name_len); utils.path_combine (fullpath, override_dir, override_dir_len, pname.get (), pname.length ()); if (!is_dll) { - fullpath.append (dll_extension, dll_extension_len); + fullpath.append (SharedConstants::DLL_EXTENSION, dll_extension_len); } log_info (LOG_ASSEMBLY, "open_from_update_dir: trying to open assembly: %s\n", fullpath.get ()); @@ -392,7 +391,7 @@ MonodroidRuntime::should_register_file ([[maybe_unused]] const char *filename) } inline void -MonodroidRuntime::gather_bundled_assemblies (jstring_array_wrapper &runtimeApks, size_t *out_user_assemblies_count) +MonodroidRuntime::gather_bundled_assemblies (jstring_array_wrapper &runtimeApks, size_t *out_user_assemblies_count, bool have_split_apks) { #if defined(DEBUG) || !defined (ANDROID) if (application_config.instant_run_enabled) { @@ -408,14 +407,37 @@ MonodroidRuntime::gather_bundled_assemblies (jstring_array_wrapper &runtimeApks, int64_t apk_count = static_cast(runtimeApks.get_length ()); size_t prev_num_assemblies = 0; - for (int64_t i = apk_count - 1; i >= 0; --i) { + bool got_split_config_abi_apk = false; + bool got_base_apk = false; + + for (int64_t i = 0; i < apk_count; i++) { jstring_wrapper &apk_file = runtimeApks [static_cast(i)]; + if (have_split_apks) { + bool scan_apk = false; + + if (!got_split_config_abi_apk && utils.ends_with (apk_file.get_cstr (), SharedConstants::split_config_abi_apk_name)) { + got_split_config_abi_apk = scan_apk = true; + } else if (!got_base_apk && utils.ends_with (apk_file.get_cstr (), base_apk_name)) { + got_base_apk = scan_apk = true; + } + + if (!scan_apk) { + continue; + } + } + size_t cur_num_assemblies = embeddedAssemblies.register_from (apk_file.get_cstr ()); *out_user_assemblies_count += (cur_num_assemblies - prev_num_assemblies); prev_num_assemblies = cur_num_assemblies; + + if (!embeddedAssemblies.keep_scanning ()) { + break; + } } + + embeddedAssemblies.ensure_valid_assembly_stores (); } #if defined (DEBUG) && !defined (WINDOWS) @@ -856,7 +878,7 @@ MonodroidRuntime::cleanup_runtime_config (MonovmRuntimeConfigArguments *args, [[ #endif // def NET6 MonoDomain* -MonodroidRuntime::create_domain (JNIEnv *env, jstring_array_wrapper &runtimeApks, bool is_root_domain) +MonodroidRuntime::create_domain (JNIEnv *env, jstring_array_wrapper &runtimeApks, bool is_root_domain, bool have_split_apks) { size_t user_assemblies_count = 0; #if defined (NET6) @@ -865,7 +887,7 @@ MonodroidRuntime::create_domain (JNIEnv *env, jstring_array_wrapper &runtimeApks bool have_mono_mkbundle_init = mono_mkbundle_init != nullptr; #endif // ndef NET6 - gather_bundled_assemblies (runtimeApks, &user_assemblies_count); + gather_bundled_assemblies (runtimeApks, &user_assemblies_count, have_split_apks); #if defined (NET6) timing_period blob_time; @@ -888,7 +910,7 @@ MonodroidRuntime::create_domain (JNIEnv *env, jstring_array_wrapper &runtimeApks log_fatal (LOG_DEFAULT, "No assemblies found in '%s' or '%s'. Assuming this is part of Fast Deployment. Exiting...", androidSystem.get_override_dir (0), (AndroidSystem::MAX_OVERRIDES > 1 && androidSystem.get_override_dir (1) != nullptr) ? androidSystem.get_override_dir (1) : ""); - exit (FATAL_EXIT_NO_ASSEMBLIES); + abort (); } MonoDomain *domain; @@ -1836,9 +1858,9 @@ MonoDomain* MonodroidRuntime::create_and_initialize_domain (JNIEnv* env, jclass runtimeClass, jstring_array_wrapper &runtimeApks, jstring_array_wrapper &assemblies, [[maybe_unused]] jobjectArray assembliesBytes, [[maybe_unused]] jstring_array_wrapper &assembliesPaths, jobject loader, bool is_root_domain, - bool force_preload_assemblies) + bool force_preload_assemblies, bool have_split_apks) { - MonoDomain* domain = create_domain (env, runtimeApks, is_root_domain); + MonoDomain* domain = create_domain (env, runtimeApks, is_root_domain, have_split_apks); // When running on desktop, the root domain is only a dummy so don't initialize it if constexpr (is_running_on_desktop) { @@ -2025,7 +2047,8 @@ MonodroidRuntime::install_logging_handlers () inline void MonodroidRuntime::Java_mono_android_Runtime_initInternal (JNIEnv *env, jclass klass, jstring lang, jobjectArray runtimeApksJava, jstring runtimeNativeLibDir, jobjectArray appDirs, jobject loader, - jobjectArray assembliesJava, jint apiLevel, jboolean isEmulator) + jobjectArray assembliesJava, jint apiLevel, jboolean isEmulator, + jboolean haveSplitApks) { char *mono_log_mask = nullptr; char *mono_log_level = nullptr; @@ -2080,7 +2103,7 @@ MonodroidRuntime::Java_mono_android_Runtime_initInternal (JNIEnv *env, jclass kl disable_external_signal_handlers (); jstring_array_wrapper runtimeApks (env, runtimeApksJava); - androidSystem.setup_app_library_directories (runtimeApks, applicationDirs); + androidSystem.setup_app_library_directories (runtimeApks, applicationDirs, haveSplitApks); init_reference_logging (androidSystem.get_primary_override_dir ()); androidSystem.create_update_dir (androidSystem.get_primary_override_dir ()); @@ -2228,7 +2251,7 @@ MonodroidRuntime::Java_mono_android_Runtime_initInternal (JNIEnv *env, jclass kl jstring_array_wrapper assemblies (env, assembliesJava); jstring_array_wrapper assembliesPaths (env); /* the first assembly is used to initialize the AppDomain name */ - create_and_initialize_domain (env, klass, runtimeApks, assemblies, nullptr, assembliesPaths, loader, /*is_root_domain:*/ true, /*force_preload_assemblies:*/ false); + create_and_initialize_domain (env, klass, runtimeApks, assemblies, nullptr, assembliesPaths, loader, /*is_root_domain:*/ true, /*force_preload_assemblies:*/ false, haveSplitApks); #if defined (ANDROID) && !defined (NET6) // Mono from mono/mono has a bug which requires us to install the handlers after `mono_init_jit_version` is called @@ -2315,14 +2338,16 @@ Java_mono_android_Runtime_init (JNIEnv *env, jclass klass, jstring lang, jobject loader, assembliesJava, apiLevel, - /* isEmulator */ JNI_FALSE + /* isEmulator */ JNI_FALSE, + /* haveSplitApks */ JNI_FALSE ); } JNIEXPORT void JNICALL Java_mono_android_Runtime_initInternal (JNIEnv *env, jclass klass, jstring lang, jobjectArray runtimeApksJava, jstring runtimeNativeLibDir, jobjectArray appDirs, jobject loader, - jobjectArray assembliesJava, jint apiLevel, jboolean isEmulator) + jobjectArray assembliesJava, jint apiLevel, jboolean isEmulator, + jboolean haveSplitApks) { monodroidRuntime.Java_mono_android_Runtime_initInternal ( env, @@ -2334,7 +2359,8 @@ Java_mono_android_Runtime_initInternal (JNIEnv *env, jclass klass, jstring lang, loader, assembliesJava, apiLevel, - isEmulator + isEmulator, + haveSplitApks ); } diff --git a/src/monodroid/jni/shared-constants.hh b/src/monodroid/jni/shared-constants.hh index 2ce65384484..f3778e7b632 100644 --- a/src/monodroid/jni/shared-constants.hh +++ b/src/monodroid/jni/shared-constants.hh @@ -1,6 +1,8 @@ #ifndef __SHARED_CONSTANTS_HH #define __SHARED_CONSTANTS_HH +#include "cpp-util.hh" + namespace xamarin::android::internal { // _WIN32 is defined with _WIN64 so _WIN64 must be checked first. @@ -22,6 +24,8 @@ namespace xamarin::android::internal static constexpr char JNIENV_CLASS_NAME[] = "JNIEnv"; static constexpr char ANDROID_ENVIRONMENT_CLASS_NAME[] = "AndroidEnvironment"; + static constexpr char DLL_EXTENSION[] = ".dll"; + #if defined (NET6) static constexpr char RUNTIME_CONFIG_BLOB_NAME[] = "rc.bin"; #endif // def NET6 @@ -39,6 +43,18 @@ namespace xamarin::android::internal static constexpr char MONO_SGEN_SO[] = "monosgen-2.0"; static constexpr char MONO_SGEN_ARCH_SO[] = "monosgen-" __BITNESS__ "-2.0"; #endif + +#if __arm__ + static constexpr char android_abi[] = "armeabi_v7a"; +#elif __aarch64__ + static constexpr char android_abi[] = "arm64_v8a"; +#elif __x86_64__ + static constexpr char android_abi[] = "x86_64"; +#elif __i386__ + static constexpr char android_abi[] = "x86"; +#endif + + static constexpr auto split_config_abi_apk_name = concat_const ("/split_config.", android_abi, ".apk"); }; } #endif // __SHARED_CONSTANTS_HH diff --git a/src/monodroid/jni/xamarin-app.hh b/src/monodroid/jni/xamarin-app.hh index 06f7668da62..41a5aee422d 100644 --- a/src/monodroid/jni/xamarin-app.hh +++ b/src/monodroid/jni/xamarin-app.hh @@ -10,6 +10,9 @@ static constexpr uint64_t FORMAT_TAG = 0x015E6972616D58; static constexpr uint32_t COMPRESSED_DATA_MAGIC = 0x5A4C4158; // 'XALZ', little-endian +static constexpr uint32_t ASSEMBLY_STORE_MAGIC = 0x41424158; // 'XABA', little-endian +static constexpr uint32_t ASSEMBLY_STORE_FORMAT_VERSION = 1; // Increase whenever an incompatible change is made to the + // assembly store format static constexpr uint32_t MODULE_MAGIC_NAMES = 0x53544158; // 'XATS', little-endian static constexpr uint32_t MODULE_INDEX_MAGIC = 0x49544158; // 'XATI', little-endian static constexpr uint8_t MODULE_FORMAT_VERSION = 2; // Keep in sync with the value in src/Xamarin.Android.Build.Tasks/Utilities/TypeMapGenerator.cs @@ -106,6 +109,86 @@ struct XamarinAndroidBundledAssembly final char *name; }; +// +// Assembly store format +// +// The separate hash indices for 32 and 64-bit hashes are required because they will be sorted differently. +// The 'index' field of each of the hashes{32,64} entry points not only into the `assemblies` array in the +// store but also into the `uint8_t*` `assembly_store_bundled_assemblies*` arrays. +// +// This way the `assemblies` array in the store can remain read only, because we write the "mapped" assembly +// pointer somewhere else. Otherwise we'd have to copy the `assemblies` array to a writable area of memory. +// +// Each store has a unique ID assigned, which is an index into an array of pointers to arrays which store +// individual assembly addresses. Only store with ID 0 comes with the hashes32 and hashes64 arrays. This is +// done to make it possible to use a single sorted array to find assemblies insted of each store having its +// own sorted array of hashes, which would require several binary searches instead of just one. +// +// AssemblyStoreHeader header; +// AssemblyStoreAssemblyDescriptor assemblies[header.local_entry_count]; +// AssemblyStoreHashEntry hashes32[header.global_entry_count]; // only in assembly store with ID 0 +// AssemblyStoreHashEntry hashes64[header.global_entry_count]; // only in assembly store with ID 0 +// [DATA] +// + +// +// The structures which are found in the store files must be packed to avoid problems when calculating offsets (runtime +// size of a structure can be different than the real data size) +// +struct [[gnu::packed]] AssemblyStoreHeader final +{ + uint32_t magic; + uint32_t version; + uint32_t local_entry_count; + uint32_t global_entry_count; + uint32_t store_id; +}; + +struct [[gnu::packed]] AssemblyStoreHashEntry final +{ + union { + uint64_t hash64; + uint32_t hash32; + }; + + // Index into the array with pointers to assembly data. + // It **must** be unique across all the stores from all the apks + uint32_t mapping_index; + + // Index into the array with assembly descriptors inside a store + uint32_t local_store_index; + + // Index into the array with assembly store mmap addresses + uint32_t store_id; +}; + +struct [[gnu::packed]] AssemblyStoreAssemblyDescriptor final +{ + uint32_t data_offset; + uint32_t data_size; + + uint32_t debug_data_offset; + uint32_t debug_data_size; + + uint32_t config_data_offset; + uint32_t config_data_size; +}; + +struct AssemblyStoreRuntimeData final +{ + uint8_t *data_start; + uint32_t assembly_count; + AssemblyStoreAssemblyDescriptor *assemblies; +}; + +struct AssemblyStoreSingleAssemblyRuntimeData final +{ + uint8_t *image_data; + uint8_t *debug_info_data; + uint8_t *config_data; + AssemblyStoreAssemblyDescriptor *descriptor; +}; + struct ApplicationConfig { bool uses_mono_llvm; @@ -116,12 +199,14 @@ struct ApplicationConfig bool instant_run_enabled; bool jni_add_native_method_registration_attribute_present; bool have_runtime_config_blob; + bool have_assembly_store; uint8_t bound_exception_type; uint32_t package_naming_policy; uint32_t environment_variable_count; uint32_t system_property_count; uint32_t number_of_assemblies_in_apk; uint32_t bundled_assembly_name_width; + uint32_t number_of_assembly_store_files; const char *android_package_name; }; @@ -145,5 +230,7 @@ MONO_API const char* app_system_properties[]; MONO_API const char* mono_aot_mode_name; MONO_API XamarinAndroidBundledAssembly bundled_assemblies[]; +MONO_API AssemblyStoreSingleAssemblyRuntimeData assembly_store_bundled_assemblies[]; +MONO_API AssemblyStoreRuntimeData assembly_stores[]; #endif // __XAMARIN_ANDROID_TYPEMAP_H diff --git a/tests/MSBuildDeviceIntegration/MSBuildDeviceIntegration.csproj b/tests/MSBuildDeviceIntegration/MSBuildDeviceIntegration.csproj index 9af5f8da79e..a180a6b851d 100644 --- a/tests/MSBuildDeviceIntegration/MSBuildDeviceIntegration.csproj +++ b/tests/MSBuildDeviceIntegration/MSBuildDeviceIntegration.csproj @@ -22,6 +22,7 @@ + diff --git a/tests/MSBuildDeviceIntegration/Tests/BundleToolTests.cs b/tests/MSBuildDeviceIntegration/Tests/BundleToolTests.cs index eac880827ed..7722a698b13 100644 --- a/tests/MSBuildDeviceIntegration/Tests/BundleToolTests.cs +++ b/tests/MSBuildDeviceIntegration/Tests/BundleToolTests.cs @@ -10,15 +10,22 @@ namespace Xamarin.Android.Build.Tests { [TestFixture] + [TestFixtureSource(nameof(FixtureArgs))] [Category ("Node-1")] public class BundleToolTests : DeviceTest { + static readonly object[] FixtureArgs = { + new object[] { false }, + new object[] { true }, + }; + static readonly string [] Abis = new [] { "armeabi-v7a", "arm64-v8a", "x86" }; XamarinAndroidLibraryProject lib; XamarinAndroidApplicationProject app; ProjectBuilder libBuilder, appBuilder; string intermediate; string bin; + bool usesAssemblyBlobs; // Disable split by language const string BuildConfig = @"{ @@ -34,6 +41,11 @@ public class BundleToolTests : DeviceTest } }"; + public BundleToolTests (bool usesAssemblyBlobs) + { + this.usesAssemblyBlobs = usesAssemblyBlobs; + } + [OneTimeSetUp] public void OneTimeSetUp () { @@ -51,6 +63,8 @@ public void OneTimeSetUp () } }; + lib.SetProperty ("AndroidUseAssemblyStore", usesAssemblyBlobs.ToString ()); + var bytes = new byte [1024]; app = new XamarinFormsMapsApplicationProject { IsRelease = true, @@ -73,6 +87,7 @@ public void OneTimeSetUp () app.SetProperty (app.ReleaseProperties, "AndroidPackageFormat", "aab"); app.SetAndroidSupportedAbis (Abis); app.SetProperty ("AndroidBundleConfigurationFile", "buildConfig.json"); + app.SetProperty ("AndroidUseAssemblyStore", usesAssemblyBlobs.ToString ()); libBuilder = CreateDllBuilder (Path.Combine (path, lib.ProjectName), cleanupOnDispose: true); Assert.IsTrue (libBuilder.Build (lib), "Library build should have succeeded."); @@ -103,14 +118,10 @@ public void OneTimeTearDown () appBuilder?.Dispose (); } - string [] ListArchiveContents (string archive) + string [] ListArchiveContents (string archive, bool usesAssembliesBlob) { - var entries = new List (); - using (var zip = ZipArchive.Open (archive, FileMode.Open)) { - foreach (var entry in zip) { - entries.Add (entry.FullName); - } - } + var helper = new ArchiveAssemblyHelper (archive, usesAssembliesBlob); + List entries = helper.ListArchiveContents (); entries.Sort (); return entries.ToArray (); } @@ -119,7 +130,7 @@ string [] ListArchiveContents (string archive) public void BaseZip () { var baseZip = Path.Combine (intermediate, "android", "bin", "base.zip"); - var contents = ListArchiveContents (baseZip); + var contents = ListArchiveContents (baseZip, usesAssemblyBlobs); var expectedFiles = new List { "dex/classes.dex", "manifest/AndroidManifest.xml", @@ -130,16 +141,33 @@ public void BaseZip () "res/drawable-xxxhdpi-v4/icon.png", "res/layout/main.xml", "resources.pb", - "root/assemblies/Java.Interop.dll", - "root/assemblies/Mono.Android.dll", - "root/assemblies/Localization.dll", - "root/assemblies/es/Localization.resources.dll", - "root/assemblies/UnnamedProject.dll", }; + + string blobEntryPrefix = ArchiveAssemblyHelper.DefaultAssemblyStoreEntryPrefix; + if (usesAssemblyBlobs) { + expectedFiles.Add ($"{blobEntryPrefix}Java.Interop.dll"); + expectedFiles.Add ($"{blobEntryPrefix}Mono.Android.dll"); + expectedFiles.Add ($"{blobEntryPrefix}Localization.dll"); + expectedFiles.Add ($"{blobEntryPrefix}es/Localization.resources.dll"); + expectedFiles.Add ($"{blobEntryPrefix}UnnamedProject.dll"); + } else { + expectedFiles.Add ("root/assemblies/Java.Interop.dll"); + expectedFiles.Add ("root/assemblies/Mono.Android.dll"); + expectedFiles.Add ("root/assemblies/Localization.dll"); + expectedFiles.Add ("root/assemblies/es/Localization.resources.dll"); + expectedFiles.Add ("root/assemblies/UnnamedProject.dll"); + } + if (Builder.UseDotNet) { - expectedFiles.Add ("root/assemblies/System.Console.dll"); - expectedFiles.Add ("root/assemblies/System.Linq.dll"); - expectedFiles.Add ("root/assemblies/System.Net.Http.dll"); + if (usesAssemblyBlobs) { + expectedFiles.Add ($"{blobEntryPrefix}System.Console.dll"); + expectedFiles.Add ($"{blobEntryPrefix}System.Linq.dll"); + expectedFiles.Add ($"{blobEntryPrefix}System.Net.Http.dll"); + } else { + expectedFiles.Add ("root/assemblies/System.Console.dll"); + expectedFiles.Add ("root/assemblies/System.Linq.dll"); + expectedFiles.Add ("root/assemblies/System.Net.Http.dll"); + } //These are random files from Google Play Services .aar files expectedFiles.Add ("root/play-services-base.properties"); @@ -147,10 +175,17 @@ public void BaseZip () expectedFiles.Add ("root/play-services-maps.properties"); expectedFiles.Add ("root/play-services-tasks.properties"); } else { - expectedFiles.Add ("root/assemblies/mscorlib.dll"); - expectedFiles.Add ("root/assemblies/System.Core.dll"); - expectedFiles.Add ("root/assemblies/System.dll"); - expectedFiles.Add ("root/assemblies/System.Runtime.Serialization.dll"); + if (usesAssemblyBlobs) { + expectedFiles.Add ($"{blobEntryPrefix}mscorlib.dll"); + expectedFiles.Add ($"{blobEntryPrefix}System.Core.dll"); + expectedFiles.Add ($"{blobEntryPrefix}System.dll"); + expectedFiles.Add ($"{blobEntryPrefix}System.Runtime.Serialization.dll"); + } else { + expectedFiles.Add ("root/assemblies/mscorlib.dll"); + expectedFiles.Add ("root/assemblies/System.Core.dll"); + expectedFiles.Add ("root/assemblies/System.dll"); + expectedFiles.Add ("root/assemblies/System.Runtime.Serialization.dll"); + } //These are random files from Google Play Services .aar files expectedFiles.Add ("root/build-data.properties"); @@ -180,7 +215,7 @@ public void AppBundle () { var aab = Path.Combine (intermediate, "android", "bin", $"{app.PackageName}.aab"); FileAssert.Exists (aab); - var contents = ListArchiveContents (aab); + var contents = ListArchiveContents (aab, usesAssemblyBlobs); var expectedFiles = new List { "base/dex/classes.dex", "base/manifest/AndroidManifest.xml", @@ -192,17 +227,34 @@ public void AppBundle () "base/res/drawable-xxxhdpi-v4/icon.png", "base/res/layout/main.xml", "base/resources.pb", - "base/root/assemblies/Java.Interop.dll", - "base/root/assemblies/Mono.Android.dll", - "base/root/assemblies/Localization.dll", - "base/root/assemblies/es/Localization.resources.dll", - "base/root/assemblies/UnnamedProject.dll", "BundleConfig.pb", }; + + string blobEntryPrefix = ArchiveAssemblyHelper.DefaultAssemblyStoreEntryPrefix; + if (usesAssemblyBlobs) { + expectedFiles.Add ($"{blobEntryPrefix}Java.Interop.dll"); + expectedFiles.Add ($"{blobEntryPrefix}Mono.Android.dll"); + expectedFiles.Add ($"{blobEntryPrefix}Localization.dll"); + expectedFiles.Add ($"{blobEntryPrefix}es/Localization.resources.dll"); + expectedFiles.Add ($"{blobEntryPrefix}UnnamedProject.dll"); + } else { + expectedFiles.Add ("base/root/assemblies/Java.Interop.dll"); + expectedFiles.Add ("base/root/assemblies/Mono.Android.dll"); + expectedFiles.Add ("base/root/assemblies/Localization.dll"); + expectedFiles.Add ("base/root/assemblies/es/Localization.resources.dll"); + expectedFiles.Add ("base/root/assemblies/UnnamedProject.dll"); + } + if (Builder.UseDotNet) { - expectedFiles.Add ("base/root/assemblies/System.Console.dll"); - expectedFiles.Add ("base/root/assemblies/System.Linq.dll"); - expectedFiles.Add ("base/root/assemblies/System.Net.Http.dll"); + if (usesAssemblyBlobs) { + expectedFiles.Add ($"{blobEntryPrefix}System.Console.dll"); + expectedFiles.Add ($"{blobEntryPrefix}System.Linq.dll"); + expectedFiles.Add ($"{blobEntryPrefix}System.Net.Http.dll"); + } else { + expectedFiles.Add ("base/root/assemblies/System.Console.dll"); + expectedFiles.Add ("base/root/assemblies/System.Linq.dll"); + expectedFiles.Add ("base/root/assemblies/System.Net.Http.dll"); + } //These are random files from Google Play Services .aar files expectedFiles.Add ("base/root/play-services-base.properties"); @@ -210,10 +262,17 @@ public void AppBundle () expectedFiles.Add ("base/root/play-services-maps.properties"); expectedFiles.Add ("base/root/play-services-tasks.properties"); } else { - expectedFiles.Add ("base/root/assemblies/mscorlib.dll"); - expectedFiles.Add ("base/root/assemblies/System.Core.dll"); - expectedFiles.Add ("base/root/assemblies/System.dll"); - expectedFiles.Add ("base/root/assemblies/System.Runtime.Serialization.dll"); + if (usesAssemblyBlobs) { + expectedFiles.Add ($"{blobEntryPrefix}mscorlib.dll"); + expectedFiles.Add ($"{blobEntryPrefix}System.Core.dll"); + expectedFiles.Add ($"{blobEntryPrefix}System.dll"); + expectedFiles.Add ($"{blobEntryPrefix}System.Runtime.Serialization.dll"); + } else { + expectedFiles.Add ("base/root/assemblies/mscorlib.dll"); + expectedFiles.Add ("base/root/assemblies/System.Core.dll"); + expectedFiles.Add ("base/root/assemblies/System.dll"); + expectedFiles.Add ("base/root/assemblies/System.Runtime.Serialization.dll"); + } //These are random files from Google Play Services .aar files expectedFiles.Add ("base/root/build-data.properties"); @@ -243,7 +302,7 @@ public void AppBundleSigned () { var aab = Path.Combine (bin, $"{app.PackageName}-Signed.aab"); FileAssert.Exists (aab); - var contents = ListArchiveContents (aab); + var contents = ListArchiveContents (aab, usesAssembliesBlob: false); Assert.IsTrue (StringAssertEx.ContainsText (contents, "META-INF/MANIFEST.MF"), $"{aab} is not signed!"); } @@ -259,11 +318,11 @@ public void ApkSet () FileAssert.Exists (aab); // Expecting: splits/base-arm64_v8a.apk, splits/base-master.apk, splits/base-xxxhdpi.apk // This are split up based on: abi, base, and dpi - var contents = ListArchiveContents (aab).Where (a => a.EndsWith (".apk", StringComparison.OrdinalIgnoreCase)).ToArray (); + var contents = ListArchiveContents (aab, usesAssembliesBlob: false).Where (a => a.EndsWith (".apk", StringComparison.OrdinalIgnoreCase)).ToArray (); Assert.AreEqual (3, contents.Length, "Expecting three APKs!"); // Language split has been removed by the bundle configuration file, and therefore shouldn't be present - var languageSplitContent = ListArchiveContents (aab).Where (a => a.EndsWith ("-en.apk", StringComparison.OrdinalIgnoreCase)).ToArray (); + var languageSplitContent = ListArchiveContents (aab, usesAssemblyBlobs).Where (a => a.EndsWith ("-en.apk", StringComparison.OrdinalIgnoreCase)).ToArray (); Assert.AreEqual (0, languageSplitContent.Length, "Found language split apk in bundle, but disabled by bundle configuration file!"); using (var stream = new MemoryStream ()) @@ -273,7 +332,16 @@ public void ApkSet () baseMaster.Extract (stream); stream.Position = 0; - var uncompressed = new [] { ".dll", ".bar", ".wav" }; + var uncompressed = new List { + ".bar", + ".wav", + }; + + if (usesAssemblyBlobs) { + uncompressed.Add (".blob"); + } else { + uncompressed.Add (".dll"); + } using (var baseApk = ZipArchive.Open (stream)) { foreach (var file in baseApk) { foreach (var ext in uncompressed) { diff --git a/tests/MSBuildDeviceIntegration/Tests/InstallAndRunTests.cs b/tests/MSBuildDeviceIntegration/Tests/InstallAndRunTests.cs index e2326d6783a..7018f402f1d 100644 --- a/tests/MSBuildDeviceIntegration/Tests/InstallAndRunTests.cs +++ b/tests/MSBuildDeviceIntegration/Tests/InstallAndRunTests.cs @@ -613,6 +613,7 @@ public void RunWithInterpreterEnabled ([Values (false, true)] bool isRelease) proj.SetAndroidSupportedAbis (abis); proj.SetProperty (proj.CommonProperties, "UseInterpreter", "True"); builder = CreateApkBuilder (); + builder.BuildLogFile = "install.log"; Assert.IsTrue (builder.Install (proj), "Install should have succeeded."); if (!Builder.UseDotNet) { @@ -627,6 +628,7 @@ public void RunWithInterpreterEnabled ([Values (false, true)] bool isRelease) var logProp = RunAdbCommand ("shell getprop debug.mono.log")?.Trim (); Assert.AreEqual (logProp, "all", "The debug.mono.log prop was not set correctly."); + builder.BuildLogFile = "run.log"; if (CommercialBuildAvailable) Assert.True (builder.RunTarget (proj, "_Run"), "Project should have run."); else diff --git a/tests/MSBuildDeviceIntegration/Tests/XASdkDeployTests.cs b/tests/MSBuildDeviceIntegration/Tests/XASdkDeployTests.cs index 418c09e15db..cc6be6e630c 100644 --- a/tests/MSBuildDeviceIntegration/Tests/XASdkDeployTests.cs +++ b/tests/MSBuildDeviceIntegration/Tests/XASdkDeployTests.cs @@ -74,6 +74,7 @@ public void DotNetInstallAndRun (bool isRelease, bool xamarinForms, bool? publis proj.CopyNuGetConfig (relativeProjDir); var dotnet = new DotNetCLI (proj, Path.Combine (fullProjDir, proj.ProjectFilePath)); + Assert.IsTrue (dotnet.Build (), "`dotnet build` should succeed"); Assert.IsTrue (dotnet.Run (), "`dotnet run` should succeed"); WaitForPermissionActivity (Path.Combine (Root, dotnet.ProjectDirectory, "permission-logcat.log")); bool didLaunch = WaitForActivityToStart (proj.PackageName, "MainActivity", diff --git a/tests/Xamarin.Forms-Performance-Integration/Droid/Xamarin.Forms.Performance.Integration.Droid.csproj b/tests/Xamarin.Forms-Performance-Integration/Droid/Xamarin.Forms.Performance.Integration.Droid.csproj index ed5e8a5dcc2..a10607edb93 100644 --- a/tests/Xamarin.Forms-Performance-Integration/Droid/Xamarin.Forms.Performance.Integration.Droid.csproj +++ b/tests/Xamarin.Forms-Performance-Integration/Droid/Xamarin.Forms.Performance.Integration.Droid.csproj @@ -20,6 +20,7 @@ armeabi-v7a;x86 arm64-v8a;x86 True + False true <_AndroidCheckedBuild Condition=" '$(UseASAN)' != '' ">asan <_AndroidCheckedBuild Condition=" '$(UseUBSAN)' != '' ">ubsan diff --git a/tests/apk-sizes-reference/com.companyname.vsandroidapp-Signed-Release.apkdesc b/tests/apk-sizes-reference/com.companyname.vsandroidapp-Signed-Release.apkdesc index 14987f2e37b..2dafe6e3ccd 100644 --- a/tests/apk-sizes-reference/com.companyname.vsandroidapp-Signed-Release.apkdesc +++ b/tests/apk-sizes-reference/com.companyname.vsandroidapp-Signed-Release.apkdesc @@ -4,68 +4,14 @@ "AndroidManifest.xml": { "Size": 2896 }, - "assemblies/Java.Interop.dll": { - "Size": 68160 + "assemblies/assemblies.blob": { + "Size": 2026794 }, - "assemblies/Mono.Android.dll": { - "Size": 330421 - }, - "assemblies/Mono.Security.dll": { - "Size": 61400 - }, - "assemblies/mscorlib.dll": { - "Size": 843651 - }, - "assemblies/System.Core.dll": { - "Size": 33884 - }, - "assemblies/System.dll": { - "Size": 208052 - }, - "assemblies/System.Net.Http.dll": { - "Size": 110901 - }, - "assemblies/System.Numerics.dll": { - "Size": 15656 - }, - "assemblies/VSAndroidApp.dll": { - "Size": 60901 - }, - "assemblies/Xamarin.Android.Arch.Lifecycle.Common.dll": { - "Size": 6971 - }, - "assemblies/Xamarin.Android.Arch.Lifecycle.LiveData.Core.dll": { - "Size": 7017 - }, - "assemblies/Xamarin.Android.Arch.Lifecycle.ViewModel.dll": { - "Size": 4779 - }, - "assemblies/Xamarin.Android.Support.Compat.dll": { - "Size": 50728 - }, - "assemblies/Xamarin.Android.Support.CoordinaterLayout.dll": { - "Size": 17769 - }, - "assemblies/Xamarin.Android.Support.Design.dll": { - "Size": 28949 - }, - "assemblies/Xamarin.Android.Support.DrawerLayout.dll": { - "Size": 14618 - }, - "assemblies/Xamarin.Android.Support.Fragment.dll": { - "Size": 41190 - }, - "assemblies/Xamarin.Android.Support.Loader.dll": { - "Size": 13489 - }, - "assemblies/Xamarin.Android.Support.v7.AppCompat.dll": { - "Size": 92177 - }, - "assemblies/Xamarin.Essentials.dll": { - "Size": 14090 + "assemblies/assemblies.manifest": { + "Size": 1574 }, "classes.dex": { - "Size": 2940492 + "Size": 2940564 }, "lib/armeabi-v7a/libmono-btls-shared.so": { "Size": 1112688 @@ -74,7 +20,7 @@ "Size": 736396 }, "lib/armeabi-v7a/libmonodroid.so": { - "Size": 229116 + "Size": 232320 }, "lib/armeabi-v7a/libmonosgen-2.0.so": { "Size": 4456332 @@ -83,7 +29,7 @@ "Size": 48844 }, "lib/armeabi-v7a/libxamarin-app.so": { - "Size": 46912 + "Size": 46832 }, "lib/x86/libmono-btls-shared.so": { "Size": 1459584 @@ -92,7 +38,7 @@ "Size": 803352 }, "lib/x86/libmonodroid.so": { - "Size": 297800 + "Size": 303316 }, "lib/x86/libmonosgen-2.0.so": { "Size": 4212220 @@ -101,7 +47,7 @@ "Size": 61112 }, "lib/x86/libxamarin-app.so": { - "Size": 45580 + "Size": 45500 }, "META-INF/android.arch.core_runtime.version": { "Size": 6 @@ -125,7 +71,7 @@ "Size": 1213 }, "META-INF/ANDROIDD.SF": { - "Size": 67139 + "Size": 65121 }, "META-INF/androidx.appcompat_appcompat.version": { "Size": 6 @@ -206,7 +152,7 @@ "Size": 10 }, "META-INF/MANIFEST.MF": { - "Size": 67012 + "Size": 64994 }, "META-INF/proguard/androidx-annotations.pro": { "Size": 308 @@ -1664,5 +1610,5 @@ "Size": 320016 } }, - "PackageSize": 9498135 + "PackageSize": 9500681 } \ No newline at end of file diff --git a/tools/assembly-store-reader/AssemblyStoreAssembly.cs b/tools/assembly-store-reader/AssemblyStoreAssembly.cs new file mode 100644 index 00000000000..b83be2acbf4 --- /dev/null +++ b/tools/assembly-store-reader/AssemblyStoreAssembly.cs @@ -0,0 +1,87 @@ +using System; +using System.IO; + +namespace Xamarin.Android.AssemblyStore +{ + class AssemblyStoreAssembly + { + public uint DataOffset { get; } + public uint DataSize { get; } + public uint DebugDataOffset { get; } + public uint DebugDataSize { get; } + public uint ConfigDataOffset { get; } + public uint ConfigDataSize { get; } + + public uint Hash32 { get; set; } + public ulong Hash64 { get; set; } + public string Name { get; set; } = String.Empty; + public uint RuntimeIndex { get; set; } + + public AssemblyStoreReader Store { get; } + public string DllName => MakeFileName ("dll"); + public string PdbName => MakeFileName ("pdb"); + public string ConfigName => MakeFileName ("dll.config"); + + internal AssemblyStoreAssembly (BinaryReader reader, AssemblyStoreReader store) + { + Store = store; + + DataOffset = reader.ReadUInt32 (); + DataSize = reader.ReadUInt32 (); + DebugDataOffset = reader.ReadUInt32 (); + DebugDataSize = reader.ReadUInt32 (); + ConfigDataOffset = reader.ReadUInt32 (); + ConfigDataSize = reader.ReadUInt32 (); + } + + public void ExtractImage (string outputDirPath, string? fileName = null) + { + Store.ExtractAssemblyImage (this, MakeOutputFilePath (outputDirPath, "dll", fileName)); + } + + public void ExtractImage (Stream output) + { + Store.ExtractAssemblyImage (this, output); + } + + public void ExtractDebugData (string outputDirPath, string? fileName = null) + { + Store.ExtractAssemblyDebugData (this, MakeOutputFilePath (outputDirPath, "pdb", fileName)); + } + + public void ExtractDebugData (Stream output) + { + Store.ExtractAssemblyDebugData (this, output); + } + + public void ExtractConfig (string outputDirPath, string? fileName = null) + { + Store.ExtractAssemblyConfig (this, MakeOutputFilePath (outputDirPath, "dll.config", fileName)); + } + + public void ExtractConfig (Stream output) + { + Store.ExtractAssemblyConfig (this, output); + } + + string MakeOutputFilePath (string outputDirPath, string extension, string? fileName) + { + return Path.Combine (outputDirPath, MakeFileName (extension, fileName)); + } + + string MakeFileName (string extension, string? fileName = null) + { + if (String.IsNullOrEmpty (fileName)) { + fileName = Name; + + if (String.IsNullOrEmpty (fileName)) { + fileName = $"{Hash32:x}_{Hash64:x}"; + } + + fileName = $"{fileName}.{extension}"; + } + + return fileName!; + } + } +} diff --git a/tools/assembly-store-reader/AssemblyStoreExplorer.cs b/tools/assembly-store-reader/AssemblyStoreExplorer.cs new file mode 100644 index 00000000000..ba2f2ae3216 --- /dev/null +++ b/tools/assembly-store-reader/AssemblyStoreExplorer.cs @@ -0,0 +1,329 @@ +using System; +using System.Collections.Generic; +using System.IO; + +using Xamarin.Tools.Zip; + +namespace Xamarin.Android.AssemblyStore +{ + class AssemblyStoreExplorer + { + AssemblyStoreReader? indexStore; + AssemblyStoreManifestReader? manifest; + int numberOfStores = 0; + Action? logger; + bool keepStoreInMemory; + + public IDictionary AssembliesByName { get; } = new SortedDictionary (StringComparer.OrdinalIgnoreCase); + public IDictionary AssembliesByHash32 { get; } = new Dictionary (); + public IDictionary AssembliesByHash64 { get; } = new Dictionary (); + public List Assemblies { get; } = new List (); + public IDictionary> Stores { get; } = new SortedDictionary> (); + public string StorePath { get; } + public string StoreSetName { get; } + public bool HasErrors { get; private set; } + public bool HasWarnings { get; private set; } + + public bool IsCompleteSet => indexStore != null && manifest != null; + public int NumberOfStores => numberOfStores; + + // storePath can point to: + // + // aab + // apk + // index store (e.g. base_assemblies.blob) + // arch store (e.g. base_assemblies.arm64_v8a.blob) + // store manifest (e.g. base_assemblies.manifest) + // store base name (e.g. base or base_assemblies) + // + // In each case the whole set of stores and manifests will be read (if available). Search for the various members of the store set (common/main store, arch stores, + // manifest) is based on this naming convention: + // + // {BASE_NAME}[.ARCH_NAME].{blob|manifest} + // + // Whichever file is referenced in `storePath`, the BASE_NAME component is extracted and all the found files are read. + // If `storePath` points to an aab or an apk, BASE_NAME will always be `assemblies` + // + public AssemblyStoreExplorer (string storePath, Action? customLogger = null, bool keepStoreInMemory = false) + { + if (String.IsNullOrEmpty (storePath)) { + throw new ArgumentException ("must not be null or empty", nameof (storePath)); + } + + if (Directory.Exists (storePath)) { + throw new ArgumentException ($"'{storePath}' points to a directory", nameof (storePath)); + } + + logger = customLogger; + this.keepStoreInMemory = keepStoreInMemory; + StorePath = storePath; + string? extension = Path.GetExtension (storePath); + string? baseName = null; + + if (String.IsNullOrEmpty (extension)) { + baseName = GetBaseNameNoExtension (storePath); + } else { + baseName = GetBaseNameHaveExtension (storePath, extension); + } + + if (String.IsNullOrEmpty (baseName)) { + throw new InvalidOperationException ($"Unable to determine base name of a store set from path '{storePath}'"); + } + + StoreSetName = baseName; + if (!IsAndroidArchive (extension)) { + Logger (AssemblyStoreExplorerLogLevel.Info, $"{storePath} is not an Android archive, reading from filesystem"); + string? directoryName = Path.GetDirectoryName (storePath); + if (String.IsNullOrEmpty (directoryName)) { + directoryName = "."; + } + + ReadStoreSetFromFilesystem (baseName, directoryName); + } else { + Logger (AssemblyStoreExplorerLogLevel.Info, $"{storePath} is an Android archive"); + ReadStoreSetFromArchive (baseName, storePath, extension); + } + + ProcessStores (); + } + + void Logger (AssemblyStoreExplorerLogLevel level, string message) + { + if (level == AssemblyStoreExplorerLogLevel.Error) { + HasErrors = true; + } else if (level == AssemblyStoreExplorerLogLevel.Warning) { + HasWarnings = true; + } + + if (logger != null) { + logger (level, message); + } else { + DefaultLogger (level, message); + } + } + + void DefaultLogger (AssemblyStoreExplorerLogLevel level, string message) + { + Console.WriteLine ($"{level}: {message}"); + } + + void ProcessStores () + { + if (Stores.Count == 0 || indexStore == null) { + return; + } + + ProcessIndex (indexStore.GlobalIndex32, "32", (AssemblyStoreHashEntry he, AssemblyStoreAssembly assembly) => { + assembly.Hash32 = (uint)he.Hash; + assembly.RuntimeIndex = he.MappingIndex; + + if (manifest != null && manifest.EntriesByHash32.TryGetValue (assembly.Hash32, out AssemblyStoreManifestEntry? me) && me != null) { + assembly.Name = me.Name; + } + + if (!AssembliesByHash32.ContainsKey (assembly.Hash32)) { + AssembliesByHash32.Add (assembly.Hash32, assembly); + } + }); + + ProcessIndex (indexStore.GlobalIndex64, "64", (AssemblyStoreHashEntry he, AssemblyStoreAssembly assembly) => { + assembly.Hash64 = he.Hash; + if (assembly.RuntimeIndex != he.MappingIndex) { + Logger (AssemblyStoreExplorerLogLevel.Warning, $"assembly with hashes 0x{assembly.Hash32} and 0x{assembly.Hash64} has a different 32-bit runtime index ({assembly.RuntimeIndex}) than the 64-bit runtime index({he.MappingIndex})"); + } + + if (manifest != null && manifest.EntriesByHash64.TryGetValue (assembly.Hash64, out AssemblyStoreManifestEntry? me) && me != null) { + if (String.IsNullOrEmpty (assembly.Name)) { + Logger (AssemblyStoreExplorerLogLevel.Warning, $"32-bit hash 0x{assembly.Hash32:x} did not match any assembly name in the manifest"); + assembly.Name = me.Name; + if (String.IsNullOrEmpty (assembly.Name)) { + Logger (AssemblyStoreExplorerLogLevel.Warning, $"64-bit hash 0x{assembly.Hash64:x} did not match any assembly name in the manifest"); + } + } else if (String.Compare (assembly.Name, me.Name, StringComparison.Ordinal) != 0) { + Logger (AssemblyStoreExplorerLogLevel.Warning, $"32-bit hash 0x{assembly.Hash32:x} maps to assembly name '{assembly.Name}', however 64-bit hash 0x{assembly.Hash64:x} for the same entry matches assembly name '{me.Name}'"); + } + } + + if (!AssembliesByHash64.ContainsKey (assembly.Hash64)) { + AssembliesByHash64.Add (assembly.Hash64, assembly); + } + }); + + foreach (var kvp in Stores) { + List list = kvp.Value; + if (list.Count < 2) { + continue; + } + + AssemblyStoreReader template = list[0]; + for (int i = 1; i < list.Count; i++) { + AssemblyStoreReader other = list[i]; + if (!template.HasIdenticalContent (other)) { + Logger (AssemblyStoreExplorerLogLevel.Error, $"Store ID {template.StoreID} for architecture {other.Arch} is not identical to other stores with the same ID"); + } + } + } + + void ProcessIndex (List index, string bitness, Action assemblyHandler) + { + foreach (AssemblyStoreHashEntry he in index) { + if (!Stores.TryGetValue (he.StoreID, out List? storeList) || storeList == null) { + Logger (AssemblyStoreExplorerLogLevel.Warning, $"store with id {he.StoreID} not part of the set"); + continue; + } + + foreach (AssemblyStoreReader store in storeList) { + if (he.LocalStoreIndex >= (uint)store.Assemblies.Count) { + Logger (AssemblyStoreExplorerLogLevel.Warning, $"{bitness}-bit index entry with hash 0x{he.Hash:x} has invalid store {store.StoreID} index {he.LocalStoreIndex} (maximum allowed is {store.Assemblies.Count})"); + continue; + } + + AssemblyStoreAssembly assembly = store.Assemblies[(int)he.LocalStoreIndex]; + assemblyHandler (he, assembly); + + if (!AssembliesByName.ContainsKey (assembly.Name)) { + AssembliesByName.Add (assembly.Name, assembly); + } + } + } + } + } + + void ReadStoreSetFromArchive (string baseName, string archivePath, string extension) + { + string basePathInArchive; + + if (String.Compare (".aab", extension, StringComparison.OrdinalIgnoreCase) == 0) { + basePathInArchive = "base/root/assemblies"; + } else if (String.Compare (".apk", extension, StringComparison.OrdinalIgnoreCase) == 0) { + basePathInArchive = "assemblies"; + } else if (String.Compare (".zip", extension, StringComparison.OrdinalIgnoreCase) == 0) { + basePathInArchive = "root/assemblies"; + } else { + throw new InvalidOperationException ($"Unrecognized archive extension '{extension}'"); + } + + basePathInArchive = $"{basePathInArchive}/{baseName}."; + using (ZipArchive archive = ZipArchive.Open (archivePath, FileMode.Open)) { + ReadStoreSetFromArchive (archive, basePathInArchive); + } + } + + void ReadStoreSetFromArchive (ZipArchive archive, string basePathInArchive) + { + foreach (ZipEntry entry in archive) { + if (!entry.FullName.StartsWith (basePathInArchive, StringComparison.Ordinal)) { + continue; + } + + using (var stream = new MemoryStream ()) { + entry.Extract (stream); + + if (entry.FullName.EndsWith (".blob", StringComparison.Ordinal)) { + AddStore (new AssemblyStoreReader (stream, GetStoreArch (entry.FullName), keepStoreInMemory)); + } else if (entry.FullName.EndsWith (".manifest", StringComparison.Ordinal)) { + manifest = new AssemblyStoreManifestReader (stream); + } + } + } + } + + void AddStore (AssemblyStoreReader reader) + { + if (reader.HasGlobalIndex) { + indexStore = reader; + } + + List? storeList; + if (!Stores.TryGetValue (reader.StoreID, out storeList)) { + storeList = new List (); + Stores.Add (reader.StoreID, storeList); + } + storeList.Add (reader); + + Assemblies.AddRange (reader.Assemblies); + } + + string? GetStoreArch (string path) + { + string? arch = Path.GetFileNameWithoutExtension (path); + if (!String.IsNullOrEmpty (arch)) { + arch = Path.GetExtension (arch); + if (!String.IsNullOrEmpty (arch)) { + arch = arch.Substring (1); + } + } + + return arch; + } + + void ReadStoreSetFromFilesystem (string baseName, string setPath) + { + foreach (string de in Directory.EnumerateFiles (setPath, $"{baseName}.*", SearchOption.TopDirectoryOnly)) { + string? extension = Path.GetExtension (de); + if (String.IsNullOrEmpty (extension)) { + continue; + } + + if (String.Compare (".blob", extension, StringComparison.OrdinalIgnoreCase) == 0) { + AddStore (ReadStore (de)); + } else if (String.Compare (".manifest", extension, StringComparison.OrdinalIgnoreCase) == 0) { + manifest = ReadManifest (de); + } + } + + AssemblyStoreReader ReadStore (string filePath) + { + string? arch = GetStoreArch (filePath); + using (var fs = File.OpenRead (filePath)) { + return CreateStoreReader (fs, arch); + } + } + + AssemblyStoreManifestReader ReadManifest (string filePath) + { + using (var fs = File.OpenRead (filePath)) { + return new AssemblyStoreManifestReader (fs); + } + } + } + + AssemblyStoreReader CreateStoreReader (Stream input, string? arch) + { + numberOfStores++; + return new AssemblyStoreReader (input, arch, keepStoreInMemory); + } + + bool IsAndroidArchive (string extension) + { + return + String.Compare (".aab", extension, StringComparison.OrdinalIgnoreCase) == 0 || + String.Compare (".apk", extension, StringComparison.OrdinalIgnoreCase) == 0 || + String.Compare (".zip", extension, StringComparison.OrdinalIgnoreCase) == 0; + } + + string GetBaseNameHaveExtension (string storePath, string extension) + { + if (IsAndroidArchive (extension)) { + return "assemblies"; + } + + string fileName = Path.GetFileNameWithoutExtension (storePath); + int dot = fileName.IndexOf ('.'); + if (dot >= 0) { + return fileName.Substring (0, dot); + } + + return fileName; + } + + string GetBaseNameNoExtension (string storePath) + { + string fileName = Path.GetFileName (storePath); + if (fileName.EndsWith ("_assemblies", StringComparison.OrdinalIgnoreCase)) { + return fileName; + } + return $"{fileName}_assemblies"; + } + } +} diff --git a/tools/assembly-store-reader/AssemblyStoreExplorerLogLevel.cs b/tools/assembly-store-reader/AssemblyStoreExplorerLogLevel.cs new file mode 100644 index 00000000000..19452be507b --- /dev/null +++ b/tools/assembly-store-reader/AssemblyStoreExplorerLogLevel.cs @@ -0,0 +1,10 @@ +namespace Xamarin.Android.AssemblyStore +{ + enum AssemblyStoreExplorerLogLevel + { + Debug, + Info, + Warning, + Error, + } +} diff --git a/tools/assembly-store-reader/AssemblyStoreHashEntry.cs b/tools/assembly-store-reader/AssemblyStoreHashEntry.cs new file mode 100644 index 00000000000..b4504778807 --- /dev/null +++ b/tools/assembly-store-reader/AssemblyStoreHashEntry.cs @@ -0,0 +1,25 @@ +using System; +using System.IO; + +namespace Xamarin.Android.AssemblyStore +{ + class AssemblyStoreHashEntry + { + public bool Is32Bit { get; } + + public ulong Hash { get; } + public uint MappingIndex { get; } + public uint LocalStoreIndex { get; } + public uint StoreID { get; } + + internal AssemblyStoreHashEntry (BinaryReader reader, bool is32Bit) + { + Is32Bit = is32Bit; + + Hash = reader.ReadUInt64 (); + MappingIndex = reader.ReadUInt32 (); + LocalStoreIndex = reader.ReadUInt32 (); + StoreID = reader.ReadUInt32 (); + } + } +} diff --git a/tools/assembly-store-reader/AssemblyStoreManifestEntry.cs b/tools/assembly-store-reader/AssemblyStoreManifestEntry.cs new file mode 100644 index 00000000000..53f2752f3a2 --- /dev/null +++ b/tools/assembly-store-reader/AssemblyStoreManifestEntry.cs @@ -0,0 +1,63 @@ +using System; +using System.Globalization; + +namespace Xamarin.Android.AssemblyStore +{ + class AssemblyStoreManifestEntry + { + // Fields are: + // Hash 32 | Hash 64 | Store ID | Store idx | Name + const int NumberOfFields = 5; + const int Hash32FieldIndex = 0; + const int Hash64FieldIndex = 1; + const int StoreIDFieldIndex = 2; + const int StoreIndexFieldIndex = 3; + const int NameFieldIndex = 4; + + public uint Hash32 { get; } + public ulong Hash64 { get; } + public uint StoreID { get; } + public uint IndexInStore { get; } + public string Name { get; } + + public AssemblyStoreManifestEntry (string[] fields) + { + if (fields.Length != NumberOfFields) { + throw new ArgumentOutOfRangeException (nameof (fields), "Invalid number of fields"); + } + + Hash32 = GetUInt32 (fields[Hash32FieldIndex]); + Hash64 = GetUInt64 (fields[Hash64FieldIndex]); + StoreID = GetUInt32 (fields[StoreIDFieldIndex]); + IndexInStore = GetUInt32 (fields[StoreIndexFieldIndex]); + Name = fields[NameFieldIndex].Trim (); + } + + uint GetUInt32 (string value) + { + if (UInt32.TryParse (PrepHexValue (value), NumberStyles.HexNumber, null, out uint hash)) { + return hash; + } + + return 0; + } + + ulong GetUInt64 (string value) + { + if (UInt64.TryParse (PrepHexValue (value), NumberStyles.HexNumber, null, out ulong hash)) { + return hash; + } + + return 0; + } + + string PrepHexValue (string value) + { + if (value.StartsWith ("0x", StringComparison.Ordinal)) { + return value.Substring (2); + } + + return value; + } + } +} diff --git a/tools/assembly-store-reader/AssemblyStoreManifestReader.cs b/tools/assembly-store-reader/AssemblyStoreManifestReader.cs new file mode 100644 index 00000000000..53a8a3c2e19 --- /dev/null +++ b/tools/assembly-store-reader/AssemblyStoreManifestReader.cs @@ -0,0 +1,48 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Text; + +namespace Xamarin.Android.AssemblyStore +{ + class AssemblyStoreManifestReader + { + static readonly char[] fieldSplit = new char[] { ' ' }; + + public List Entries { get; } = new List (); + public Dictionary EntriesByHash32 { get; } = new Dictionary (); + public Dictionary EntriesByHash64 { get; } = new Dictionary (); + + public AssemblyStoreManifestReader (Stream manifest) + { + manifest.Seek (0, SeekOrigin.Begin); + using (var sr = new StreamReader (manifest, Encoding.UTF8, detectEncodingFromByteOrderMarks: false)) { + ReadManifest (sr); + } + } + + void ReadManifest (StreamReader reader) + { + // First line is ignored, it contains headers + reader.ReadLine (); + + // Each subsequent line consists of fields separated with any number of spaces (for the pleasure of a human being reading the manifest) + while (!reader.EndOfStream) { + string[]? fields = reader.ReadLine ()?.Split (fieldSplit, StringSplitOptions.RemoveEmptyEntries); + if (fields == null) { + continue; + } + + var entry = new AssemblyStoreManifestEntry (fields); + Entries.Add (entry); + if (entry.Hash32 != 0) { + EntriesByHash32.Add (entry.Hash32, entry); + } + + if (entry.Hash64 != 0) { + EntriesByHash64.Add (entry.Hash64, entry); + } + } + } + } +} diff --git a/tools/assembly-store-reader/AssemblyStoreReader.cs b/tools/assembly-store-reader/AssemblyStoreReader.cs new file mode 100644 index 00000000000..8bb36bac01c --- /dev/null +++ b/tools/assembly-store-reader/AssemblyStoreReader.cs @@ -0,0 +1,184 @@ +using System; +using System.Buffers; +using System.Collections.Generic; +using System.IO; +using System.Text; + +namespace Xamarin.Android.AssemblyStore +{ + class AssemblyStoreReader + { + // These two constants must be identical to the native ones in src/monodroid/jni/xamarin-app.hh + const uint ASSEMBLY_STORE_MAGIC = 0x41424158; // 'XABA', little-endian + const uint ASSEMBLY_STORE_FORMAT_VERSION = 1; // The highest format version this reader understands + + MemoryStream? storeData; + + public uint Version { get; private set; } + public uint LocalEntryCount { get; private set; } + public uint GlobalEntryCount { get; private set; } + public uint StoreID { get; private set; } + public List Assemblies { get; } + public List GlobalIndex32 { get; } = new List (); + public List GlobalIndex64 { get; } = new List (); + public string Arch { get; } + + public bool HasGlobalIndex => StoreID == 0; + + public AssemblyStoreReader (Stream store, string? arch = null, bool keepStoreInMemory = false) + { + Arch = arch ?? String.Empty; + + store.Seek (0, SeekOrigin.Begin); + if (keepStoreInMemory) { + storeData = new MemoryStream (); + store.CopyTo (storeData); + storeData.Flush (); + store.Seek (0, SeekOrigin.Begin); + } + + using (var reader = new BinaryReader (store, Encoding.UTF8, leaveOpen: true)) { + ReadHeader (reader); + + Assemblies = new List (); + ReadLocalEntries (reader, Assemblies); + if (HasGlobalIndex) { + ReadGlobalIndex (reader, GlobalIndex32, GlobalIndex64); + } + } + } + + internal void ExtractAssemblyImage (AssemblyStoreAssembly assembly, string outputFilePath) + { + SaveDataToFile (outputFilePath, assembly.DataOffset, assembly.DataSize); + } + + internal void ExtractAssemblyImage (AssemblyStoreAssembly assembly, Stream output) + { + SaveDataToStream (output, assembly.DataOffset, assembly.DataSize); + } + + internal void ExtractAssemblyDebugData (AssemblyStoreAssembly assembly, string outputFilePath) + { + if (assembly.DebugDataOffset == 0 || assembly.DebugDataSize == 0) { + return; + } + SaveDataToFile (outputFilePath, assembly.DebugDataOffset, assembly.DebugDataSize); + } + + internal void ExtractAssemblyDebugData (AssemblyStoreAssembly assembly, Stream output) + { + if (assembly.DebugDataOffset == 0 || assembly.DebugDataSize == 0) { + return; + } + SaveDataToStream (output, assembly.DebugDataOffset, assembly.DebugDataSize); + } + + internal void ExtractAssemblyConfig (AssemblyStoreAssembly assembly, string outputFilePath) + { + if (assembly.ConfigDataOffset == 0 || assembly.ConfigDataSize == 0) { + return; + } + + SaveDataToFile (outputFilePath, assembly.ConfigDataOffset, assembly.ConfigDataSize); + } + + internal void ExtractAssemblyConfig (AssemblyStoreAssembly assembly, Stream output) + { + if (assembly.ConfigDataOffset == 0 || assembly.ConfigDataSize == 0) { + return; + } + SaveDataToStream (output, assembly.ConfigDataOffset, assembly.ConfigDataSize); + } + + void SaveDataToFile (string outputFilePath, uint offset, uint size) + { + EnsureStoreDataAvailable (); + using (var fs = File.Open (outputFilePath, FileMode.Create, FileAccess.Write, FileShare.Read)) { + SaveDataToStream (fs, offset, size); + } + } + + void SaveDataToStream (Stream output, uint offset, uint size) + { + EnsureStoreDataAvailable (); + ArrayPool pool = ArrayPool.Shared; + + storeData!.Seek (offset, SeekOrigin.Begin); + byte[] buf = pool.Rent (16384); + int nread; + long toRead = size; + while (toRead > 0 && (nread = storeData.Read (buf, 0, buf.Length)) > 0) { + if (nread > toRead) { + nread = (int)toRead; + } + + output.Write (buf, 0, nread); + toRead -= nread; + } + output.Flush (); + pool.Return (buf); + } + + void EnsureStoreDataAvailable () + { + if (storeData != null) { + return; + } + + throw new InvalidOperationException ("Store data not available. AssemblyStore/AssemblyStoreExplorer must be instantiated with the `keepStoreInMemory` argument set to `true`"); + } + + public bool HasIdenticalContent (AssemblyStoreReader other) + { + return + other.Version == Version && + other.LocalEntryCount == LocalEntryCount && + other.GlobalEntryCount == GlobalEntryCount && + other.StoreID == StoreID && + other.Assemblies.Count == Assemblies.Count && + other.GlobalIndex32.Count == GlobalIndex32.Count && + other.GlobalIndex64.Count == GlobalIndex64.Count; + } + + void ReadHeader (BinaryReader reader) + { + uint magic = reader.ReadUInt32 (); + if (magic != ASSEMBLY_STORE_MAGIC) { + throw new InvalidOperationException ("Invalid header magic number"); + } + + Version = reader.ReadUInt32 (); + if (Version == 0) { + throw new InvalidOperationException ("Invalid version number: 0"); + } + + if (Version > ASSEMBLY_STORE_FORMAT_VERSION) { + throw new InvalidOperationException ($"Store format version {Version} is higher than the one understood by this reader, {ASSEMBLY_STORE_FORMAT_VERSION}"); + } + + LocalEntryCount = reader.ReadUInt32 (); + GlobalEntryCount = reader.ReadUInt32 (); + StoreID = reader.ReadUInt32 (); + } + + void ReadLocalEntries (BinaryReader reader, List assemblies) + { + for (uint i = 0; i < LocalEntryCount; i++) { + assemblies.Add (new AssemblyStoreAssembly (reader, this)); + } + } + + void ReadGlobalIndex (BinaryReader reader, List index32, List index64) + { + ReadIndex (true, index32); + ReadIndex (true, index64); + + void ReadIndex (bool is32Bit, List index) { + for (uint i = 0; i < GlobalEntryCount; i++) { + index.Add (new AssemblyStoreHashEntry (reader, is32Bit)); + } + } + } + } +} diff --git a/tools/assembly-store-reader/Directory.Build.targets b/tools/assembly-store-reader/Directory.Build.targets new file mode 100644 index 00000000000..e58eed5ca2c --- /dev/null +++ b/tools/assembly-store-reader/Directory.Build.targets @@ -0,0 +1,6 @@ + + + + + diff --git a/tools/assembly-store-reader/Program.cs b/tools/assembly-store-reader/Program.cs new file mode 100644 index 00000000000..d5ee1806f64 --- /dev/null +++ b/tools/assembly-store-reader/Program.cs @@ -0,0 +1,125 @@ +using System; +using System.IO; + +using Mono.Options; + +namespace Xamarin.Android.AssemblyStore +{ + class Program + { + static void ShowAssemblyStoreInfo (string storePath) + { + var explorer = new AssemblyStoreExplorer (storePath); + + string yesno = explorer.IsCompleteSet ? "yes" : "no"; + Console.WriteLine ($"Store set '{explorer.StoreSetName}':"); + Console.WriteLine ($" Is complete set? {yesno}"); + Console.WriteLine ($" Number of stores in the set: {explorer.NumberOfStores}"); + Console.WriteLine (); + Console.WriteLine ("Assemblies:"); + + string infoIndent = " "; + foreach (AssemblyStoreAssembly assembly in explorer.Assemblies) { + Console.WriteLine ($" {assembly.RuntimeIndex}:"); + Console.Write ($"{infoIndent}Name: "); + if (String.IsNullOrEmpty (assembly.Name)) { + Console.WriteLine ("unknown"); + } else { + Console.WriteLine (assembly.Name); + } + + Console.Write ($"{infoIndent}Store ID: {assembly.Store.StoreID} ("); + if (String.IsNullOrEmpty (assembly.Store.Arch)) { + Console.Write ("shared"); + } else { + Console.Write (assembly.Store.Arch); + } + Console.WriteLine (")"); + + Console.Write ($"{infoIndent}Hashes: 32-bit == "); + WriteHashValue (assembly.Hash32); + + Console.Write ("; 64-bit == "); + WriteHashValue (assembly.Hash64); + Console.WriteLine (); + + Console.WriteLine ($"{infoIndent}Assembly image: offset == {assembly.DataOffset}; size == {assembly.DataSize}"); + WriteOptionalDataLine ("Debug data", assembly.DebugDataOffset, assembly.DebugDataOffset); + WriteOptionalDataLine ("Config file", assembly.ConfigDataOffset, assembly.ConfigDataSize); + + Console.WriteLine (); + } + + void WriteOptionalDataLine (string label, uint offset, uint size) + { + Console.Write ($"{infoIndent}{label}: "); + if (offset == 0) { + Console.WriteLine ("absent"); + } else { + Console.WriteLine ("offset == {offset}; size == {size}"); + } + } + + void WriteHashValue (ulong hash) + { + if (hash == 0) { + Console.Write ("unknown"); + } else { + Console.Write ($"0x{hash:x}"); + } + } + } + + static int Main (string[] args) + { + if (args.Length == 0) { + Console.Error.WriteLine ("Usage: read-assembly-store BLOB_PATH [BLOB_PATH ...]"); + Console.Error.WriteLine (); + Console.Error.WriteLine (@" where each BLOB_PATH can point to: + * aab file + * apk file + * index store file (e.g. base_assemblies.blob) + * arch store file (e.g. base_assemblies.arm64_v8a.blob) + * store manifest file (e.g. base_assemblies.manifest) + * store base name (e.g. base or base_assemblies) + + In each case the whole set of stores and manifests will be read (if available). Search for the + various members of the store set (common/main store, arch stores, manifest) is based on this naming + convention: + + {BASE_NAME}[.ARCH_NAME].{blob|manifest} + + Whichever file is referenced in `BLOB_PATH`, the BASE_NAME component is extracted and all the found files are read. + If `BLOB_PATH` points to an aab or an apk, BASE_NAME will always be `assemblies` + +"); + return 1; + } + + bool first = true; + foreach (string path in args) { + ShowAssemblyStoreInfo (path); + if (first) { + first = false; + continue; + } + + Console.WriteLine (); + Console.WriteLine ("***********************************"); + Console.WriteLine (); + } + + return 0; + } + + static void WriteAssemblySegment (string label, uint offset, uint size) + { + if (offset == 0) { + Console.Write ($"no {label}"); + return; + } + + Console.Write ($"{label} starts at {offset}, {size} bytes"); + } + } +} diff --git a/tools/assembly-store-reader/assembly-store-reader.csproj b/tools/assembly-store-reader/assembly-store-reader.csproj new file mode 100644 index 00000000000..66b47ed293b --- /dev/null +++ b/tools/assembly-store-reader/assembly-store-reader.csproj @@ -0,0 +1,24 @@ + + + + + Microsoft Corporation + 2021 Microsoft Corporation + 0.0.1 + false + ../../bin/$(Configuration)/bin/assembly-store-reader + Exe + net6.0 + Xamarin.Android.AssemblyStoreReader + disable + enable + + + + + + + + + + diff --git a/tools/decompress-assemblies/decompress-assemblies.csproj b/tools/decompress-assemblies/decompress-assemblies.csproj index e79f0d93510..a21f994bcc8 100644 --- a/tools/decompress-assemblies/decompress-assemblies.csproj +++ b/tools/decompress-assemblies/decompress-assemblies.csproj @@ -16,6 +16,10 @@ + + + + diff --git a/tools/decompress-assemblies/main.cs b/tools/decompress-assemblies/main.cs index 749c3a691db..c69f50c6625 100644 --- a/tools/decompress-assemblies/main.cs +++ b/tools/decompress-assemblies/main.cs @@ -4,6 +4,7 @@ using K4os.Compression.LZ4; using Xamarin.Tools.Zip; +using Xamarin.Android.AssemblyStore; namespace Xamarin.Android.Tools.DecompressAssemblies { @@ -17,14 +18,14 @@ static int Usage () { Console.WriteLine ("Usage: decompress-assemblies {file.{dll,apk,aab}} [{file.{dll,apk,aab} ...]"); Console.WriteLine (); - Console.WriteLine ("DLL files passed on command are uncompressed to the current directory with the `uncompressed-` prefix added to their name."); + Console.WriteLine ("DLL files passed on command line are uncompressed to the current directory with the `uncompressed-` prefix added to their name."); Console.WriteLine ("DLL files from AAB/APK archives are uncompressed to a subdirectory of the current directory named after the archive with extension removed"); return 1; } static bool UncompressDLL (Stream inputStream, string fileName, string filePath, string prefix) { - string outputFile = $"{prefix}{Path.GetFileName (filePath)}"; + string outputFile = $"{prefix}{filePath}"; bool retVal = true; Console.WriteLine ($"Processing {fileName}"); @@ -55,7 +56,7 @@ static bool UncompressDLL (Stream inputStream, string fileName, string filePath, Directory.CreateDirectory (outputDir); } using (var fs = File.Open (outputFile, FileMode.Create, FileAccess.Write)) { - fs.Write (assemblyBytes, 0, assemblyBytes.Length); + fs.Write (assemblyBytes, 0, decoded); fs.Flush (); } Console.WriteLine ($" uncompressed to: {outputFile}"); @@ -74,34 +75,64 @@ static bool UncompressDLL (Stream inputStream, string fileName, string filePath, static bool UncompressDLL (string filePath, string prefix) { using (var fs = File.Open (filePath, FileMode.Open, FileAccess.Read)) { - return UncompressDLL (fs, filePath, filePath, prefix); + return UncompressDLL (fs, filePath, Path.GetFileName (filePath), prefix); } } - static bool UncompressFromAPK (string filePath, string assembliesPath) + static bool UncompressFromAPK_IndividualEntries (ZipArchive apk, string filePath, string assembliesPath, string prefix) { - string prefix = $"uncompressed-{Path.GetFileNameWithoutExtension (filePath)}{Path.DirectorySeparatorChar}"; - using (ZipArchive apk = ZipArchive.Open (filePath, FileMode.Open)) { - foreach (ZipEntry entry in apk) { - if (!entry.FullName.StartsWith (assembliesPath, StringComparison.Ordinal)) { - continue; - } + foreach (ZipEntry entry in apk) { + if (!entry.FullName.StartsWith (assembliesPath, StringComparison.Ordinal)) { + continue; + } - if (!entry.FullName.EndsWith (".dll", StringComparison.Ordinal)) { - continue; - } + if (!entry.FullName.EndsWith (".dll", StringComparison.Ordinal)) { + continue; + } - using (var stream = new MemoryStream ()) { - entry.Extract (stream); - stream.Seek (0, SeekOrigin.Begin); - UncompressDLL (stream, $"{filePath}!{entry.FullName}", entry.FullName, prefix); - } + using (var stream = new MemoryStream ()) { + entry.Extract (stream); + stream.Seek (0, SeekOrigin.Begin); + string fileName = entry.FullName.Substring (assembliesPath.Length); + UncompressDLL (stream, $"{filePath}!{entry.FullName}", fileName, prefix); } } return true; } + static bool UncompressFromAPK_AssemblyStores (string filePath, string prefix) + { + var explorer = new AssemblyStoreExplorer (filePath, keepStoreInMemory: true); + foreach (AssemblyStoreAssembly assembly in explorer.Assemblies) { + string assemblyName = assembly.DllName; + + if (!String.IsNullOrEmpty (assembly.Store.Arch)) { + assemblyName = $"{assembly.Store.Arch}/{assemblyName}"; + } + + using (var stream = new MemoryStream ()) { + assembly.ExtractImage (stream); + stream.Seek (0, SeekOrigin.Begin); + UncompressDLL (stream, $"{filePath}!{assemblyName}", assemblyName, prefix); + } + } + + return true; + } + + static bool UncompressFromAPK (string filePath, string assembliesPath) + { + string prefix = $"uncompressed-{Path.GetFileNameWithoutExtension (filePath)}{Path.DirectorySeparatorChar}"; + using (ZipArchive apk = ZipArchive.Open (filePath, FileMode.Open)) { + if (!apk.ContainsEntry ($"{assembliesPath}assemblies.blob")) { + return UncompressFromAPK_IndividualEntries (apk, filePath, assembliesPath, prefix); + } + } + + return UncompressFromAPK_AssemblyStores (filePath, prefix); + } + static int Main (string[] args) { if (args.Length == 0) { diff --git a/tools/scripts/read-assembly-store b/tools/scripts/read-assembly-store new file mode 100755 index 00000000000..3386ec91783 --- /dev/null +++ b/tools/scripts/read-assembly-store @@ -0,0 +1,10 @@ +#!/bin/bash +truepath=$(readlink "$0" || echo "$0") +mydir=$(dirname ${truepath}) +binariesdir="${mydir}/assembly-store-reader" + +if [ -x "${binariesdir}/assembly-store-reader" ]; then + exec "${binariesdir}/assembly-store-reader" "$@" +else + exec dotnet "${binariesdir}/assembly-store-reader.dll" +fi diff --git a/tools/tmt/tmt.csproj b/tools/tmt/tmt.csproj index 8b98c4a56e6..998c541c70d 100644 --- a/tools/tmt/tmt.csproj +++ b/tools/tmt/tmt.csproj @@ -10,6 +10,7 @@ Exe true enable + Major