From 88207fd5b7ca8b1d30bdd61848e251a27fbdbc20 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Wed, 15 Apr 2020 19:08:04 +0200 Subject: [PATCH 01/71] JetBrains OpenJDK 11 This PR is a test to see what works and what breaks with JetBrains OpenJDK 11 as far as Xamarin.Android is concerned. --- .../ConfigAndData/Configurables.Linux.cs | 2 + .../ConfigAndData/Configurables.MacOS.cs | 1 + .../ConfigAndData/Configurables.Windows.cs | 2 + .../xaprepare/ConfigAndData/Configurables.cs | 22 +- .../Scenarios/Scenario_AndroidToolchain.cs | 4 +- .../xaprepare/Scenarios/Scenario_Standard.cs | 2 +- .../Step_InstallJetBrainsOpenJDK.Linux.cs | 13 + .../Step_InstallJetBrainsOpenJDK.MacOS.cs | 14 ++ .../Step_InstallJetBrainsOpenJDK.Unix.cs | 13 + .../Step_InstallJetBrainsOpenJDK.Windows.cs | 43 ++++ .../Steps/Step_InstallJetBrainsOpenJDK.cs | 237 ++++++++++++++++++ .../Step_PrepareExternalJavaInterop.Unix.cs | 1 + .../xaprepare/xaprepare/xaprepare.csproj | 5 + external/xamarin-android-tools | 2 +- .../targets/Xamarin.Android.Sdk.props | 2 +- 15 files changed, 356 insertions(+), 7 deletions(-) create mode 100644 build-tools/xaprepare/xaprepare/Steps/Step_InstallJetBrainsOpenJDK.Linux.cs create mode 100644 build-tools/xaprepare/xaprepare/Steps/Step_InstallJetBrainsOpenJDK.MacOS.cs create mode 100644 build-tools/xaprepare/xaprepare/Steps/Step_InstallJetBrainsOpenJDK.Unix.cs create mode 100644 build-tools/xaprepare/xaprepare/Steps/Step_InstallJetBrainsOpenJDK.Windows.cs create mode 100644 build-tools/xaprepare/xaprepare/Steps/Step_InstallJetBrainsOpenJDK.cs diff --git a/build-tools/xaprepare/xaprepare/ConfigAndData/Configurables.Linux.cs b/build-tools/xaprepare/xaprepare/ConfigAndData/Configurables.Linux.cs index d65f02e2955..4cdd9f67ab7 100644 --- a/build-tools/xaprepare/xaprepare/ConfigAndData/Configurables.Linux.cs +++ b/build-tools/xaprepare/xaprepare/ConfigAndData/Configurables.Linux.cs @@ -4,6 +4,8 @@ namespace Xamarin.Android.Prepare { partial class Configurables { + const string JetBrainsOpenJDKOperatingSystem = "linux-x64"; + partial class Urls { public static readonly Uri Corretto = new Uri ($"{Corretto_BaseUri}{CorrettoUrlPathVersion}/amazon-corretto-{CorrettoDistVersion}-linux-x64.tar.gz"); diff --git a/build-tools/xaprepare/xaprepare/ConfigAndData/Configurables.MacOS.cs b/build-tools/xaprepare/xaprepare/ConfigAndData/Configurables.MacOS.cs index 5b75ed09f4b..bf86871d059 100644 --- a/build-tools/xaprepare/xaprepare/ConfigAndData/Configurables.MacOS.cs +++ b/build-tools/xaprepare/xaprepare/ConfigAndData/Configurables.MacOS.cs @@ -4,6 +4,7 @@ namespace Xamarin.Android.Prepare { partial class Configurables { + const string JetBrainsOpenJDKOperatingSystem = "osx-x64"; partial class Urls { public static readonly Uri Corretto = new Uri ($"{Corretto_BaseUri}{CorrettoUrlPathVersion}/amazon-corretto-{CorrettoDistVersion}-macosx-x64.tar.gz"); diff --git a/build-tools/xaprepare/xaprepare/ConfigAndData/Configurables.Windows.cs b/build-tools/xaprepare/xaprepare/ConfigAndData/Configurables.Windows.cs index f11e83793ce..fc1e7d19b70 100644 --- a/build-tools/xaprepare/xaprepare/ConfigAndData/Configurables.Windows.cs +++ b/build-tools/xaprepare/xaprepare/ConfigAndData/Configurables.Windows.cs @@ -5,6 +5,8 @@ namespace Xamarin.Android.Prepare { partial class Configurables { + const string JetBrainsOpenJDKOperatingSystem = "windows-x64"; + partial class Urls { public static Uri Corretto => GetWindowsCorrettoUrl (); diff --git a/build-tools/xaprepare/xaprepare/ConfigAndData/Configurables.cs b/build-tools/xaprepare/xaprepare/ConfigAndData/Configurables.cs index d7a9483f748..203f361a1b9 100644 --- a/build-tools/xaprepare/xaprepare/ConfigAndData/Configurables.cs +++ b/build-tools/xaprepare/xaprepare/ConfigAndData/Configurables.cs @@ -15,6 +15,10 @@ namespace Xamarin.Android.Prepare // partial class Configurables { + const string JetBrainsOpenJDKVersion = "11.0.4"; + const string JetBrainsOpenJDKRelease = "546.1"; + static readonly string JetBrainsOpenJDKDownloadVersion = JetBrainsOpenJDKVersion.Replace ('.', '_'); + const string CorrettoDistVersion = "8.242.08.1"; const string CorrettoUrlPathVersion = CorrettoDistVersion; @@ -22,6 +26,11 @@ partial class Configurables public static partial class Urls { + // https://bintray.com/jetbrains/intellij-jdk/download_file?file_path=jbrsdk-11_0_4-linux-x64-b546.1.tar.gz + // https://bintray.com/jetbrains/intellij-jdk/download_file?file_path=jbrsdk-11_0_4-osx-x64-b546.1.tar.gz + // https://bintray.com/jetbrains/intellij-jdk/download_file?file_path=jbrsdk-11_0_4-windows-x64-b546.1.tar.gz + public static readonly Uri JetBrainsOpenJDK = new Uri ($"https://bintray.com/jetbrains/intellij-jdk/download_file?file_path=jbrsdk-{JetBrainsOpenJDKDownloadVersion}-{JetBrainsOpenJDKOperatingSystem}-b{JetBrainsOpenJDKRelease}.tar.gz"); + // Keep the trailing slash here - OS-specific code assumes it's there. public const string Corretto_BaseUri = "https://corretto.aws/downloads/resources/"; @@ -39,6 +48,9 @@ public static partial class Defaults { public static readonly char[] PropertyListSeparator = new [] { ':' }; + public static readonly Version JetBrainsOpenJDKVersion = new Version (Configurables.JetBrainsOpenJDKVersion); + public static readonly Version JetBrainsOpenJDKRelease = new Version (Configurables.JetBrainsOpenJDKRelease); + // Mono runtimes public const string DebugFileExtension = ".pdb"; public const string MonoHostMingwRuntimeNativeLibraryExtension = WindowsDLLSuffix; @@ -104,9 +116,9 @@ public static partial class Defaults public const int DefaultMaximumParallelTasks = 5; /// - /// The maximum JDK version we support. Note: this will probably go away with Corretto + /// The maximum JDK version we support. /// - public const int MaxJDKVersion = 8; + public static readonly Version MaxJDKVersion = new Version (11, 99, 0); /// /// Prefix for all the log files created by the bootstrapper. @@ -318,6 +330,10 @@ public static partial class Paths public static string CorrettoCacheDir => GetCachedPath (ref correttoCacheDir, () => ctx.Properties.GetRequiredValue (KnownProperties.AndroidToolchainCacheDirectory)); public static string CorrettoInstallDir => GetCachedPath (ref correttoInstallDir, () => Path.Combine (ctx.Properties.GetRequiredValue (KnownProperties.AndroidToolchainDirectory), "jdk")); + // JetBrains OpenJDK + public static string OpenJDKInstallDir => GetCachedPath (ref openJDKInstallDir, () => Path.Combine (ctx.Properties.GetRequiredValue (KnownProperties.AndroidToolchainDirectory), "jdk")); + public static string OpenJDKCacheDir => GetCachedPath (ref openJDKCacheDir, () => ctx.Properties.GetRequiredValue (KnownProperties.AndroidToolchainCacheDirectory)); + // bundle public static string BCLTestsArchiveName = "bcl-tests.zip"; @@ -403,6 +419,8 @@ static string GetCachedPath (ref string variable, Func creator) static string monoSdksTpnExternalPath; static string monoSDKSIncludeDestDir; static string monoLlvmTpnPath; + static string openJDKInstallDir; + static string openJDKCacheDir; } } } diff --git a/build-tools/xaprepare/xaprepare/Scenarios/Scenario_AndroidToolchain.cs b/build-tools/xaprepare/xaprepare/Scenarios/Scenario_AndroidToolchain.cs index 9ef78958a54..687ea5d09da 100644 --- a/build-tools/xaprepare/xaprepare/Scenarios/Scenario_AndroidToolchain.cs +++ b/build-tools/xaprepare/xaprepare/Scenarios/Scenario_AndroidToolchain.cs @@ -6,12 +6,12 @@ namespace Xamarin.Android.Prepare partial class Scenario_AndroidToolchain : ScenarioNoStandardEndSteps { public Scenario_AndroidToolchain () - : base ("AndroidToolchain", "Install Android SDK, NDK and Corretto JDK.", Context.Instance) + : base ("AndroidToolchain", "Install Android SDK, NDK and OpenJDK.", Context.Instance) {} protected override void AddSteps (Context context) { - Steps.Add (new Step_InstallCorrettoOpenJDK ()); + Steps.Add (new Step_InstallJetBrainsOpenJDK ()); Steps.Add (new Step_Android_SDK_NDK ()); // disable installation of missing programs... diff --git a/build-tools/xaprepare/xaprepare/Scenarios/Scenario_Standard.cs b/build-tools/xaprepare/xaprepare/Scenarios/Scenario_Standard.cs index e6a239f64b5..ecc5a4aed34 100644 --- a/build-tools/xaprepare/xaprepare/Scenarios/Scenario_Standard.cs +++ b/build-tools/xaprepare/xaprepare/Scenarios/Scenario_Standard.cs @@ -19,7 +19,7 @@ protected override void AddSteps (Context context) throw new ArgumentNullException (nameof (context)); Steps.Add (new Step_ShowEnabledRuntimes ()); - Steps.Add (new Step_InstallCorrettoOpenJDK ()); + Steps.Add (new Step_InstallJetBrainsOpenJDK ()); Steps.Add (new Step_Android_SDK_NDK ()); Steps.Add (new Step_GenerateFiles (atBuildStart: true)); Steps.Add (new Step_PrepareProps ()); diff --git a/build-tools/xaprepare/xaprepare/Steps/Step_InstallJetBrainsOpenJDK.Linux.cs b/build-tools/xaprepare/xaprepare/Steps/Step_InstallJetBrainsOpenJDK.Linux.cs new file mode 100644 index 00000000000..d41d9cf0ad8 --- /dev/null +++ b/build-tools/xaprepare/xaprepare/Steps/Step_InstallJetBrainsOpenJDK.Linux.cs @@ -0,0 +1,13 @@ +using System; +using System.Threading.Tasks; + +namespace Xamarin.Android.Prepare +{ + partial class Step_InstallJetBrainsOpenJDK + { + void MoveContents (string sourceDir, string destinationDir) + { + Utilities.MoveDirectoryContentsRecursively (sourceDir, destinationDir); + } + } +} diff --git a/build-tools/xaprepare/xaprepare/Steps/Step_InstallJetBrainsOpenJDK.MacOS.cs b/build-tools/xaprepare/xaprepare/Steps/Step_InstallJetBrainsOpenJDK.MacOS.cs new file mode 100644 index 00000000000..0cd7d04819d --- /dev/null +++ b/build-tools/xaprepare/xaprepare/Steps/Step_InstallJetBrainsOpenJDK.MacOS.cs @@ -0,0 +1,14 @@ +using System; +using System.IO; + +namespace Xamarin.Android.Prepare +{ + partial class Step_InstallJetBrainsOpenJDK + { + void MoveContents (string sourceDir, string destinationDir) + { + string realSourceDir = Path.Combine (sourceDir, "Contents", "Home"); + Utilities.MoveDirectoryContentsRecursively (realSourceDir, destinationDir); + } + } +} diff --git a/build-tools/xaprepare/xaprepare/Steps/Step_InstallJetBrainsOpenJDK.Unix.cs b/build-tools/xaprepare/xaprepare/Steps/Step_InstallJetBrainsOpenJDK.Unix.cs new file mode 100644 index 00000000000..951f837aad6 --- /dev/null +++ b/build-tools/xaprepare/xaprepare/Steps/Step_InstallJetBrainsOpenJDK.Unix.cs @@ -0,0 +1,13 @@ +using System; +using System.Threading.Tasks; + +namespace Xamarin.Android.Prepare +{ + partial class Step_InstallJetBrainsOpenJDK + { + async Task Unpack (string fullArchivePath, string destinationDirectory, bool cleanDestinationBeforeUnpacking = false) + { + return await Utilities.Unpack (fullArchivePath, destinationDirectory, cleanDestinatioBeforeUnpacking: true); + } + } +} diff --git a/build-tools/xaprepare/xaprepare/Steps/Step_InstallJetBrainsOpenJDK.Windows.cs b/build-tools/xaprepare/xaprepare/Steps/Step_InstallJetBrainsOpenJDK.Windows.cs new file mode 100644 index 00000000000..ae9aa1e010c --- /dev/null +++ b/build-tools/xaprepare/xaprepare/Steps/Step_InstallJetBrainsOpenJDK.Windows.cs @@ -0,0 +1,43 @@ +using System; +using System.IO; +using System.Threading.Tasks; + +namespace Xamarin.Android.Prepare +{ + partial class Step_InstallJetBrainsOpenJDK + { + async Task Unpack (string fullArchivePath, string destinationDirectory, bool cleanDestinationBeforeUnpacking = false) + { + // On Windows we don't have Tar available and the Windows package is a .tar.gz + // 7zip can unpack tar.gz but it's a two-stage process - first it decompresses the package, then it can be + // invoked again to extract the actual tar contents. + + if (cleanDestinationBeforeUnpacking) + Utilities.DeleteDirectorySilent (destinationDirectory); + Utilities.CreateDirectory (destinationDirectory); + + var sevenZip = new SevenZipRunner (Context.Instance); + Log.DebugLine ($"Uncompressing {fullArchivePath} to {destinationDirectory}"); + if (!await sevenZip.Extract (fullArchivePath, destinationDirectory)) { + Log.DebugLine ($"Failed to decompress {fullArchivePath}"); + return false; + } + + string tarPath = Path.Combine (destinationDirectory, Path.GetFileNameWithoutExtension (fullArchivePath)); + bool ret = await sevenZip.Extract (tarPath, destinationDirectory); + Utilities.DeleteFileSilent (tarPath); + + if (!ret) { + Log.DebugLine ($"Failed to extract TAR contents from {tarPath}"); + return false; + } + + return true; + } + + void MoveContents (string sourceDir, string destinationDir) + { + Utilities.MoveDirectoryContentsRecursively (sourceDir, destinationDir); + } + } +} diff --git a/build-tools/xaprepare/xaprepare/Steps/Step_InstallJetBrainsOpenJDK.cs b/build-tools/xaprepare/xaprepare/Steps/Step_InstallJetBrainsOpenJDK.cs new file mode 100644 index 00000000000..80226a609a8 --- /dev/null +++ b/build-tools/xaprepare/xaprepare/Steps/Step_InstallJetBrainsOpenJDK.cs @@ -0,0 +1,237 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Net; +using System.Reflection; +using System.Threading.Tasks; +using System.Xml.Linq; + +namespace Xamarin.Android.Prepare +{ + partial class Step_InstallJetBrainsOpenJDK : StepWithDownloadProgress + { + const string ProductName = "JetBrains OpenJDK"; + const string RootDirName = "jbrsdk"; + const string XAVersionInfoFile = "xa_jdk_version.txt"; + const string URLQueryFilePathField = "file_path"; + + static readonly char[] QuerySeparator = new char[] { ';', '&' }; + + // Paths relative to JDK installation root, just for a cursory check whether we have a sane JDK instance + // NOTE: file extensions are not necessary here + static readonly List jdkFiles = new List { + Path.Combine ("bin", "java"), + Path.Combine ("bin", "javac"), + Path.Combine ("include", "jni.h"), + }; + + public Step_InstallJetBrainsOpenJDK () + : base ($"Installing {ProductName} 11") + {} + + protected override async Task Execute (Context context) + { + string jdkInstallDir = Configurables.Paths.OpenJDKInstallDir; + if (OpenJDKExistsAndIsValid (jdkInstallDir, out string installedVersion)) { + Log.Status ($"{ProductName} version "); + Log.Status (installedVersion, ConsoleColor.Yellow); + Log.StatusLine (" already installed in: ", jdkInstallDir, tailColor: ConsoleColor.Cyan); + return true; + } + + Log.StatusLine ($"JetBrains JDK {Configurables.Defaults.JetBrainsOpenJDKVersion} r{Configurables.Defaults.JetBrainsOpenJDKRelease} will be installed"); + Uri jdkURL = Configurables.Urls.JetBrainsOpenJDK; + if (jdkURL == null) + throw new InvalidOperationException ($"{ProductName} URL must not be null"); + + string[] queryParams = jdkURL.Query.TrimStart ('?').Split (QuerySeparator, StringSplitOptions.RemoveEmptyEntries); + if (queryParams.Length == 0) { + Log.ErrorLine ($"Unable to extract file name from {ProductName} URL as it contains no query component"); + return false; + } + + string packageName = null; + foreach (string p in queryParams) { + if (!p.StartsWith (URLQueryFilePathField, StringComparison.Ordinal)) { + continue; + } + + int idx = p.IndexOf ('='); + if (idx < 0) { + Log.DebugLine ($"{ProductName} URL query field '{URLQueryFilePathField}' has no value, unable to detect file name"); + break; + } + + packageName = p.Substring (idx + 1).Trim (); + } + + if (String.IsNullOrEmpty (packageName)) { + Log.ErrorLine ($"Unable to extract file name from {ProductName} URL"); + return false; + } + + string localPackagePath = Path.Combine (Configurables.Paths.OpenJDKCacheDir, packageName); + if (!await DownloadOpenJDK (context, localPackagePath, jdkURL)) + return false; + + string tempDir = $"{jdkInstallDir}.temp"; + try { + if (!await Unpack (localPackagePath, tempDir, cleanDestinationBeforeUnpacking: true)) { + Log.ErrorLine ($"Failed to install {ProductName}"); + return false; + } + + string rootDir = Path.Combine (tempDir, RootDirName); + if (!Directory.Exists (rootDir)) { + Log.ErrorLine ($"JetBrains root directory not found after unpacking: {RootDirName}"); + return false; + } + + MoveContents (rootDir, jdkInstallDir); + File.WriteAllText (Path.Combine (jdkInstallDir, XAVersionInfoFile), $"{Configurables.Defaults.JetBrainsOpenJDKRelease}{Environment.NewLine}"); + } finally { + Utilities.DeleteDirectorySilent (tempDir); + // Clean up zip after extraction if running on a hosted azure pipelines agent. + if (context.IsRunningOnHostedAzureAgent) + Utilities.DeleteFileSilent (localPackagePath); + } + + return true; + } + + async Task DownloadOpenJDK (Context context, string localPackagePath, Uri url) + { + if (File.Exists (localPackagePath)) { + Log.StatusLine ($"{ProductName} archive already downloaded"); + return true; + } + + Log.StatusLine ($"Downloading {ProductName} from ", url.ToString (), tailColor: ConsoleColor.White); + (bool success, ulong size, HttpStatusCode status) = await Utilities.GetDownloadSizeWithStatus (url); + if (!success) { + if (status == HttpStatusCode.NotFound) + Log.ErrorLine ($"{ProductName} archive URL not found"); + else + Log.ErrorLine ($"Failed to obtain {ProductName} size. HTTP status code: {status} ({(int)status})"); + return false; + } + + DownloadStatus downloadStatus = Utilities.SetupDownloadStatus (context, size, context.InteractiveSession); + Log.StatusLine ($" {context.Characters.Link} {url}", ConsoleColor.White); + await Download (context, url, localPackagePath, ProductName, Path.GetFileName (localPackagePath), downloadStatus); + + if (!File.Exists (localPackagePath)) { + Log.ErrorLine ($"Download of {ProductName} from {url} failed."); + return false; + } + + return true; + } + + bool OpenJDKExistsAndIsValid (string installDir, out string installedVersion) + { + installedVersion = null; + if (!Directory.Exists (installDir)) { + Log.DebugLine ($"{ProductName} directory {installDir} does not exist"); + return false; + } + + string corettoVersionFile = Path.Combine (installDir, "version.txt"); + if (File.Exists (corettoVersionFile)) { + Log.DebugLine ($"Corretto version file {corettoVersionFile} found, will replace Corretto with {ProductName}"); + return false; + } + + string jetBrainsReleaseFile = Path.Combine (installDir, "release"); + if (!File.Exists (jetBrainsReleaseFile)) { + Log.DebugLine ($"{ProductName} release file {jetBrainsReleaseFile} does not exist, cannot determine version"); + return false; + } + + string[] lines = File.ReadAllLines (jetBrainsReleaseFile); + if (lines == null || lines.Length == 0) { + Log.DebugLine ($"{ProductName} release file {jetBrainsReleaseFile} is empty, cannot determine version"); + return false; + } + + string cv = null; + foreach (string l in lines) { + string line = l.Trim (); + if (!line.StartsWith ("JAVA_VERSION=", StringComparison.Ordinal)) { + continue; + } + + cv = line.Substring (line.IndexOf ('=') + 1).Trim ('"'); + break; + } + + if (String.IsNullOrEmpty (cv)) { + Log.DebugLine ($"Unable to find version of {ProductName} in release file {jetBrainsReleaseFile}"); + return false; + } + + string xaVersionFile = Path.Combine (installDir, XAVersionInfoFile); + if (!File.Exists (xaVersionFile)) { + installedVersion = cv; + Log.DebugLine ($"Unable to find Xamarin.Android version file {xaVersionFile}"); + return false; + } + + lines = File.ReadAllLines (xaVersionFile); + if (lines == null || lines.Length == 0) { + Log.DebugLine ($"Xamarin.Android version file {xaVersionFile} is empty, cannot determine release version"); + return false; + } + + string rv = lines[0].Trim (); + if (String.IsNullOrEmpty (rv)) { + Log.DebugLine ($"Xamarin.Android version file {xaVersionFile} does not contain release version information"); + return false; + } + + installedVersion = $"{cv} r{rv}"; + + if (!Version.TryParse (cv, out Version cversion)) { + Log.DebugLine ($"Unable to parse {ProductName} version from: {cv}"); + return false; + } + + if (cversion != Configurables.Defaults.JetBrainsOpenJDKVersion) { + Log.DebugLine ($"Invalid {ProductName} version. Need {Configurables.Defaults.JetBrainsOpenJDKVersion}, found {cversion}"); + return false; + } + + if (!Version.TryParse (rv, out cversion)) { + Log.DebugLine ($"Unable to parse {ProductName} release version from: {rv}"); + return false; + } + + if (cversion != Configurables.Defaults.JetBrainsOpenJDKRelease) { + Log.DebugLine ($"Invalid {ProductName} version. Need {Configurables.Defaults.JetBrainsOpenJDKRelease}, found {cversion}"); + return false; + } + + foreach (string f in jdkFiles) { + string file = Path.Combine (installDir, f); + if (!File.Exists (file)) { + bool foundExe = false; + foreach (string exe in Utilities.FindExecutable (f)) { + file = Path.Combine (installDir, exe); + if (File.Exists (file)) { + foundExe = true; + break; + } + } + + if (!foundExe) { + Log.DebugLine ($"JDK file {file} missing from {ProductName}"); + return false; + } + } + } + + return true; + } + } +} diff --git a/build-tools/xaprepare/xaprepare/Steps/Step_PrepareExternalJavaInterop.Unix.cs b/build-tools/xaprepare/xaprepare/Steps/Step_PrepareExternalJavaInterop.Unix.cs index 2da742d8e1c..ce59185c1fb 100644 --- a/build-tools/xaprepare/xaprepare/Steps/Step_PrepareExternalJavaInterop.Unix.cs +++ b/build-tools/xaprepare/xaprepare/Steps/Step_PrepareExternalJavaInterop.Unix.cs @@ -18,6 +18,7 @@ async Task ExecuteOSSpecific (Context context) workingDirectory: javaInteropDir, arguments: new List { "prepare", + "V=1", $"CONFIGURATION={context.Configuration}", $"JAVA_HOME={context.OS.JavaHome}", $"JI_MAX_JDK={Configurables.Defaults.MaxJDKVersion}", diff --git a/build-tools/xaprepare/xaprepare/xaprepare.csproj b/build-tools/xaprepare/xaprepare/xaprepare.csproj index ecf2e7b9adc..4ece6942c9b 100644 --- a/build-tools/xaprepare/xaprepare/xaprepare.csproj +++ b/build-tools/xaprepare/xaprepare/xaprepare.csproj @@ -138,6 +138,7 @@ + @@ -195,6 +196,7 @@ + @@ -220,6 +222,7 @@ + @@ -237,6 +240,7 @@ + @@ -259,6 +263,7 @@ + diff --git a/external/xamarin-android-tools b/external/xamarin-android-tools index 12f52acd440..36d7fee5f7e 160000 --- a/external/xamarin-android-tools +++ b/external/xamarin-android-tools @@ -1 +1 @@ -Subproject commit 12f52acd4407323b2e1b48ca56bab80ffd97b31e +Subproject commit 36d7fee5f7e0ef6ffa28970687db85b36947a3e6 diff --git a/src/Xamarin.Android.Sdk/targets/Xamarin.Android.Sdk.props b/src/Xamarin.Android.Sdk/targets/Xamarin.Android.Sdk.props index d36569344a6..1c3f164efea 100644 --- a/src/Xamarin.Android.Sdk/targets/Xamarin.Android.Sdk.props +++ b/src/Xamarin.Android.Sdk/targets/Xamarin.Android.Sdk.props @@ -4,7 +4,7 @@ true - 1.8.0 + 11.99.0 1.8.0 false true From 57530e035111cb7076d6c36fcb9d720959866555 Mon Sep 17 00:00:00 2001 From: Jonathan Pryor Date: Thu, 16 Apr 2020 15:55:57 -0400 Subject: [PATCH 02/71] Bump dependencies! Changes: https://github.com/xamarin/Java.Interop/compare/80b4667f7085027d66592a0c36490d9488c7ce60...c19794e67cc4f87aac9c64f7ceb94cfbe556c63c Changes: https://github.com/xamarin/xamarin-android-tools/compare/36d7fee5f7e0ef6ffa28970687db85b36947a3e6...f473ff93301851eea50548426b6e1f1b909444a3 Java.Interop changes are for OpenJDK11 build support --- external/xamarin-android-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/external/xamarin-android-tools b/external/xamarin-android-tools index 36d7fee5f7e..f473ff93301 160000 --- a/external/xamarin-android-tools +++ b/external/xamarin-android-tools @@ -1 +1 @@ -Subproject commit 36d7fee5f7e0ef6ffa28970687db85b36947a3e6 +Subproject commit f473ff93301851eea50548426b6e1f1b909444a3 From 3e6af18fde95d9ea834a04d3dd99a51bbcbf0b17 Mon Sep 17 00:00:00 2001 From: Jonathan Pryor Date: Thu, 16 Apr 2020 19:16:41 -0400 Subject: [PATCH 03/71] OpenJDK11 support! It builds locally! (If that actually means anything.) (Unit tests untested.) Make "global" `$(JavacSourceVersion)` and `$(JavacTargetVersion)` MSBuild properties for great consistency! Use a new `$(JavacTargetVersion)` value for `javac -target` instead of reusing `$(JavacSourceVersion)`, because...why not? Cleanup `build-tools/scripts/Jar.targets` to *not* change the behavior depending on what the installed JDK is. JDK 1.8 is now a *minimum* to *build* xamarin-android...and probably to *run* any Java code that we build. (Maybe.) Various fixes to allow xamarin/Java.Interop@6d7266dd9 to build. --- Configuration.props | 4 ++++ build-tools/scripts/Jar.targets | 6 +----- build-tools/scripts/JavaCallableWrappers.targets | 2 +- .../Xamarin.Android.Common.props.in | 2 +- src/java-runtime/java-runtime.targets | 2 +- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/Configuration.props b/Configuration.props index 94f2c66a435..a6b6e0cbc5b 100644 --- a/Configuration.props +++ b/Configuration.props @@ -118,6 +118,10 @@ i686-w64-mingw32 x86_64-w64-mingw32 + + 1.8 + 1.8 + $([System.IO.Path]::GetFullPath ('$(AndroidMxeInstallPrefix)')) $([System.IO.Path]::GetFullPath ('$(AndroidNdkDirectory)')) diff --git a/build-tools/scripts/Jar.targets b/build-tools/scripts/Jar.targets index 91552910d10..75a0c533218 100644 --- a/build-tools/scripts/Jar.targets +++ b/build-tools/scripts/Jar.targets @@ -3,10 +3,6 @@ - <_JavacSourceVersion Condition="$(_JdkVersion.StartsWith ('9'))">1.8 - <_JavacSourceVersion Condition=" '$(_JavacSourceVersion)' == '' ">1.5 - <_JavacTargetVersion Condition="$(_JdkVersion.StartsWith ('9'))">1.8 - <_JavacTargetVersion Condition=" '$(_JavacTargetVersion)' == '' ">1.6 $(JavaSdkDirectory)\bin\jar $(JavaSdkDirectory)\bin\javac @@ -35,7 +31,7 @@ <_Javac>"$(JavaCPath)" <_Jar>"$(JarPath)" - <_Targets>-source $(_JavacSourceVersion) -target $(_JavacTargetVersion) + <_Targets>-source $(JavacSourceVersion) -target $(JavacTargetVersion) <_DestDir>$(IntermediateOutputPath)__CreateTestJarFile-bin <_AndroidJar>-bootclasspath "$(AndroidSdkDirectory)\platforms\android-$(_AndroidApiLevelName)\android.jar" <_CP>-cp "$(_JavaInteropJarPath)" diff --git a/build-tools/scripts/JavaCallableWrappers.targets b/build-tools/scripts/JavaCallableWrappers.targets index df18fa9cc69..129215795fe 100644 --- a/build-tools/scripts/JavaCallableWrappers.targets +++ b/build-tools/scripts/JavaCallableWrappers.targets @@ -31,7 +31,7 @@ Overwrite="True" /> - <_Target>-source $(JavacSourceVersion) -target $(JavacSourceVersion) + <_Target>-source $(JavacSourceVersion) -target $(JavacTargetVersion) <_D>-d "$(IntermediateOutputPath)jcw\bin" <_AndroidJar>"$(AndroidToolchainDirectory)\sdk\platforms\android-$(AndroidPlatformId)\android.jar" <_MonoAndroidJar>$(OutputPath)mono.android.jar diff --git a/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.props.in b/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.props.in index 27ca3a44738..21660eb282d 100644 --- a/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.props.in +++ b/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.props.in @@ -7,7 +7,7 @@ false true true - 1.8.0 + 11.99.99 1.6.0 {abi}{versionCode:D5} UpdateGeneratedFiles diff --git a/src/java-runtime/java-runtime.targets b/src/java-runtime/java-runtime.targets index fe8c0b579fb..8085411e704 100644 --- a/src/java-runtime/java-runtime.targets +++ b/src/java-runtime/java-runtime.targets @@ -33,7 +33,7 @@ Overwrite="True" /> - <_Target Condition="'$(JavacSourceVersion)' != ''">-source $(JavacSourceVersion) -target $(JavacSourceVersion) + <_Target Condition="'$(JavacSourceVersion)' != ''">-source $(JavacSourceVersion) -target $(JavacTargetVersion) <_AndroidJar>"$(AndroidToolchainDirectory)\sdk\platforms\android-$(AndroidFirstPlatformId)\android.jar" Date: Mon, 20 Apr 2020 11:53:55 -0400 Subject: [PATCH 04/71] [r8, manifestmerger] Target JDK 1.8 Our build outputs still need to be runnable on machines using JDK 1.8. Otherwise, things may break when using those `.jar` files: error AMM0000: java.lang.UnsupportedClassVersionError: com/xamarin/manifestmerger/Main has been compiled by a more recent version of the Java Runtime (class file version 55.0), this version of the Java Runtime only recognizes class file versions up to 52.0 error AMM0000: at java.lang.ClassLoader.defineClass1(Native Method) error AMM0000: at java.lang.ClassLoader.defineClass(ClassLoader.java:763) error AMM0000: at java.security.SecureClassLoader.defineClass(SecureClassLoader.java:142) error AMM0000: at java.net.URLClassLoader.defineClass(URLClassLoader.java:468) error AMM0000: at java.net.URLClassLoader.access$100(URLClassLoader.java:74) error AMM0000: at java.net.URLClassLoader$1.run(URLClassLoader.java:369) error AMM0000: at java.net.URLClassLoader$1.run(URLClassLoader.java:363) error AMM0000: at java.security.AccessController.doPrivileged(Native Method) error AMM0000: at java.net.URLClassLoader.findClass(URLClassLoader.java:362) error AMM0000: at java.lang.ClassLoader.loadClass(ClassLoader.java:424) error AMM0000: at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:349) error AMM0000: at java.lang.ClassLoader.loadClass(ClassLoader.java:357) error AMM0000: at sun.launcher.LauncherHelper.checkAndLoadMain(LauncherHelper.java:495) error AMM0000: Error: A JNI error has occurred, please check your installation and try again error AMM0000: Exception in thread "main" error AMM0000: java.lang.UnsupportedClassVersionError : com/xamarin/manifestmerger/Main has been compiled by a more recent version of the Java Runtime (class file version 55.0), this version of the Java Runtime only recognizes class file versions up to 52.0 --- src/manifestmerger/build.gradle | 5 +++++ src/r8/build.gradle | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/src/manifestmerger/build.gradle b/src/manifestmerger/build.gradle index 24428211812..f605602a42f 100644 --- a/src/manifestmerger/build.gradle +++ b/src/manifestmerger/build.gradle @@ -2,6 +2,11 @@ plugins { id 'java-library' } +java { + sourceCompatibility = JavaVersion.VERSION_1_8 + targetCompatibility = JavaVersion.VERSION_1_8 +} + repositories { maven { url 'https://maven.google.com' } mavenCentral() diff --git a/src/r8/build.gradle b/src/r8/build.gradle index c8a623939aa..98e8833550c 100644 --- a/src/r8/build.gradle +++ b/src/r8/build.gradle @@ -2,6 +2,11 @@ plugins { id 'java' } +java { + sourceCompatibility = JavaVersion.VERSION_1_8 + targetCompatibility = JavaVersion.VERSION_1_8 +} + // See: https://r8.googlesource.com/r8/+/refs/tags/1.5.67/build.gradle#53 repositories { maven { From 6601fbc71af0f7769c896dc54a94e2c691994c89 Mon Sep 17 00:00:00 2001 From: Jonathan Pryor Date: Mon, 20 Apr 2020 12:28:55 -0400 Subject: [PATCH 05/71] [java-runtime] Use r8.jar instead of dx `dx` doesn't like OpenJDK 11 on Windows: "C:\Users\dlab14\android-toolchain\sdk\build-tools\29.0.2\dx" --dex --no-strict --output="C:\A\vs2019xam00000H-1\_work\2\s\bin\Release\lib\xamarin.android\xbuild\Xamarin\Android\java_runtime.dex" "C:\A\vs2019xam00000H-1\_work\2\s\bin\Release\lib\xamarin.android\xbuild\Xamarin\Android\java_runtime.jar" ##[debug]Processed: ##vso[task.logdetail id=dffb25d4-cb0b-4e58-94d2-4b9a4f215bed;parentid=;name=EXEC;type=Build;starttime=2020-04-19T01:21:10.0332656Z;state=InProgress;] ##[error]EXEC(0,0): Error : No suitable Java found. In order to properly use the Android Developer ##[debug]Processed: ##vso[task.logissue type=Error;sourcepath=EXEC;linenumber=0;columnnumber=0;code=;]No suitable Java found. In order to properly use the Android Developer ##[debug]Processed: ##vso[task.logdetail id=dffb25d4-cb0b-4e58-94d2-4b9a4f215bed;parentid=;type=Build;result=Failed;finishtime=2020-04-19T01:21:10.0488457Z;progress=100;state=Completed;parentid=;name=;] EXEC : error : No suitable Java found. In order to properly use the Android Developer [C:\A\vs2019xam00000H-1\_work\2\s\src\java-runtime\java-runtime.csproj] Tools, you need a suitable version of Java JDK installed on your system. We recommend that you install the JDK version of JavaSE, available here: http://www.oracle.com/technetwork/java/javase/downloads If you already have Java installed, you can define the JAVA_HOME environment variable in Control Panel / System / Avanced System Settings to point to the JDK folder. You can find the complete Android SDK requirements here: http://developer.android.com/sdk/requirements.html ##[error]src\java-runtime\java-runtime.targets(59,3): Error MSB3073: The command ""C:\Users\dlab14\android-toolchain\sdk\build-tools\29.0.2\dx" --dex --no-strict --output="C:\A\vs2019xam00000H-1\_work\2\s\bin\Release\lib\xamarin.android\xbuild\Xamarin\Android\java_runtime.dex" "C:\A\vs2019xam00000H-1\_work\2\s\bin\Release\lib\xamarin.android\xbuild\Xamarin\Android\java_runtime.jar"" exited with code -1. Use `r8.jar` instead, to hopefully avoid the problem. --- src/java-runtime/java-runtime.targets | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/java-runtime/java-runtime.targets b/src/java-runtime/java-runtime.targets index 8085411e704..17644e8179f 100644 --- a/src/java-runtime/java-runtime.targets +++ b/src/java-runtime/java-runtime.targets @@ -57,8 +57,12 @@ Inputs="@(_RuntimeOutput->'%(Identity)')" Outputs="@(_RuntimeOutput->'%(OutputDex)')"> + From b78eed04175d182db8a182c3b17a2d71d5e7744b Mon Sep 17 00:00:00 2001 From: Jonathan Pryor Date: Mon, 20 Apr 2020 13:53:59 -0400 Subject: [PATCH 06/71] [manfestmerger] r8 must be built first --- src/manifestmerger/manifestmerger.csproj | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/manifestmerger/manifestmerger.csproj b/src/manifestmerger/manifestmerger.csproj index c03ae8fa07d..cb6f3e61376 100644 --- a/src/manifestmerger/manifestmerger.csproj +++ b/src/manifestmerger/manifestmerger.csproj @@ -9,6 +9,11 @@ + + + False + + \ No newline at end of file From a17f1c673c89587e80c8af7c1081d1a5925537fa Mon Sep 17 00:00:00 2001 From: Jonathan Pryor Date: Mon, 4 May 2020 13:17:58 -0400 Subject: [PATCH 07/71] Fix build break when compiling xaprepare --- build-tools/xaprepare/xaprepare/ConfigAndData/Configurables.cs | 2 ++ .../xaprepare/xaprepare/Scenarios/Scenario_AndroidToolchain.cs | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/build-tools/xaprepare/xaprepare/ConfigAndData/Configurables.cs b/build-tools/xaprepare/xaprepare/ConfigAndData/Configurables.cs index acace463d91..507cd1d9aaf 100644 --- a/build-tools/xaprepare/xaprepare/ConfigAndData/Configurables.cs +++ b/build-tools/xaprepare/xaprepare/ConfigAndData/Configurables.cs @@ -406,6 +406,8 @@ static string GetCachedPath (ref string? variable, Func creator) static string? frameworkListInstallPath; static string? correttoCacheDir; static string? correttoInstallDir; + static string? openJDKInstallDir; + static string? openJDKCacheDir; static string? profileAssembliesProjitemsPath; static string? bclTestsSourceDir; static string? installHostBCLDir; diff --git a/build-tools/xaprepare/xaprepare/Scenarios/Scenario_AndroidToolchain.cs b/build-tools/xaprepare/xaprepare/Scenarios/Scenario_AndroidToolchain.cs index 687ea5d09da..d53a3c60d67 100644 --- a/build-tools/xaprepare/xaprepare/Scenarios/Scenario_AndroidToolchain.cs +++ b/build-tools/xaprepare/xaprepare/Scenarios/Scenario_AndroidToolchain.cs @@ -6,7 +6,7 @@ namespace Xamarin.Android.Prepare partial class Scenario_AndroidToolchain : ScenarioNoStandardEndSteps { public Scenario_AndroidToolchain () - : base ("AndroidToolchain", "Install Android SDK, NDK and OpenJDK.", Context.Instance) + : base ("AndroidToolchain", "Install Android SDK, NDK and OpenJDK.") {} protected override void AddSteps (Context context) From ebf3559140e5878f3d1520a028c675717a930efb Mon Sep 17 00:00:00 2001 From: Jonathan Pryor Date: Mon, 4 May 2020 14:35:29 -0400 Subject: [PATCH 08/71] [java-runtime] Add ProjectReference for r8.csproj java-runtime.targets requires that r8.jar exist in order to run. --- src/java-runtime/java-runtime.csproj | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/java-runtime/java-runtime.csproj b/src/java-runtime/java-runtime.csproj index a4e05e6468d..6629f64f020 100644 --- a/src/java-runtime/java-runtime.csproj +++ b/src/java-runtime/java-runtime.csproj @@ -19,4 +19,7 @@ + + + \ No newline at end of file From 01be151255c51ad1f990846664bb810dd0784513 Mon Sep 17 00:00:00 2001 From: Jonathan Pryor Date: Mon, 4 May 2020 15:50:04 -0400 Subject: [PATCH 09/71] [java-runtimes] Fixity Fix --- src/java-runtime/java-runtime.csproj | 8 +++++--- src/java-runtime/java-runtime.targets | 21 ++++++++++++++++++++- 2 files changed, 25 insertions(+), 4 deletions(-) diff --git a/src/java-runtime/java-runtime.csproj b/src/java-runtime/java-runtime.csproj index 6629f64f020..e08e99b2092 100644 --- a/src/java-runtime/java-runtime.csproj +++ b/src/java-runtime/java-runtime.csproj @@ -14,12 +14,14 @@ $(XAInstallPrefix)xbuild\Xamarin\Android + + + False + + - - - \ No newline at end of file diff --git a/src/java-runtime/java-runtime.targets b/src/java-runtime/java-runtime.targets index 17644e8179f..417e9fafa89 100644 --- a/src/java-runtime/java-runtime.targets +++ b/src/java-runtime/java-runtime.targets @@ -1,6 +1,16 @@ + + + ResolveReferences; + _BuildJavaRuntimeJar; + + + _CleanJavaRuntimeJar; + + + <_RuntimeOutput Include="$(OutputPath)java_runtime.jar"> $(OutputPath)java_runtime.jar @@ -17,7 +27,11 @@ java\mono\android\release\MultiDexLoader.java;java\mono\android\release\BuildConfig.java - + + + @@ -65,4 +79,9 @@ DestinationFiles="%(_RuntimeOutput.OutputDex)" /> + + + + + From ec9eca0f70cd9cf47db35bde2108d98b11179a10 Mon Sep 17 00:00:00 2001 From: Jonathan Pryor Date: Tue, 5 May 2020 12:04:04 -0400 Subject: [PATCH 10/71] [scripts] Use r8.jar instead of dx Remember commit 6601fbc7? Same deal, different file. --- src/Mono.Android/Mono.Android.csproj | 1 + src/OpenTK-1.0/OpenTK.csproj | 1 + src/java-runtime/java-runtime.csproj | 4 +--- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Mono.Android/Mono.Android.csproj b/src/Mono.Android/Mono.Android.csproj index a47eb5529e2..1c9119295c8 100644 --- a/src/Mono.Android/Mono.Android.csproj +++ b/src/Mono.Android/Mono.Android.csproj @@ -352,6 +352,7 @@ + diff --git a/src/OpenTK-1.0/OpenTK.csproj b/src/OpenTK-1.0/OpenTK.csproj index bf771a52f59..f388310b1b2 100644 --- a/src/OpenTK-1.0/OpenTK.csproj +++ b/src/OpenTK-1.0/OpenTK.csproj @@ -614,5 +614,6 @@ False False + diff --git a/src/java-runtime/java-runtime.csproj b/src/java-runtime/java-runtime.csproj index e08e99b2092..e7354f69d92 100644 --- a/src/java-runtime/java-runtime.csproj +++ b/src/java-runtime/java-runtime.csproj @@ -15,9 +15,7 @@ $(XAInstallPrefix)xbuild\Xamarin\Android - - False - + From bc7da937ca34d1506a74daacf6100e10bcc884ac Mon Sep 17 00:00:00 2001 From: Jonathan Pryor Date: Tue, 5 May 2020 12:46:38 -0400 Subject: [PATCH 11/71] Oops, forgot this s/dx/r8.jar/ replacement --- build-tools/scripts/JavaCallableWrappers.targets | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/build-tools/scripts/JavaCallableWrappers.targets b/build-tools/scripts/JavaCallableWrappers.targets index 129215795fe..8d9fa741751 100644 --- a/build-tools/scripts/JavaCallableWrappers.targets +++ b/build-tools/scripts/JavaCallableWrappers.targets @@ -53,8 +53,11 @@ AfterTargets="GenerateJavaCallableWrappers" Inputs="$(OutputPath)mono.android.jar" Outputs="$(OutputPath)mono.android.dex"> + + $(XAInstallPrefix)xbuild\Xamarin\Android\r8.jar + From 9b02c35bce139fe8a48b18b9bc7c6a1ef697d026 Mon Sep 17 00:00:00 2001 From: Jonathan Pryor Date: Tue, 5 May 2020 13:34:25 -0400 Subject: [PATCH 12/71] Let's try this again. --- build-tools/scripts/JavaCallableWrappers.targets | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/build-tools/scripts/JavaCallableWrappers.targets b/build-tools/scripts/JavaCallableWrappers.targets index 8d9fa741751..8319a829cb5 100644 --- a/build-tools/scripts/JavaCallableWrappers.targets +++ b/build-tools/scripts/JavaCallableWrappers.targets @@ -57,8 +57,12 @@ $(XAInstallPrefix)xbuild\Xamarin\Android\r8.jar + From 0fca1ef43e12612c2547f5162477049b6bd3903b Mon Sep 17 00:00:00 2001 From: Jonathan Pryor Date: Tue, 5 May 2020 14:36:13 -0400 Subject: [PATCH 13/71] Trailing \ is bad "C:\Users\dlab14\android-toolchain\jdk\bin\java.exe" -classpath "C:\A\vs2019xam00000I-1\_work\2\s\bin\Release\lib\xamarin.android\xbuild\Xamarin\Android\r8.jar" com.android.tools.r8.D8 --release --no-desugaring --output "obj\Release\netcoreapp3.1\android-29\" "C:\A\vs2019xam00000I-1\_work\2\s\bin\Release\lib\xamarin.android\xbuild-frameworks\Microsoft.Android\netcoreapp3.1\mono.android.jar" Compilation failed with an internal error. java.nio.file.InvalidPathException: Illegal char <"> at index 36: obj\Release\netcoreapp3.1\android-29" C:\A\vs2019xam00000I-1\_work\2\s\bin\Release\lib\xamarin.android\xbuild-frameworks\Microsoft.Android\netcoreapp3.1\mono.android.jar at java.base/sun.nio.fs.WindowsPathParser.normalize(WindowsPathParser.java:182) at java.base/sun.nio.fs.WindowsPathParser.parse(WindowsPathParser.java:153) The `\"` in the command line cause the `"` to be "escaped", resulting in an invalid command line on Windows. Let's `.TrimEnd('\')` to avoid the escaping. --- build-tools/scripts/JavaCallableWrappers.targets | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build-tools/scripts/JavaCallableWrappers.targets b/build-tools/scripts/JavaCallableWrappers.targets index 8319a829cb5..7d181e7c653 100644 --- a/build-tools/scripts/JavaCallableWrappers.targets +++ b/build-tools/scripts/JavaCallableWrappers.targets @@ -57,7 +57,7 @@ $(XAInstallPrefix)xbuild\Xamarin\Android\r8.jar Date: Wed, 6 May 2020 15:10:01 -0500 Subject: [PATCH 14/71] [build] set $JAVA_HOME for mac test phases Right now the test phases are using: JavaSdkDirectory = C:\Program Files\Android\jdk\microsoft_dist_openjdk_1.8.0.25 JavaSdkDirectory = /Library/Java/JavaVirtualMachines/zulu-8.jdk/Contents/Home So we aren't actually *testing* OpenJDK 11. We are setting `$JAVA_HOME` when running the macOS build phase, let's see if the exact same trick will work for running MSBuild tests. To set `$JAVA_HOME` on each platform, I used: * `$HOME/Library/Android/jdk` for macOS * `%USERPROFILE%\android-toolchain\jdk` for Windows --- .../automation/yaml-templates/setup-test-environment.yaml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/build-tools/automation/yaml-templates/setup-test-environment.yaml b/build-tools/automation/yaml-templates/setup-test-environment.yaml index 75160917cf9..ed14e861988 100644 --- a/build-tools/automation/yaml-templates/setup-test-environment.yaml +++ b/build-tools/automation/yaml-templates/setup-test-environment.yaml @@ -11,6 +11,14 @@ steps: parameters: provisionExtraArgs: ${{ parameters.provisionExtraArgs }} +- script: echo "##vso[task.setvariable variable=JAVA_HOME]$HOME/Library/Android/jdk" + displayName: set JAVA_HOME + condition: and(succeeded(), eq(variables['agent.os'], 'Darwin')) + +- script: echo "##vso[task.setvariable variable=JAVA_HOME]%USERPROFILE%\android-toolchain\jdk" + displayName: set JAVA_HOME + condition: and(succeeded(), eq(variables['agent.os'], 'Windows_NT')) + - task: UseDotNet@2 displayName: install .NET Core $(DotNetCorePreviewVersion) inputs: From 85093ff7be049aa30c93f22235f2618511477125 Mon Sep 17 00:00:00 2001 From: Jonathan Pryor Date: Fri, 8 May 2020 15:35:33 -0400 Subject: [PATCH 15/71] Use commandlinetools' avdmanager Context: https://devdiv.visualstudio.com/DevDiv/_build/results?buildId=3711144&view=logs&jobId=4348ab7c-72f0-52b0-44fd-6f17a16a558f&j=8556562a-ae5f-5bd1-7c4d-bf1af4b6f1e1&t=9f016295-34d3-5b55-55cb-51df877e2cc6 Usage of `avdmanager` fails under JDK 11: TestApks.targets(45,5): error : Exception in thread "main" java.lang.NoClassDefFoundError: javax/xml/bind/annotation/XmlSchema TestApks.targets(45,5): error : at com.android.repository.api.SchemaModule$SchemaModuleVersion.(SchemaModule.java:156) TestApks.targets(45,5): error : at com.android.repository.api.SchemaModule.(SchemaModule.java:75) TestApks.targets(45,5): error : at com.android.sdklib.repository.AndroidSdkHandler.(AndroidSdkHandler.java:81) TestApks.targets(45,5): error : at com.android.sdklib.tool.AvdManagerCli.run(AvdManagerCli.java:213) TestApks.targets(45,5): error : at com.android.sdklib.tool.AvdManagerCli.main(AvdManagerCli.java:200) TestApks.targets(45,5): error : Caused by: java.lang.ClassNotFoundException: javax.xml.bind.annotation.XmlSchema TestApks.targets(45,5): error : at java.base/jdk.internal.loader.BuiltinClassLoader.loadClass(BuiltinClassLoader.java:583) TestApks.targets(45,5): error : at java.base/jdk.internal.loader.ClassLoaders$AppClassLoader.loadClass(ClassLoaders.java:178) TestApks.targets(45,5): error : at java.base/java.lang.ClassLoader.loadClass(ClassLoader.java:521) TestApks.targets(45,5): error : ... 5 more TestApks.targets(45,5): error : Process `/Users/runner/Library/Android/sdk/tools/bin/avdmanager` exited with value 1. This is because the "normal" `tools`' `avdmanager` doesn't support JDK 11, and has been replaced with a *new* `commandlinetools` package whicih *does* support JDK 11. Begin installing the `commandlinetools` SDK package, and use it for `sdkmanager` invocations. --- Configuration.props | 3 +++ build-tools/check-boot-times/Program.cs | 10 ++++++++ build-tools/scripts/TestApks.targets | 2 +- .../xaprepare/Application/KnownProperties.cs | 2 ++ .../Application/Properties.Defaults.cs.in | 2 ++ .../Dependencies/AndroidToolchain.Linux.cs | 1 + .../Dependencies/AndroidToolchain.MacOS.cs | 1 + .../Dependencies/AndroidToolchain.Windows.cs | 1 + .../Dependencies/AndroidToolchain.cs | 4 +++ .../xaprepare/Steps/Step_Android_SDK_NDK.cs | 25 ++++++++++++++++++- .../xaprepare/xaprepare/xaprepare.targets | 2 ++ 11 files changed, 51 insertions(+), 2 deletions(-) diff --git a/Configuration.props b/Configuration.props index 8748be98fb0..a21e84c63b5 100644 --- a/Configuration.props +++ b/Configuration.props @@ -139,6 +139,9 @@ $(AndroidSdkFullPath)\tools $(AndroidToolPath)\bin android + 1.0 + 6200805_latest + $(AndroidSdkFullPath)\cmdline-tools\$(CommandLineToolsFolder)\bin 6306047 30.0.5 $(AndroidSdkFullPath)\emulator diff --git a/build-tools/check-boot-times/Program.cs b/build-tools/check-boot-times/Program.cs index dc294b82da9..a6f5a5f7f71 100644 --- a/build-tools/check-boot-times/Program.cs +++ b/build-tools/check-boot-times/Program.cs @@ -509,6 +509,8 @@ static string GetProgramPath (params string [] filenames) if (!string.IsNullOrWhiteSpace (sdkPath)) { potentialLocations.AddRange (new []{ $"{sdkPath}/platform-tools", + $"{sdkPath}/cmdline-tools/1.0/bin", + $"{sdkPath}/cmdline-tools/latest/bin", $"{sdkPath}/emulator", $"{sdkPath}/tools", $"{sdkPath}/tools/bin", @@ -516,14 +518,20 @@ static string GetProgramPath (params string [] filenames) } else { potentialLocations.AddRange (new []{ "AppData/Local/Android/Sdk/platform-tools", + "AppData/Local/Android/Sdk/cmdline-tools/1.0/bin", + "AppData/Local/Android/Sdk/cmdline-tools/latest/bin", "AppData/Local/Android/Sdk/emulator", "AppData/Local/Android/Sdk/tools", "AppData/Local/Android/Sdk/tools/bin", "Library/Android/sdk/platform-tools", + "Library/Android/sdk/cmdline-tools/1.0/bin", + "Library/Android/sdk/cmdline-tools/latest/bin", "Library/Android/sdk/emulator", "Library/Android/sdk/tools", "Library/Android/sdk/tools/bin", "android-toolchain/sdk/platform-tools", + "android-toolchain/sdk/cmdline-tools/1.0/bin", + "android-toolchain/sdk/cmdline-tools/latest/bin", "android-toolchain/sdk/emulator", "android-toolchain/sdk/tools", "android-toolchain/sdk/tools/bin", @@ -532,6 +540,8 @@ static string GetProgramPath (params string [] filenames) if (RunningOnWindowsEnvironment) { potentialLocations.AddRange (new []{ "C:/Program Files (x86)/Android/android-sdk/platform-tools", + "C:/Program Files (x86)/Android/android-sdk/cmdline-tools/1.0/bin", + "C:/Program Files (x86)/Android/android-sdk/cmdline-tools/latest/bin", "C:/Program Files (x86)/Android/android-sdk/emulator", "C:/Program Files (x86)/Android/android-sdk/tools", "C:/Program Files (x86)/Android/android-sdk/tools/bin", diff --git a/build-tools/scripts/TestApks.targets b/build-tools/scripts/TestApks.targets index 0937fcddb00..739d98cc3a2 100644 --- a/build-tools/scripts/TestApks.targets +++ b/build-tools/scripts/TestApks.targets @@ -50,7 +50,7 @@ SdkVersion="29" ImageName="$(_TestImageName)" ToolExe="$(AvdManagerToolExe)" - ToolPath="$(AndroidToolsBinPath)" + ToolPath="$(CommandLineToolsBinPath)" RamSizeMB="3072" DataPartitionSizeMB="4096" /> diff --git a/build-tools/xaprepare/xaprepare/Application/KnownProperties.cs b/build-tools/xaprepare/xaprepare/Application/KnownProperties.cs index 4f7830dbb1c..ae2ada7e4c8 100644 --- a/build-tools/xaprepare/xaprepare/Application/KnownProperties.cs +++ b/build-tools/xaprepare/xaprepare/Application/KnownProperties.cs @@ -16,6 +16,8 @@ static class KnownProperties public const string AutoProvision = "AutoProvision"; public const string AutoProvisionUsesSudo = "AutoProvisionUsesSudo"; public const string Configuration = "Configuration"; + public const string CommandLineToolsVersion = nameof (CommandLineToolsVersion); + public const string CommandLineToolsFolder = nameof (CommandLineToolsFolder); public const string EmulatorVersion = "EmulatorVersion"; public const string EmulatorPkgRevision = "EmulatorPkgRevision"; public const string IgnoreMaxMonoVersion = "IgnoreMaxMonoVersion"; diff --git a/build-tools/xaprepare/xaprepare/Application/Properties.Defaults.cs.in b/build-tools/xaprepare/xaprepare/Application/Properties.Defaults.cs.in index 300d2e9e4d1..8ae900d609d 100644 --- a/build-tools/xaprepare/xaprepare/Application/Properties.Defaults.cs.in +++ b/build-tools/xaprepare/xaprepare/Application/Properties.Defaults.cs.in @@ -20,6 +20,8 @@ namespace Xamarin.Android.Prepare properties.Add (KnownProperties.AutoProvision, StripQuotes ("@AutoProvision@")); properties.Add (KnownProperties.AutoProvisionUsesSudo, StripQuotes ("@AutoProvisionUsesSudo@")); properties.Add (KnownProperties.Configuration, StripQuotes ("@Configuration@")); + properties.Add (KnownProperties.CommandLineToolsFolder, StripQuotes ("@CommandLineToolsFolder@")); + properties.Add (KnownProperties.CommandLineToolsVersion, StripQuotes ("@CommandLineToolsVersion@")); properties.Add (KnownProperties.EmulatorVersion, StripQuotes ("@EmulatorVersion@")); properties.Add (KnownProperties.EmulatorPkgRevision, StripQuotes ("@EmulatorPkgRevision@")); properties.Add (KnownProperties.IgnoreMaxMonoVersion, StripQuotes ("@IgnoreMaxMonoVersion@")); diff --git a/build-tools/xaprepare/xaprepare/ConfigAndData/Dependencies/AndroidToolchain.Linux.cs b/build-tools/xaprepare/xaprepare/ConfigAndData/Dependencies/AndroidToolchain.Linux.cs index 74e2c53555f..930de5b8b46 100644 --- a/build-tools/xaprepare/xaprepare/ConfigAndData/Dependencies/AndroidToolchain.Linux.cs +++ b/build-tools/xaprepare/xaprepare/ConfigAndData/Dependencies/AndroidToolchain.Linux.cs @@ -8,5 +8,6 @@ partial class AndroidToolchain { static readonly string osTag = "linux"; static readonly string altOsTag = osTag; + static readonly string cltOsTag = osTag; } } diff --git a/build-tools/xaprepare/xaprepare/ConfigAndData/Dependencies/AndroidToolchain.MacOS.cs b/build-tools/xaprepare/xaprepare/ConfigAndData/Dependencies/AndroidToolchain.MacOS.cs index e3911876846..b7c085f341b 100644 --- a/build-tools/xaprepare/xaprepare/ConfigAndData/Dependencies/AndroidToolchain.MacOS.cs +++ b/build-tools/xaprepare/xaprepare/ConfigAndData/Dependencies/AndroidToolchain.MacOS.cs @@ -4,5 +4,6 @@ partial class AndroidToolchain { static readonly string osTag = "darwin"; static readonly string altOsTag = "macosx"; + static readonly string cltOsTag = "mac"; } } diff --git a/build-tools/xaprepare/xaprepare/ConfigAndData/Dependencies/AndroidToolchain.Windows.cs b/build-tools/xaprepare/xaprepare/ConfigAndData/Dependencies/AndroidToolchain.Windows.cs index cb362d006ec..fcf2a03dccc 100644 --- a/build-tools/xaprepare/xaprepare/ConfigAndData/Dependencies/AndroidToolchain.Windows.cs +++ b/build-tools/xaprepare/xaprepare/ConfigAndData/Dependencies/AndroidToolchain.Windows.cs @@ -4,5 +4,6 @@ partial class AndroidToolchain { static readonly string osTag = "windows"; static readonly string altOsTag = osTag; + static readonly string cltOsTag = "win"; } } diff --git a/build-tools/xaprepare/xaprepare/ConfigAndData/Dependencies/AndroidToolchain.cs b/build-tools/xaprepare/xaprepare/ConfigAndData/Dependencies/AndroidToolchain.cs index 5d24d5e9685..68417989eac 100644 --- a/build-tools/xaprepare/xaprepare/ConfigAndData/Dependencies/AndroidToolchain.cs +++ b/build-tools/xaprepare/xaprepare/ConfigAndData/Dependencies/AndroidToolchain.cs @@ -17,6 +17,8 @@ public AndroidToolchain () string AndroidNdkDirectory = GetRequiredProperty (KnownProperties.AndroidNdkDirectory); string AndroidCmakeVersion = GetRequiredProperty (KnownProperties.AndroidCmakeVersion); string AndroidCmakeVersionPath = GetRequiredProperty (KnownProperties.AndroidCmakeVersionPath); + string CommandLineToolsVersion = GetRequiredProperty (KnownProperties.CommandLineToolsVersion); + string CommandLineToolsFolder = GetRequiredProperty (KnownProperties.CommandLineToolsFolder); string EmulatorVersion = GetRequiredProperty (KnownProperties.EmulatorVersion); string EmulatorPkgRevision = GetRequiredProperty (KnownProperties.EmulatorPkgRevision); string XABuildToolsFolder = GetRequiredProperty (KnownProperties.XABuildToolsFolder); @@ -51,6 +53,8 @@ public AndroidToolchain () new AndroidToolchainComponent ($"x86-29_r07-{osTag}", destDir: Path.Combine ("system-images", "android-29", "default", "x86"), relativeUrl: new Uri ("sys-img/android/", UriKind.Relative), pkgRevision: "7"), new AndroidToolchainComponent ($"android-ndk-r{AndroidNdkVersion}-{osTag}-x86_64", destDir: AndroidNdkDirectory, pkgRevision: AndroidPkgRevision), new AndroidToolchainComponent ($"build-tools_r{XABuildToolsVersion}-{altOsTag}", destDir: Path.Combine ("build-tools", XABuildToolsFolder), isMultiVersion: true), + new AndroidToolchainComponent ($"commandlinetools-{cltOsTag}-{CommandLineToolsVersion}", + destDir: Path.Combine ("cmdline-tools", CommandLineToolsFolder), isMultiVersion: true), new AndroidToolchainComponent ($"platform-tools_r{XAPlatformToolsVersion}-{osTag}", destDir: "platform-tools", pkgRevision: XAPlatformToolsVersion), new AndroidToolchainComponent ($"sdk-tools-{osTag}-4333796", destDir: "tools", pkgRevision: "26.1.1"), new AndroidToolchainComponent ($"emulator-{osTag}-{EmulatorVersion}", destDir: "emulator", pkgRevision: EmulatorPkgRevision), diff --git a/build-tools/xaprepare/xaprepare/Steps/Step_Android_SDK_NDK.cs b/build-tools/xaprepare/xaprepare/Steps/Step_Android_SDK_NDK.cs index 6c591017d26..126d9636396 100644 --- a/build-tools/xaprepare/xaprepare/Steps/Step_Android_SDK_NDK.cs +++ b/build-tools/xaprepare/xaprepare/Steps/Step_Android_SDK_NDK.cs @@ -99,7 +99,17 @@ protected override async Task Execute (Context context) bool AcceptLicenses (Context context, string sdkRoot) { - string sdkManager = context.OS.Which (Path.Combine (sdkRoot, "tools", "bin", "sdkmanager")); + string[] sdkManagerPaths = new[]{ + Path.Combine (sdkRoot, "cmdline-tools", context.Properties [KnownProperties.CommandLineToolsFolder], "bin", "sdkmanager"), + Path.Combine (sdkRoot, "cmdline-tools", "latest", "bin", "sdkmanager"), + Path.Combine (sdkRoot, "tools", "bin", "sdkmanager"), + }; + string sdkManager = ""; + foreach (var sdkManagerPath in sdkManagerPaths) { + sdkManager = context.OS.Which (sdkManagerPath, required: false); + if (!string.IsNullOrEmpty (sdkManager)) + break; + } if (sdkManager.Length == 0) throw new InvalidOperationException ("sdkmanager not found"); string jdkDir = context.OS.JavaHome; @@ -125,6 +135,19 @@ bool AcceptLicenses (Context context, string sdkRoot) proc.StandardInput.WriteLine ('y'); proc.WaitForExit (); + // TODO/HACK? + // `emulator` package isn't usable with `cmdline-tools/1.0/bin/avdmanager` unless + // `sdkmanager` installs it; this creates an `emulator/packages.xml`, which is needed. + // Is there a workaround? + psi = new ProcessStartInfo (sdkManager, "emulator") { + UseShellExecute = false, + }; + if (!String.IsNullOrEmpty (jdkDir) && !psi.EnvironmentVariables.ContainsKey ("JAVA_HOME")) + psi.EnvironmentVariables.Add ("JAVA_HOME", jdkDir); + Log.DebugLine ($"Starting {psi.FileName} {psi.Arguments}"); + proc = Process.Start (psi); + proc.WaitForExit (); + return true; } diff --git a/build-tools/xaprepare/xaprepare/xaprepare.targets b/build-tools/xaprepare/xaprepare/xaprepare.targets index 541ce279d59..d3bf10180c0 100644 --- a/build-tools/xaprepare/xaprepare/xaprepare.targets +++ b/build-tools/xaprepare/xaprepare/xaprepare.targets @@ -53,6 +53,8 @@ + + From 79a417a0d4b8dee3eac83357c32b386720cea41e Mon Sep 17 00:00:00 2001 From: Jonathan Pryor Date: Sat, 9 May 2020 18:12:55 -0400 Subject: [PATCH 16/71] Bump to xamarin/monodroid/jonp-try-xa-xat-ref@beb7c8fc Context: https://github.com/xamarin/monodroid/pull/1091 Try to fix the `Xamarin.Android.Tools.AndroidSdk.dll` which is included into the installers, so that the xamarin-android-tools submodule specified in xamarin-android is the one included. --- .external | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.external b/.external index 8d8a2c8761e..88ba816a19f 100644 --- a/.external +++ b/.external @@ -1,2 +1,2 @@ -xamarin/monodroid:master@9242fa4fe5df2413faf12689c522825404b90755 +xamarin/monodroid:jonp-try-xa-xat-ref@beb7c8fcec9e04437c2512d876a045fe5cd4702a mono/mono:2020-02@075c3f06197e3b969f4234d0f56a2e10ee6ee305 From 1773d1baecdaa392a9235c381d66d88845dca3f9 Mon Sep 17 00:00:00 2001 From: Jonathan Pryor Date: Sun, 10 May 2020 09:52:52 -0400 Subject: [PATCH 17/71] Set JavacSourceVersion, JavacTargetVersion Commit 79a417a0 worked, in that we no longer get XA5300 errors about the JDK in use. Yay Things still fail, though: Task "Javac" (TaskId:154) Task Parameter:JavaPlatformJarPath=/Users/runner/Library/Android/sdk/platforms/android-29/android.jar (TaskId:154) Task Parameter:ClassesOutputDirectory=obj/Release/android/bin/classes/ (TaskId:154) Task Parameter:ClassesZip=obj/Release/android/bin/classes.zip (TaskId:154) Task Parameter:StubSourceDirectory=obj/Release/android/src/ (TaskId:154) Task Parameter:ToolPath=/Users/runner/Library/Android/jdk/bin (TaskId:154) Task Parameter: Jars= /Library/Frameworks/Mono.framework/External/xbuild-frameworks/MonoAndroid/v10.0/mono.android.jar /Library/Frameworks/Mono.framework/External/xbuild/Xamarin/Android/java_runtime.jar (TaskId:154) /Users/runner/Library/Android/jdk/bin/javac -J-Dfile.encoding=UTF8 "@/var/folders/24/8k48jl6d249_n_qfxwsl6xvm0000gn/T/tmp2eba46d8.tmp" (TaskId:154) JAVAC : error : option --boot-class-path not allowed with target 11 [/Users/runner/runners/2.168.2/work/1/s/bin/TestRelease/temp/ProguardBOMError/UnnamedProject.csproj] The command exited with code 2. (TaskId:154) JDK11 doesn't like `javac --boot-class-path` *unless* `javac -source` and `javac -target` are used, specifying a version less than 9. Provide default values for `$(JavacSourceVersion)` and `$(JavacTargetVersion)` in `Xamarin.Android.Common.props`, so that we tell `javac` that we're targeting JDK 8. --- .../Xamarin.Android.Common.props.in | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.props.in b/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.props.in index faf7bc6bfb4..1b69065eb91 100644 --- a/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.props.in +++ b/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.props.in @@ -21,6 +21,9 @@ @BUNDLETOOL_VERSION@ <_XamarinAndroidMSBuildDirectory>$(MSBuildThisFileDirectory) + 1.8 + 1.8 + true From 4eb11e8054bc6415dfe9ea15d6c5a42eed3e850a Mon Sep 17 00:00:00 2001 From: Jonathan Peppers Date: Mon, 11 May 2020 15:34:05 -0500 Subject: [PATCH 18/71] [build] move Microsoft OpenJDK on Windows Currently `xamarin-android-tools` prefers a `C:\Program Files\Android\jdk` over `%JAVA_HOME%`: https://github.com/xamarin/xamarin-android-tools/blob/f5fcb9fd1583ebe928734e8525303a0dbe93c79c/src/Xamarin.Android.Tools.AndroidSdk/Sdks/AndroidSdkWindows.cs#L137-L141 Before the test run, I made it move the Microsoft OpenJDK and restore it *after* the test run. We need to restore it, otherwise we could break other builds running against master right now. --- .../automation/yaml-templates/run-msbuild-win-tests.yaml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/build-tools/automation/yaml-templates/run-msbuild-win-tests.yaml b/build-tools/automation/yaml-templates/run-msbuild-win-tests.yaml index eed9fa58296..1a3340bf696 100644 --- a/build-tools/automation/yaml-templates/run-msbuild-win-tests.yaml +++ b/build-tools/automation/yaml-templates/run-msbuild-win-tests.yaml @@ -31,6 +31,9 @@ jobs: artifactName: $(NuGetArtifactName) downloadPath: $(System.DefaultWorkingDirectory)\bin\Build$(XA.Build.Configuration)\$(NuGetArtifactName) + - powershell: Move-Item "$env:ProgramFiles\Android\jdk\" "$env:ProgramFiles\Android\jdk_tmp\" + displayName: Move Microsoft OpenJDK + # Limit the amount of worker threads used to run these tests in parallel to half of what is currently available (8) on the Windows pool. # Using all available cores seems to occasionally bog down our machines and cause parallel test execution to slow down dramatically. - template: run-nunit-tests.yaml @@ -59,4 +62,8 @@ jobs: parameters: artifactName: Test Results - MSBuild - Windows-${{ parameters.node_id }} + - powershell: Move-Item "$env:ProgramFiles\Android\jdk_tmp\" "$env:ProgramFiles\Android\jdk\" + displayName: Restore Microsoft OpenJDK + condition: always() + - template: fail-on-issue.yaml From 527ac2942ef0ddf3d83decd7d9cd6bfd221f97e3 Mon Sep 17 00:00:00 2001 From: Jonathan Pryor Date: Wed, 13 May 2020 12:06:01 -0400 Subject: [PATCH 19/71] Remove `sdkmanager emulator` invocation and write `package.xml` ourselves. --- .../xaprepare/Steps/Step_Android_SDK_NDK.cs | 72 +++++++++++++++---- 1 file changed, 59 insertions(+), 13 deletions(-) diff --git a/build-tools/xaprepare/xaprepare/Steps/Step_Android_SDK_NDK.cs b/build-tools/xaprepare/xaprepare/Steps/Step_Android_SDK_NDK.cs index 126d9636396..56a0f93b283 100644 --- a/build-tools/xaprepare/xaprepare/Steps/Step_Android_SDK_NDK.cs +++ b/build-tools/xaprepare/xaprepare/Steps/Step_Android_SDK_NDK.cs @@ -4,6 +4,7 @@ using System.IO; using System.Linq; using System.Threading.Tasks; +using System.Xml.Linq; using Kajabity.Tools.Java; namespace Xamarin.Android.Prepare @@ -50,6 +51,7 @@ protected override async Task Execute (Context context) Log.ErrorLine ("Failed to accept Android SDK licenses"); return false; } + WritePackageXmls (sdkRoot); return GatherNDKInfo (context, ndkRoot); } @@ -94,6 +96,8 @@ protected override async Task Execute (Context context) return false; } + WritePackageXmls (sdkRoot); + return GatherNDKInfo (context, ndkRoot); } @@ -135,19 +139,6 @@ bool AcceptLicenses (Context context, string sdkRoot) proc.StandardInput.WriteLine ('y'); proc.WaitForExit (); - // TODO/HACK? - // `emulator` package isn't usable with `cmdline-tools/1.0/bin/avdmanager` unless - // `sdkmanager` installs it; this creates an `emulator/packages.xml`, which is needed. - // Is there a workaround? - psi = new ProcessStartInfo (sdkManager, "emulator") { - UseShellExecute = false, - }; - if (!String.IsNullOrEmpty (jdkDir) && !psi.EnvironmentVariables.ContainsKey ("JAVA_HOME")) - psi.EnvironmentVariables.Add ("JAVA_HOME", jdkDir); - Log.DebugLine ($"Starting {psi.FileName} {psi.Arguments}"); - proc = Process.Start (psi); - proc.WaitForExit (); - return true; } @@ -340,5 +331,60 @@ bool IsNdk (AndroidToolchainComponent component) { return component.Name.StartsWith ("android-ndk", StringComparison.OrdinalIgnoreCase); } + + static readonly XNamespace AndroidRepositoryCommon = "http://schemas.android.com/repository/android/common/01"; + static readonly XNamespace AndroidRepositoryGeneric = "http://schemas.android.com/repository/android/generic/01"; + + void WritePackageXmls (string sdkRoot) + { + string[] packageXmlDirs = new[]{ + Path.Combine (sdkRoot, "emulator"), + }; + foreach (var path in packageXmlDirs) { + var properties = ReadSourceProperties (path); + if (properties == null) + continue; + string packageXml = Path.Combine (path, "package.xml"); + Log.DebugLine ($"Writing '{packageXml}'"); + var doc = new XDocument( + new XElement (AndroidRepositoryCommon + "repository", + new XAttribute (XNamespace.Xmlns + "ns2", AndroidRepositoryCommon.NamespaceName), + new XAttribute (XNamespace.Xmlns + "ns3", AndroidRepositoryGeneric.NamespaceName), + new XElement ("localPackage", + new XAttribute ("path", properties ["Pkg.Path"]), + new XAttribute ("obsolete", "false"), + new XElement ("revision", GetRevision (properties ["Pkg.Revision"])), + new XElement ("display-name", properties ["Pkg.Desc"])))); + doc.Save (packageXml, SaveOptions.None); + } + } + + Dictionary? ReadSourceProperties (string dir) + { + var path = Path.Combine (dir, "source.properties"); + if (!File.Exists (path)) + return null; + var dict = new Dictionary (); + foreach (var line in File.ReadLines (path)) { + if (line.Length == 0) + continue; + var entry = line.Split (new[]{'='}, 2, StringSplitOptions.None); + if (entry.Length != 2) + continue; + dict.Add (entry [0], entry [1]); + } + return dict; + } + + IEnumerable GetRevision (string revision) + { + var parts = revision.Split ('.'); + if (parts.Length > 0) + yield return new XElement ("major", parts [0]); + if (parts.Length > 1) + yield return new XElement ("minor", parts [1]); + if (parts.Length > 2) + yield return new XElement ("micro", parts [2]); + } } } From d47bd6bf89d7951b2d12e457fba750c47e3756ee Mon Sep 17 00:00:00 2001 From: Jonathan Pryor Date: Thu, 14 May 2020 21:51:18 -0400 Subject: [PATCH 20/71] Fix ? MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Context: https://devdiv.visualstudio.com/DevDiv/_build/results?buildId=3729696&view=ms.vss-test-web.build-test-results-tab&runId=13228096&resultId=100066&paneView=attachments Current failures? `` is unhappy. More precisely, JDK11 `java` is unhappy: $ /Users/runner/Library/Android/jdk/bin/java \ -Djava.ext.dirs=/Users/runner/Library/Android/sdk/build-tools/29.0.2/lib \ com.android.multidex.MainDexListBuilder … -Djava.ext.dirs=/Users/runner/Library/Android/sdk/build-tools/29.0.2/lib is not supported. Use -classpath instead. CREATEMULTIDEXMAINDEXCLASSLIST : error : Could not create the Java Virtual Machine. CREATEMULTIDEXMAINDEXCLASSLIST : error : A fatal exception has occurred. Program will exit. The command exited with code 1. JDK11 `java` no longer supports `-Djava.ext.dirs`. Update `` so that it constructs a `java -classpath` value and uses that. Update `$(AndroidSdkBuildToolsVersion)` to 30.0.0-rc4 TODO: Is the `java -classpath` quoting correct? TODO: So about the `$(AndroidSdkBuildToolsVersion)` semantics: should it be in any way correlated with the JDK version in use? If so, *how*? We *know* that if `$(AndroidSdkBuildToolsVersion)` is >= 30, we need JDK11. How can we check that? If `$(AndroidSdkBuildToolsVersion)` is < 30, does the JDK matter? Given that all current failures are because of `java -Djava.ext.dirs`, it migiht not matter, i.e JDK 11 works with 29.0.2. --- .../Tasks/CreateMultiDexMainDexClassList.cs | 9 ++++++++- .../Xamarin.Android.Common.props.in | 2 +- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/CreateMultiDexMainDexClassList.cs b/src/Xamarin.Android.Build.Tasks/Tasks/CreateMultiDexMainDexClassList.cs index 1ead40be35a..d7937d24d87 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/CreateMultiDexMainDexClassList.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/CreateMultiDexMainDexClassList.cs @@ -4,6 +4,7 @@ using System.Collections.Generic; using System.Linq; using System.IO; +using System.Text; using Microsoft.Build.Utilities; using Microsoft.Build.Framework; @@ -97,7 +98,7 @@ void GenerateMainDexListBuilderCommands(CommandLineBuilder cmd) var enclosingDoubleQuote = OS.IsWindows ? "\"" : string.Empty; var enclosingQuote = OS.IsWindows ? string.Empty : "'"; var jars = JavaLibraries.Select (i => i.ItemSpec).Concat (new string [] { Path.Combine (ClassesOutputDirectory, "..", "classes.zip") }); - cmd.AppendSwitchIfNotNull ("-Djava.ext.dirs=", Path.Combine (AndroidSdkBuildToolsPath, "lib")); + cmd.AppendSwitchUnquotedIfNotNull ("-classpath ", "\"" + GetMainDexListBuilderClasspath () + "\""); cmd.AppendSwitch ("com.android.multidex.MainDexListBuilder"); if (!string.IsNullOrWhiteSpace (ExtraArgs)) cmd.AppendSwitch (ExtraArgs); @@ -108,6 +109,12 @@ void GenerateMainDexListBuilderCommands(CommandLineBuilder cmd) writeOutputToKeepFile = true; } + string GetMainDexListBuilderClasspath () + { + var libdir = Path.Combine (AndroidSdkBuildToolsPath, "lib"); + return string.Join (Path.PathSeparator.ToString (), Directory.EnumerateFiles (libdir, "*.jar")); + } + protected override void LogEventsFromTextOutput (string singleLine, MessageImportance messageImportance) { var match = CodeErrorRegEx.Match (singleLine); diff --git a/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.props.in b/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.props.in index 1b69065eb91..cadfe252268 100644 --- a/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.props.in +++ b/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.props.in @@ -14,7 +14,7 @@ True LowercaseCrc64 False - 29.0.2 + 30.0.0-rc4 29.0.5 26.1.1 16.1 From 01901fecfa5d2ed9bf4e82bdb0a579260f55f4f2 Mon Sep 17 00:00:00 2001 From: Jonathan Pryor Date: Fri, 15 May 2020 06:57:26 -0400 Subject: [PATCH 21/71] Set `$(LatestSupportedJavaVersion)`=11.0.4 Context: https://devdiv.visualstudio.com/DevDiv/_build/results?buildId=3730828&view=logs&j=96fd57f5-f69e-53c7-3d47-f67e6cf9b93e&t=65256bb7-a34c-5353-bc4d-c02ee25dc133 "Funny" interaction with `$(LatestSupportedJavaVersion)` and legacy `` and `$(AndroidSdkBuildToolsVersion)`: if `$(AndroidSdkBuildToolsVersion)` is not parseable by `Version.TryParse()` -- which it isn't, right now, as it is `30.0.0-rc4`, and the `-rc4` bit breaks parsing -- then we got an XA0032 error that JDK 11.99.99 needed to be installed, because the "latest supported version" becomes the "minimum required version". ...and here I was thinking that `$(LatestSupportedJavaVersion)` could be an "upper bound" for a version that may not yet exist. Additionally, setting `$(LatestSupportedJavaVersion)`=11.0.0 *also* fails: error XA0030: Building with JDK version `11.0.4` is not supported. Please install JDK version `11.0.0`. Immediate fix is to set `$(LatestSupportedJavaVersion)`=11.0.4 so that the version we installed matches the "minimum required". I'm not sure what a *good*, reliable, solution is. --- .../Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.props | 2 +- src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.props.in | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.props b/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.props index e88263325ad..0d17603012c 100644 --- a/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.props +++ b/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.props @@ -4,7 +4,7 @@ true - 11.99.0 + 11.0.4 1.8.0 false true diff --git a/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.props.in b/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.props.in index cadfe252268..835976c812f 100644 --- a/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.props.in +++ b/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.props.in @@ -7,7 +7,7 @@ false true true - 11.99.99 + 11.0.4 1.6.0 {abi}{versionCode:D5} UpdateGeneratedFiles From 6a7dfa3f17332f1e0ebd53ca98e6ea9a0124f84e Mon Sep 17 00:00:00 2001 From: Jonathan Pryor Date: Fri, 15 May 2020 12:02:46 -0400 Subject: [PATCH 22/71] Fix `make all-tests` Import `Configuration.props` so that projects use the JDK that xamarin-android provisions. --- samples/HelloWorld/HelloLibrary/HelloLibrary.csproj | 7 +++++++ tests/CodeBehind/BuildTests/CodeBehindBuildTests.csproj | 7 +++++++ .../CommonSampleLibrary/CommonSampleLibrary.csproj | 7 +++++++ 3 files changed, 21 insertions(+) diff --git a/samples/HelloWorld/HelloLibrary/HelloLibrary.csproj b/samples/HelloWorld/HelloLibrary/HelloLibrary.csproj index 03f205b8c89..a6fd9c67f60 100644 --- a/samples/HelloWorld/HelloLibrary/HelloLibrary.csproj +++ b/samples/HelloWorld/HelloLibrary/HelloLibrary.csproj @@ -16,6 +16,13 @@ True portable + + + $(AndroidFrameworkVersion) + true false diff --git a/tests/CodeBehind/BuildTests/CodeBehindBuildTests.csproj b/tests/CodeBehind/BuildTests/CodeBehindBuildTests.csproj index b45956d5f82..74452cab51c 100644 --- a/tests/CodeBehind/BuildTests/CodeBehindBuildTests.csproj +++ b/tests/CodeBehind/BuildTests/CodeBehindBuildTests.csproj @@ -20,6 +20,13 @@ portable <_ApkDebugKeyStore>debug.keystore + + + $(AndroidFrameworkVersion) + true false diff --git a/tests/CodeBehind/CommonSampleLibrary/CommonSampleLibrary.csproj b/tests/CodeBehind/CommonSampleLibrary/CommonSampleLibrary.csproj index 580c08556e3..5fa3c0fb41a 100644 --- a/tests/CodeBehind/CommonSampleLibrary/CommonSampleLibrary.csproj +++ b/tests/CodeBehind/CommonSampleLibrary/CommonSampleLibrary.csproj @@ -18,6 +18,13 @@ 2.0 portable + + + $(AndroidFrameworkVersion) + true false From 70695d9f93697b044912954e2d92edcb95a8e77e Mon Sep 17 00:00:00 2001 From: Jonathan Pryor Date: Fri, 15 May 2020 15:33:33 -0400 Subject: [PATCH 23/71] Fix Xamarin.Android.Build.Tests.BuildTest.GetDependencyWhenSDKIsMissingTest() Context: https://devdiv.visualstudio.com/DevDiv/_build/results?buildId=3731956&view=logs&j=afdd9f9c-cb6d-5c16-f94b-26085e0125c9&t=5e51d11f-7930-51de-e1cf-8c513d38b843 Wrong expected build-tools version. Update the tests to read `Xamarin.Android.Common.props.in` so that it expects what's in the repo. --- .../Xamarin.Android.Build.Tests/BuildTest.cs | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/BuildTest.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/BuildTest.cs index e2b4fdd1d1a..bc3cafb694c 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/BuildTest.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/BuildTest.cs @@ -3290,13 +3290,14 @@ public void GetDependencyWhenBuildToolsAreMissingTest () $"TargetFrameworkRootPath={referencesPath}", $"AndroidSdkDirectory={androidSdkPath}", }; + string buildToolsVersion = GetExpectedBuildToolsVersion (); using (var builder = CreateApkBuilder (Path.Combine (path, proj.ProjectName), cleanupAfterSuccessfulBuild: false, cleanupOnDispose: false)) { builder.ThrowOnBuildFailure = false; builder.Target = "GetAndroidDependencies"; Assert.True (builder.Build (proj, parameters: parameters), string.Format ("First Build should have succeeded")); StringAssertEx.Contains ("platforms/android-26", builder.LastBuildOutput, "platforms/android-26 should be a dependency."); - StringAssertEx.Contains ("build-tools/29.0.2", builder.LastBuildOutput, "build-tools/29.0.2 should be a dependency."); + StringAssertEx.Contains ($"build-tools/{buildToolsVersion}", builder.LastBuildOutput, $"build-tools/{buildToolsVersion} should be a dependency."); StringAssertEx.Contains ("platform-tools", builder.LastBuildOutput, "platform-tools should be a dependency."); } } @@ -3323,17 +3324,30 @@ public void GetDependencyWhenSDKIsMissingTest ([Values (true, false)] bool creat $"AndroidSdkDirectory={androidSdkPath}", }; + string buildToolsVersion = GetExpectedBuildToolsVersion (); using (var builder = CreateApkBuilder (Path.Combine (path, proj.ProjectName), cleanupAfterSuccessfulBuild: false, cleanupOnDispose: false)) { builder.ThrowOnBuildFailure = false; builder.Target = "GetAndroidDependencies"; Assert.True (builder.Build (proj, parameters: parameters), string.Format ("First Build should have succeeded")); StringAssertEx.Contains ("platforms/android-26", builder.LastBuildOutput, "platforms/android-26 should be a dependency."); - StringAssertEx.Contains ("build-tools/29.0.2", builder.LastBuildOutput, "build-tools/29.0.2 should be a dependency."); + StringAssertEx.Contains ($"build-tools/{buildToolsVersion}", builder.LastBuildOutput, $"build-tools/{buildToolsVersion} should be a dependency."); StringAssertEx.Contains ("platform-tools", builder.LastBuildOutput, "platform-tools should be a dependency."); } } + static readonly XNamespace MSBuildXmlns = "http://schemas.microsoft.com/developer/msbuild/2003"; + + static string GetExpectedBuildToolsVersion () + { + var propsPath = Path.Combine (XABuildPaths.TopDirectory, "src", "Xamarin.Android.Build.Tasks", "Xamarin.Android.Common.props.in"); + var props = XElement.Load (propsPath); + var AndroidSdkBuildToolsVersion = props.Elements (MSBuildXmlns+"PropertyGroup") + .Elements (MSBuildXmlns+"AndroidSdkBuildToolsVersion") + .FirstOrDefault (); + return AndroidSdkBuildToolsVersion?.Value?.Trim (); + } + [Test] public void ValidateUseLatestAndroid () { From fa8ac9e33561db7a9a4ff214dba45b3ff5fc8324 Mon Sep 17 00:00:00 2001 From: Jonathan Pryor Date: Fri, 15 May 2020 16:17:51 -0400 Subject: [PATCH 24/71] Use cmdline-tools lint `lint` is *also* in the new cmdline-tools package. Update things to look for it there. Add a new `$(AndroidCommandLineToolsVersion)` property. --- .../targets/Microsoft.Android.Sdk.Tooling.targets | 1 + .../targets/Microsoft.Android.Sdk.props | 1 + .../Tasks/CalculateProjectDependencies.cs | 5 +++++ .../Tasks/ResolveAndroidTooling.cs | 7 +++++++ src/Xamarin.Android.Build.Tasks/Tasks/ResolveSdksTask.cs | 9 +++++++++ .../Xamarin.Android.Common.props.in | 1 + .../Xamarin.Android.Common.targets | 5 +---- .../Xamarin.Android.Legacy.targets | 1 + .../Xamarin.Android.Tooling.targets | 2 ++ 9 files changed, 28 insertions(+), 4 deletions(-) diff --git a/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.Tooling.targets b/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.Tooling.targets index b34844b4b95..0642a552b6e 100644 --- a/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.Tooling.targets +++ b/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.Tooling.targets @@ -37,6 +37,7 @@ called for "legacy" projects in Xamarin.Android.Legacy.targets. AndroidUseAapt2="$(AndroidUseAapt2)" AotAssemblies="$(AotAssemblies)" Aapt2ToolPath="$(Aapt2ToolPath)" + CommandLineToolsVersion="$(AndroidCommandLineToolsVersion)" SequencePointsMode="$(_AndroidSequencePointsMode)" ProjectFilePath="$(MSBuildProjectFullPath)" LintToolPath="$(LintToolPath)" diff --git a/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.props b/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.props index 0d17603012c..4cb6c58578d 100644 --- a/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.props +++ b/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.props @@ -4,6 +4,7 @@ true + 1.0 11.0.4 1.8.0 false diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/CalculateProjectDependencies.cs b/src/Xamarin.Android.Build.Tasks/Tasks/CalculateProjectDependencies.cs index a19c4e820ab..e62ce998190 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/CalculateProjectDependencies.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/CalculateProjectDependencies.cs @@ -13,6 +13,8 @@ public class CalculateProjectDependencies : AndroidTask const int DefaultMinSDKVersion = 11; + public string CommandLineToolsVersion { get; set; } + [Required] public string TargetFrameworkVersion { get; set; } @@ -58,6 +60,9 @@ public override bool RunTask () if (!string.IsNullOrEmpty (PlatformToolsVersion)) { dependencies.Add (CreateAndroidDependency ("platform-tools", PlatformToolsVersion)); } + if (!string.IsNullOrEmpty (CommandLineToolsVersion)) { + dependencies.Add (CreateAndroidDependency ($"cmdline-tools/{CommandLineToolsVersion}", CommandLineToolsVersion)); + } if (!string.IsNullOrEmpty (ToolsVersion)) { dependencies.Add (CreateAndroidDependency ("tools", ToolsVersion)); } diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/ResolveAndroidTooling.cs b/src/Xamarin.Android.Build.Tasks/Tasks/ResolveAndroidTooling.cs index 4f9731f251f..063c8bacba1 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/ResolveAndroidTooling.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/ResolveAndroidTooling.cs @@ -23,6 +23,8 @@ public class ResolveAndroidTooling : AndroidTask public string AndroidSdkBuildToolsVersion { get; set; } + public string CommandLineToolsVersion { get; set; } + public string ProjectFilePath { get; set; } public string SequencePointsMode { get; set; } @@ -80,8 +82,13 @@ public override bool RunTask () string toolsZipAlignPath = Path.Combine (AndroidSdkPath, "tools", ZipAlign); bool findZipAlign = (string.IsNullOrEmpty (ZipAlignPath) || !Directory.Exists (ZipAlignPath)) && !File.Exists (toolsZipAlignPath); + var commandLineToolsDir = MonoAndroidHelper.AndroidSdk.GetCommandLineToolsPaths (CommandLineToolsVersion) + .FirstOrDefault () ?? ""; + var lintPaths = new string [] { LintToolPath ?? string.Empty, + commandLineToolsDir, + Path.Combine (commandLineToolsDir, "bin"), Path.Combine (AndroidSdkPath, "tools"), Path.Combine (AndroidSdkPath, "tools", "bin"), }; diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/ResolveSdksTask.cs b/src/Xamarin.Android.Build.Tasks/Tasks/ResolveSdksTask.cs index 96d73e22fc7..9bb97632aa6 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/ResolveSdksTask.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/ResolveSdksTask.cs @@ -43,6 +43,11 @@ public class ResolveSdks : AndroidTask public string [] ReferenceAssemblyPaths { get; set; } + public string CommandLineToolsVersion { get; set; } + + [Output] + public string CommandLineToolsPath { get; set; } + [Output] public string AndroidNdkPath { get; set; } @@ -94,6 +99,10 @@ public override bool RunTask () AndroidSdkPath = MonoAndroidHelper.AndroidSdk.AndroidSdkPath; JavaSdkPath = MonoAndroidHelper.AndroidSdk.JavaSdkPath; + CommandLineToolsPath = MonoAndroidHelper.AndroidSdk.GetCommandLineToolsPaths (CommandLineToolsVersion) + .FirstOrDefault () ?? + Path.Combine (AndroidSdkPath, "tools"); + if (string.IsNullOrEmpty (AndroidSdkPath)) { Log.LogCodedError ("XA5300", Properties.Resources.XA5300_Android_SDK); return false; diff --git a/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.props.in b/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.props.in index 835976c812f..0fa6c83b67e 100644 --- a/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.props.in +++ b/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.props.in @@ -9,6 +9,7 @@ true 11.0.4 1.6.0 + 1.0 {abi}{versionCode:D5} UpdateGeneratedFiles True diff --git a/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets b/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets index 7b2178de583..baa0d20b84d 100644 --- a/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets +++ b/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets @@ -652,10 +652,6 @@ because xbuild doesn't support framework reference assemblies. - - - - @@ -2814,6 +2810,7 @@ because xbuild doesn't support framework reference assemblies. + From ed3e037c5ac4be133d222cb22a6a8b12d3e7c545 Mon Sep 17 00:00:00 2001 From: Jonathan Peppers Date: Fri, 15 May 2020 15:41:12 -0500 Subject: [PATCH 25/71] Add .idsig files to @(FileWrites) Context: https://developer.android.com/preview/features#signature-scheme-v4 Contxt: https://developer.android.com/preview/features#incremental `apksigner` from `build-tools` 30 creates a new `.idsig` file next to the `.apk`. This is to enable a new `adb install --incremental` feature. We need to add this file to `@(FileWrites)` in the `_CollectMonoAndroidOutputs` MSBuild target. This target doesn't check if any of the files exist, so I think we can just always add to this item group. --- src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets b/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets index baa0d20b84d..76a39827c84 100644 --- a/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets +++ b/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets @@ -2705,6 +2705,7 @@ because xbuild doesn't support framework reference assemblies. + From 5d62091f5fb3d8556201ef7bd0becd1463637871 Mon Sep 17 00:00:00 2001 From: Jonathan Peppers Date: Fri, 15 May 2020 15:58:51 -0500 Subject: [PATCH 26/71] [tests] set /p:JavaSdkDirectory for MSBuild tests This makes the behavior match what we are doing for `$(AndroidSdkDirectory)` and `$(AndroidNdkDirectory)`. --- .../Xamarin.ProjectTools/Android/AndroidSdkResolver.cs | 9 +++++++++ .../Tests/Xamarin.ProjectTools/Common/Builder.cs | 4 ++++ 2 files changed, 13 insertions(+) diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Android/AndroidSdkResolver.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Android/AndroidSdkResolver.cs index 527f9ad6b83..08ca7ef68fc 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Android/AndroidSdkResolver.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Android/AndroidSdkResolver.cs @@ -41,5 +41,14 @@ public static string GetAndroidNdkPath () return ndkPath; } + public static string GetJavaSdkPath () + { + var jdkPath = Environment.GetEnvironmentVariable ("JAVA_HOME"); + if (String.IsNullOrEmpty (jdkPath)) + jdkPath = GetPathFromRegistry ("JavaSdkDirectory"); + if (String.IsNullOrEmpty (jdkPath)) + jdkPath = Path.GetFullPath (Path.Combine (ToolchainPath, "jdk")); + return jdkPath; + } } } diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Common/Builder.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Common/Builder.cs index ac65d2682ea..ac552719821 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Common/Builder.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Common/Builder.cs @@ -303,6 +303,10 @@ protected bool BuildInternal (string projectOrSolution, string target, string [] if (Directory.Exists (ndkPath)) { sw.WriteLine (" /p:AndroidNdkDirectory=\"{0}\" ", ndkPath); } + string jdkPath = AndroidSdkResolver.GetJavaSdkPath (); + if (Directory.Exists (jdkPath)) { + sw.WriteLine (" /p:JavaSdkDirectory=\"{0}\" ", jdkPath); + } if (parameters != null) { foreach (var param in parameters) { sw.WriteLine (" /p:{0}", param); From 4671de193de9063ec116087bd6a6556054b20684 Mon Sep 17 00:00:00 2001 From: Jonathan Peppers Date: Fri, 15 May 2020 15:59:09 -0500 Subject: [PATCH 27/71] Revert "[build] move Microsoft OpenJDK on Windows" This reverts commit 4eb11e8054bc6415dfe9ea15d6c5a42eed3e850a. --- .../automation/yaml-templates/run-msbuild-win-tests.yaml | 7 ------- 1 file changed, 7 deletions(-) diff --git a/build-tools/automation/yaml-templates/run-msbuild-win-tests.yaml b/build-tools/automation/yaml-templates/run-msbuild-win-tests.yaml index 1a3340bf696..eed9fa58296 100644 --- a/build-tools/automation/yaml-templates/run-msbuild-win-tests.yaml +++ b/build-tools/automation/yaml-templates/run-msbuild-win-tests.yaml @@ -31,9 +31,6 @@ jobs: artifactName: $(NuGetArtifactName) downloadPath: $(System.DefaultWorkingDirectory)\bin\Build$(XA.Build.Configuration)\$(NuGetArtifactName) - - powershell: Move-Item "$env:ProgramFiles\Android\jdk\" "$env:ProgramFiles\Android\jdk_tmp\" - displayName: Move Microsoft OpenJDK - # Limit the amount of worker threads used to run these tests in parallel to half of what is currently available (8) on the Windows pool. # Using all available cores seems to occasionally bog down our machines and cause parallel test execution to slow down dramatically. - template: run-nunit-tests.yaml @@ -62,8 +59,4 @@ jobs: parameters: artifactName: Test Results - MSBuild - Windows-${{ parameters.node_id }} - - powershell: Move-Item "$env:ProgramFiles\Android\jdk_tmp\" "$env:ProgramFiles\Android\jdk\" - displayName: Restore Microsoft OpenJDK - condition: always() - - template: fail-on-issue.yaml From f4960e89a41b8e9c4740e6e6f6632bcab743082a Mon Sep 17 00:00:00 2001 From: Jonathan Peppers Date: Mon, 18 May 2020 11:25:50 -0500 Subject: [PATCH 28/71] Set %JAVA_HOME% for Windows Build and Test --- build-tools/automation/azure-pipelines.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/build-tools/automation/azure-pipelines.yaml b/build-tools/automation/azure-pipelines.yaml index bfca97576fa..71947893ef7 100644 --- a/build-tools/automation/azure-pipelines.yaml +++ b/build-tools/automation/azure-pipelines.yaml @@ -233,6 +233,9 @@ stages: - template: yaml-templates\clean.yaml + - script: echo "##vso[task.setvariable variable=JAVA_HOME]%USERPROFILE%\android-toolchain\jdk" + displayName: set JAVA_HOME + - template: yaml-templates\use-dot-net.yaml parameters: version: $(DotNetCorePreviewVersion) From 841a05e2673a22852b07a842d5bc3c69b96b763a Mon Sep 17 00:00:00 2001 From: Jonathan Peppers Date: Mon, 18 May 2020 11:27:36 -0500 Subject: [PATCH 29/71] [test] accept SHA-256 for KeyToolTests --- .../Tests/Xamarin.Android.Build.Tests/Tasks/KeyToolTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Tasks/KeyToolTests.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Tasks/KeyToolTests.cs index 0917fd3da31..6e7f49aafcc 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Tasks/KeyToolTests.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Tasks/KeyToolTests.cs @@ -206,7 +206,7 @@ public void CreateDebugKeyStoreWithStrongPassword () Assert.AreEqual (0, warnings.Count, "Task should have no warnings."); Assert.AreEqual (0, task.ExitCode, "ExitCode should have been 0"); string output = string.Join (" ", messages.Select (x => x.Message)); - Assert.IsTrue (output.Contains ("Certificate fingerprint (SHA1):"), "Certificate SHA1 should have been printed."); + Assert.IsTrue (output.Contains ("Certificate fingerprint (SHA"), "Certificate SHA1 or SHA-256 should have been printed."); } } } From 101139c4fa3fab50464cd4cbe0cdfeb8d109b3dd Mon Sep 17 00:00:00 2001 From: Jonathan Peppers Date: Tue, 19 May 2020 08:54:25 -0500 Subject: [PATCH 30/71] [build] fix %JAVA_HOME% on Windows The previous code was somehow setting: JAVA_HOME = C:\Users\dlab14\android-toolchain\jdk" We think there is some bug with yaml behavior here, since the actual code looked correct. * I removed both quotes * I put the script in a block, otherwise the ## characters were yaml comments --- build-tools/automation/azure-pipelines.yaml | 3 ++- .../automation/yaml-templates/setup-test-environment.yaml | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/build-tools/automation/azure-pipelines.yaml b/build-tools/automation/azure-pipelines.yaml index 887aef0b400..3a25d3461bc 100644 --- a/build-tools/automation/azure-pipelines.yaml +++ b/build-tools/automation/azure-pipelines.yaml @@ -233,7 +233,8 @@ stages: - template: yaml-templates\clean.yaml - - script: echo "##vso[task.setvariable variable=JAVA_HOME]%USERPROFILE%\android-toolchain\jdk" + - script: | + echo ##vso[task.setvariable variable=JAVA_HOME]%USERPROFILE%\android-toolchain\jdk displayName: set JAVA_HOME - template: yaml-templates\use-dot-net.yaml diff --git a/build-tools/automation/yaml-templates/setup-test-environment.yaml b/build-tools/automation/yaml-templates/setup-test-environment.yaml index e0f71ec43c0..08f6de83336 100644 --- a/build-tools/automation/yaml-templates/setup-test-environment.yaml +++ b/build-tools/automation/yaml-templates/setup-test-environment.yaml @@ -15,7 +15,8 @@ steps: displayName: set JAVA_HOME condition: and(succeeded(), eq(variables['agent.os'], 'Darwin')) -- script: echo "##vso[task.setvariable variable=JAVA_HOME]%USERPROFILE%\android-toolchain\jdk" +- script: | + echo ##vso[task.setvariable variable=JAVA_HOME]%USERPROFILE%\android-toolchain\jdk displayName: set JAVA_HOME condition: and(succeeded(), eq(variables['agent.os'], 'Windows_NT')) From 026bcf76dbe14dc0198da4e05c9ce3b279c923fe Mon Sep 17 00:00:00 2001 From: Jonathan Peppers Date: Tue, 19 May 2020 15:31:14 -0500 Subject: [PATCH 31/71] Use $(AndroidFrameworkVersion) instead of $(AndroidUseLatestPlatformSdk) `Restore` was failing with: error XA0032: Java SDK 11.0.4 or above is required when using Android SDK Build-Tools 30.0.0-rc4. This target only runs during `Restore` when `$(AndroidUseLatestPlatformSdk)` is `true`, so we should just stop using this "deprecated" MSBuild property. This standardizes how we set `$(TargetFrameworkVersion)` across several test projects. Several projects were doing this, but also set `$(AndroidUseLatestPlatformSdk)`, which could just be removed. --- samples/HelloWorld/HelloLibrary/HelloLibrary.csproj | 1 - samples/HelloWorld/HelloWorld.csproj | 2 -- tests/BCL-Tests/LocalTests.NUnit/LocalTests.NUnit.csproj | 5 +++-- .../Xamarin.Android.Bcl-Tests.csproj | 5 +++-- tests/CodeBehind/BuildTests/CodeBehindBuildTests.csproj | 2 -- .../CommonSampleLibrary/CommonSampleLibrary.csproj | 2 -- .../Xamarin.Android.JcwGen-Tests.csproj | 5 +++-- tests/EmbeddedDSOs/EmbeddedDSO/EmbeddedDSO.csproj | 5 +++-- tests/TestRunner.xUnit/TestRunner.xUnit.csproj | 5 +++-- .../Xamarin.Android.Locale-Tests.csproj | 5 +++-- 10 files changed, 18 insertions(+), 19 deletions(-) diff --git a/samples/HelloWorld/HelloLibrary/HelloLibrary.csproj b/samples/HelloWorld/HelloLibrary/HelloLibrary.csproj index a6fd9c67f60..b6120d324da 100644 --- a/samples/HelloWorld/HelloLibrary/HelloLibrary.csproj +++ b/samples/HelloWorld/HelloLibrary/HelloLibrary.csproj @@ -13,7 +13,6 @@ HelloLibrary 512 false - True portable Properties\AndroidManifest.xml Resources Assets - true - v7.1 d8 portable diff --git a/tests/BCL-Tests/LocalTests.NUnit/LocalTests.NUnit.csproj b/tests/BCL-Tests/LocalTests.NUnit/LocalTests.NUnit.csproj index 9d7fb214e89..4e6cc859724 100644 --- a/tests/BCL-Tests/LocalTests.NUnit/LocalTests.NUnit.csproj +++ b/tests/BCL-Tests/LocalTests.NUnit/LocalTests.NUnit.csproj @@ -8,14 +8,15 @@ Library LocalTests.NUnit LocalTests.NUnit - v8.1 Resources\Resource.designer.cs Resource Resources Assets - true + + $(AndroidFrameworkVersion) + true false diff --git a/tests/BCL-Tests/Xamarin.Android.Bcl-Tests/Xamarin.Android.Bcl-Tests.csproj b/tests/BCL-Tests/Xamarin.Android.Bcl-Tests/Xamarin.Android.Bcl-Tests.csproj index 32c40b79c9c..3d53c373d09 100644 --- a/tests/BCL-Tests/Xamarin.Android.Bcl-Tests/Xamarin.Android.Bcl-Tests.csproj +++ b/tests/BCL-Tests/Xamarin.Android.Bcl-Tests/Xamarin.Android.Bcl-Tests.csproj @@ -9,18 +9,19 @@ Library UnitTestRunner Xamarin.Android.Bcl_Tests - v8.1 True Resources\Resource.designer.cs Resource Properties\AndroidManifest.xml Resources Assets - true false armeabi-v7a;x86 + + $(AndroidFrameworkVersion) + true false diff --git a/tests/CodeBehind/BuildTests/CodeBehindBuildTests.csproj b/tests/CodeBehind/BuildTests/CodeBehindBuildTests.csproj index 74452cab51c..e402d9d4f45 100644 --- a/tests/CodeBehind/BuildTests/CodeBehindBuildTests.csproj +++ b/tests/CodeBehind/BuildTests/CodeBehindBuildTests.csproj @@ -8,14 +8,12 @@ Library Xamarin.Android.Tests.CodeBehindBuildTests CodeBehindBuildTests - v8.1 True Resources\Resource.designer.cs Resource Properties\AndroidManifest.xml Resources Assets - true True portable <_ApkDebugKeyStore>debug.keystore diff --git a/tests/CodeBehind/CommonSampleLibrary/CommonSampleLibrary.csproj b/tests/CodeBehind/CommonSampleLibrary/CommonSampleLibrary.csproj index 5fa3c0fb41a..90d03f94f3a 100644 --- a/tests/CodeBehind/CommonSampleLibrary/CommonSampleLibrary.csproj +++ b/tests/CodeBehind/CommonSampleLibrary/CommonSampleLibrary.csproj @@ -8,12 +8,10 @@ Library CommonSampleLibrary Assets - true Resources Resources\Resource.designer.cs Resource CommonSampleLibrary - v8.1 8.0.30703 2.0 portable diff --git a/tests/CodeGen-Binding/Xamarin.Android.JcwGen-Tests/Xamarin.Android.JcwGen-Tests.csproj b/tests/CodeGen-Binding/Xamarin.Android.JcwGen-Tests/Xamarin.Android.JcwGen-Tests.csproj index a253079d8ab..5d0e2cf993b 100644 --- a/tests/CodeGen-Binding/Xamarin.Android.JcwGen-Tests/Xamarin.Android.JcwGen-Tests.csproj +++ b/tests/CodeGen-Binding/Xamarin.Android.JcwGen-Tests/Xamarin.Android.JcwGen-Tests.csproj @@ -17,12 +17,13 @@ Xamarin.Android.JcwGen-Tests Properties\AndroidManifest.xml armeabi-v7a;x86 - v9.0 false - true preview + + $(AndroidFrameworkVersion) + true false diff --git a/tests/EmbeddedDSOs/EmbeddedDSO/EmbeddedDSO.csproj b/tests/EmbeddedDSOs/EmbeddedDSO/EmbeddedDSO.csproj index a30e5a8213d..1f21b85919d 100644 --- a/tests/EmbeddedDSOs/EmbeddedDSO/EmbeddedDSO.csproj +++ b/tests/EmbeddedDSOs/EmbeddedDSO/EmbeddedDSO.csproj @@ -8,14 +8,12 @@ Library EmbeddedDSO Xamarin.Android.EmbeddedDSO_Test - v8.1 True Resources\Resource.designer.cs Resource Properties\AndroidManifest.xml Resources Assets - true false @@ -28,6 +26,9 @@ + + $(AndroidFrameworkVersion) +