From 35d790302b47a9354bf07886bfc59f67eddb900f Mon Sep 17 00:00:00 2001 From: Jonathan Peppers Date: Thu, 25 Oct 2018 13:32:19 -0500 Subject: [PATCH] [tests] NO dropbox, fix locking in Xamarin.ProjectTools Context: https://jenkins.mono-project.com/job/xamarin-android-pr-builder/4408/testReport/junit/Xamarin.Android.Build.Tests/BuildTest/BuildBasicApplicationCheckPdb___Debug/ A random test failure has occurred such as: Xamarin.Android.Build.Tests.BuildTest.BuildBasicApplicationCheckPdb System.IO.IOException : Sharing violation on path /Users/builder/.local/share/Xamarin.ProjectTools/PdbTestLibrary.pdb at System.IO.FileStream..ctor (System.String path, System.IO.FileMode mode, System.IO.FileAccess access, System.IO.FileShare share, System.Int32 bufferSize, System.Boolean anonymous, System.IO.FileOptions options) [0x0019e] in <9e4df56871b74651838be0f6a9e1bc80>:0 at System.IO.FileStream..ctor (System.String path, System.IO.FileMode mode, System.IO.FileAccess access, System.IO.FileShare share) [0x00000] in <9e4df56871b74651838be0f6a9e1bc80>:0 at (wrapper remoting-invoke-with-check) System.IO.FileStream..ctor(string,System.IO.FileMode,System.IO.FileAccess,System.IO.FileShare) at System.IO.File.OpenRead (System.String path) [0x00000] in <9e4df56871b74651838be0f6a9e1bc80>:0 at System.IO.File.ReadAllBytes (System.String path) [0x00000] in <9e4df56871b74651838be0f6a9e1bc80>:0 at Xamarin.ProjectTools.BuildItem+<>c__DisplayClass71_0.b__0 () [0x0004f] in <7ee8601bba4749f49f3dee0ccd7e3cca>:0 at Xamarin.ProjectTools.XamarinProject+<>c.b__88_4 (Xamarin.ProjectTools.BuildItem s) [0x00050] in <7ee8601bba4749f49f3dee0ccd7e3cca>:0 at System.Linq.Enumerable+SelectListIterator`2[TSource,TResult].MoveNext () [0x00048] in <7505d9a21cfc4fc7b41cb4768918fdb4>:0 at System.Collections.Generic.List`1[T].AddEnumerable (System.Collections.Generic.IEnumerable`1[T] enumerable) [0x00059] in <9e4df56871b74651838be0f6a9e1bc80>:0 at System.Collections.Generic.List`1[T].InsertRange (System.Int32 index, System.Collections.Generic.IEnumerable`1[T] collection) [0x000f4] in <9e4df56871b74651838be0f6a9e1bc80>:0 at System.Collections.Generic.List`1[T].AddRange (System.Collections.Generic.IEnumerable`1[T] collection) [0x00000] in <9e4df56871b74651838be0f6a9e1bc80>:0 at Xamarin.ProjectTools.XamarinProject.Save (System.Boolean saveProject) [0x001ec] in <7ee8601bba4749f49f3dee0ccd7e3cca>:0 at Xamarin.ProjectTools.ProjectBuilder.Save (Xamarin.ProjectTools.XamarinProject project, System.Boolean doNotCleanupOnUpdate, System.Boolean saveProject) [0x00001] in <7ee8601bba4749f49f3dee0ccd7e3cca>:0 at Xamarin.ProjectTools.ProjectBuilder.Build (Xamarin.ProjectTools.XamarinProject project, System.Boolean doNotCleanupOnUpdate, System.String[] parameters, System.Boolean saveProject, System.Collections.Generic.Dictionary`2[TKey,TValue] environmentVariables) [0x00001] in <7ee8601bba4749f49f3dee0ccd7e3cca>:0 at Xamarin.Android.Build.Tests.BuildTest.BuildBasicApplicationCheckPdb () [0x00102] in :0 at (wrapper managed-to-native) System.Reflection.MonoMethod.InternalInvoke(System.Reflection.MonoMethod,object,object[],System.Exception&) at System.Reflection.MonoMethod.Invoke (System.Object obj, System.Reflection.BindingFlags invokeAttr, System.Reflection.Binder binder, System.Object[] parameters, System.Globalization.CultureInfo culture) [0x0003b] in <9e4df56871b74651838be0f6a9e1bc80>:0 ~~ BuildTest ~~ Looking at the test which this occurred, we have two tests using different URLs, but the same filename: https://www.dropbox.com/s/s4br29kvuy8ygz1/PdbTestLibrary.dll?dl=1 https://dl.dropboxusercontent.com/u/18881050/Xamarin/PdbTestLibrary.dll The second one 404's! Meaning, this test could only possibly *succeed* when the first file has already been downloaded and cached! This test could certainly fail on a new build bot, depending on the ordering of the test run. So that brings up a point... How do we know these dropbox URLs will always work? We don't. I went through and uploaded all the files to our Azure storage account that is already used for the Mono bundle and other build artifacts. These files will be downloaded from: https://xamjenkinsartifact.azureedge.net/mono-jenkins/xamarin-android-test/{filename} I added a new `WebContentFileNameFromAzure` to `BuildItem` that can be used here instead of `WebContent`. Then the filename can just be specified, assuming the file has been uploaded. ~~ DownloadedCache ~~ In 88d98aa I added some locking to help concurrent downloads. This greatly helped test runs on Windows. However, it was not quite precise enough: The locking was done upon the URL string, and in this instance the URLs differ, but the cache filename would be the same. I moved the locking *into* the `DownloadedCache` cache class, where it should have been in the first place. The locking functionality is the same, except it will now lock on the cached filename instead of the URL string. I also removed a `Directory.Exists` check that isn't needed. --- .../BindingBuildTest.cs | 16 ++++++------- .../Xamarin.Android.Build.Tests/BuildTest.cs | 16 ++++++------- .../Xamarin.ProjectTools/Common/BuildItem.cs | 17 ++++++++------ .../Common/DownloadedCache.cs | 23 +++++++++++-------- 4 files changed, 39 insertions(+), 33 deletions(-) diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/BindingBuildTest.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/BindingBuildTest.cs index 221c81a44b4..3395f68e543 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/BindingBuildTest.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/BindingBuildTest.cs @@ -111,7 +111,7 @@ public void BuildLibraryZipBindigLibraryWithAarOfJar (string classParser) proj.AndroidClassParser = classParser; proj.Packages.Add (KnownPackages.AndroidSupportV4_22_1_1_1); proj.Jars.Add (new AndroidItem.LibraryProjectZip ("Jars\\aFileChooserBinaries.zip") { - WebContent = "https://www.dropbox.com/s/hl98jrvlw4d9vjy/aFileChooserBinaries.zip?dl=1" + WebContentFileNameFromAzure = "aFileChooserBinaries.zip" }); proj.MetadataXml = @" @@ -167,7 +167,7 @@ public void BindByteArrayInMethodParameter () AndroidClassParser = "class-parse", }; proj.Jars.Add (new AndroidItem.EmbeddedJar ("Jars\\svg-android.jar") { - WebContent = "https://www.dropbox.com/s/5ovudccigydohys/javaBindingIssue.jar?dl=1" + WebContentFileNameFromAzure = "javaBindingIssue.jar" }); using (var b = CreateDllBuilder ("temp/BindByteArrayInMethodParameter")) { Assert.IsTrue (b.Build (proj), "Build should have succeeded."); @@ -182,7 +182,7 @@ public void MergeAndroidManifest () }; binding.AndroidClassParser = "class-parse"; binding.Jars.Add (new AndroidItem.LibraryProjectZip ("Jars\\adal-1.0.7.aar") { - WebContent = "https://www.dropbox.com/s/bubopadhd9d1l4b/adal-1.0.7.aar?dl=1" + WebContentFileNameFromAzure = "adal-1.0.7.aar" }); binding.MetadataXml = @" @@ -213,7 +213,7 @@ public void AnnotationSupport () }; binding.AndroidClassParser = "class-parse"; binding.Jars.Add (new AndroidItem.LibraryProjectZip ("Jars\\mylibrary.aar") { - WebContent = "https://www.dropbox.com/s/53679881kg9rdiq/mylibrary-debug.aar?dl=1" + WebContentFileNameFromAzure = "mylibrary-debug.aar" }); var bindingBuilder = CreateDllBuilder ("temp/AnnotationSupport"); Assert.IsTrue (bindingBuilder.Build (binding), "binding build failed"); @@ -251,7 +251,7 @@ public void BindngFilterUnsupportedNativeAbiLibraries () }; binding.AndroidClassParser = "class-parse"; binding.Jars.Add (new AndroidItem.LibraryProjectZip ("Jars\\mylibrary.aar") { - WebContent = "https://www.dropbox.com/s/apphdrh9cjqvtye/card.io-5.3.0.aar?dl=1" + WebContentFileNameFromAzure = "card.io-5.3.0.aar" }); using (var bindingBuilder = CreateDllBuilder (Path.Combine ("temp", "BindngFilterUnsupportedNativeAbiLibraries", "Binding"))) { Assert.IsTrue (bindingBuilder.Build (binding), "binding build should have succeeded"); @@ -272,10 +272,10 @@ public void BindingCheckHiddenFiles ([Values (true, false)] bool useShortFileNam }; binding.AndroidClassParser = "class-parse"; binding.Jars.Add (new AndroidItem.LibraryProjectZip ("Jars\\mylibrary.aar") { - WebContent = "https://www.dropbox.com/s/astiqp8jo97x91h/mylibrary.aar?dl=1" + WebContentFileNameFromAzure = "mylibrary.aar" }); binding.Jars.Add (new AndroidItem.EmbeddedJar ("Jars\\svg-android.jar") { - WebContent = "https://www.dropbox.com/s/5ovudccigydohys/javaBindingIssue.jar?dl=1" + WebContentFileNameFromAzure = "javaBindingIssue.jar" }); var path = Path.Combine ("temp", TestContext.CurrentContext.Test.Name); binding.SetProperty (binding.ActiveConfigurationProperties, "UseShortFileNames", useShortFileNames); @@ -321,7 +321,7 @@ public void BindingDoNotPackage () IsRelease = true, Jars = { new AndroidItem.EmbeddedJar ("Jars\\svg-android.jar") { - WebContent = "https://www.dropbox.com/s/5ovudccigydohys/javaBindingIssue.jar?dl=1" + WebContentFileNameFromAzure = "javaBindingIssue.jar" } }, AssemblyInfo = @" diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/BuildTest.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/BuildTest.cs index 7f9bd1ab2c2..0778c1d382a 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/BuildTest.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/BuildTest.cs @@ -1035,19 +1035,19 @@ public void BuildBasicApplicationCheckMdbAndPortablePdb () using (var b = CreateApkBuilder ("temp/BuildBasicApplicationCheckMdbAndPortablePdb")) { b.Verbosity = LoggerVerbosity.Diagnostic; var reference = new BuildItem.Reference ("PdbTestLibrary.dll") { - WebContent = "https://www.dropbox.com/s/s4br29kvuy8ygz1/PdbTestLibrary.dll?dl=1" + WebContentFileNameFromAzure = "PdbTestLibrary.dll" }; proj.References.Add (reference); var pdb = new BuildItem.NoActionResource ("PdbTestLibrary.pdb") { - WebContent = "https://www.dropbox.com/s/033jif54ma0e01m/PdbTestLibrary.pdb?dl=1" + WebContentFileNameFromAzure = "PdbTestLibrary.pdb" }; proj.References.Add (pdb); var netStandardRef = new BuildItem.Reference ("NetStandard16.dll") { - WebContent = "https://www.dropbox.com/s/g7v0d4irzvaw5pl/NetStandard16.dll?dl=1" + WebContentFileNameFromAzure = "NetStandard16.dll" }; proj.References.Add (netStandardRef); var netStandardpdb = new BuildItem.NoActionResource ("NetStandard16.pdb") { - WebContent = "https://www.dropbox.com/s/m898ix2m2il631y/NetStandard16.pdb?dl=1" + WebContentFileNameFromAzure = "NetStandard16.pdb" }; proj.References.Add (netStandardpdb); Assert.IsTrue (b.Build (proj), "Build should have succeeded."); @@ -2436,19 +2436,19 @@ public void BuildBasicApplicationCheckPdb () using (var b = CreateApkBuilder ("temp/BuildBasicApplicationCheckPdb", false, false)) { b.Verbosity = LoggerVerbosity.Diagnostic; var reference = new BuildItem.Reference ("PdbTestLibrary.dll") { - WebContent = "https://dl.dropboxusercontent.com/u/18881050/Xamarin/PdbTestLibrary.dll" + WebContentFileNameFromAzure = "PdbTestLibrary.dll" }; proj.References.Add (reference); var pdb = new BuildItem.NoActionResource ("PdbTestLibrary.pdb") { - WebContent = "https://dl.dropboxusercontent.com/u/18881050/Xamarin/PdbTestLibrary.pdb" + WebContentFileNameFromAzure = "PdbTestLibrary.pdb" }; proj.References.Add (pdb); var netStandardRef = new BuildItem.Reference ("NetStandard16.dll") { - WebContent = "https://dl.dropboxusercontent.com/u/18881050/Xamarin/NetStandard16.dll" + WebContentFileNameFromAzure = "NetStandard16.dll" }; proj.References.Add (netStandardRef); var netStandardpdb = new BuildItem.NoActionResource ("NetStandard16.pdb") { - WebContent = "https://dl.dropboxusercontent.com/u/18881050/Xamarin/NetStandard16.pdb" + WebContentFileNameFromAzure = "NetStandard16.pdb" }; proj.References.Add (netStandardpdb); Assert.IsTrue (b.Build (proj), "Build should have succeeded."); diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Common/BuildItem.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Common/BuildItem.cs index 5eae5161088..5a21c3b5edf 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Common/BuildItem.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Common/BuildItem.cs @@ -1,5 +1,4 @@ using System; -using System.Collections.Concurrent; using System.Collections.Generic; using System.IO; using System.Linq; @@ -123,20 +122,24 @@ public BuildItem (string buildAction, Func include = null) public bool Deleted { get; set; } public FileAttributes Attributes { get; set;} - static readonly ConcurrentDictionary locks = new ConcurrentDictionary (); - public string WebContent { get { throw new NotSupportedException (); } set { BinaryContent = () => { - lock (locks.GetOrAdd (value, _ => new object ())) { - var file = new DownloadedCache ().GetAsFile (value); - return File.ReadAllBytes (file); - } + var file = new DownloadedCache ().GetAsFile (value); + return File.ReadAllBytes (file); }; } } + /// + /// NOTE: downloads a file from our https://xamjenkinsartifact.azureedge.net/ Azure Storage Account + /// + public string WebContentFileNameFromAzure { + get { throw new NotSupportedException (); } + set { WebContent = $"https://xamjenkinsartifact.azureedge.net/mono-jenkins/xamarin-android-test/{value}"; } + } + public string MetadataValues { get { return string.Join (";", Metadata.Select (p => p.Key + '=' + p.Value)); } set { diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Common/DownloadedCache.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Common/DownloadedCache.cs index a51a6c2f182..500f3ae00ab 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Common/DownloadedCache.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Common/DownloadedCache.cs @@ -1,13 +1,14 @@ using System; +using System.Collections.Concurrent; using System.IO; -using System.Security.Cryptography; -using System.Xml; using System.Linq; +using System.Security.Cryptography; namespace Xamarin.ProjectTools { public class DownloadedCache { + static readonly ConcurrentDictionary locks = new ConcurrentDictionary (); public DownloadedCache () : this (Path.Combine (Environment.GetFolderPath (Environment.SpecialFolder.LocalApplicationData), "Xamarin.ProjectTools")) @@ -25,16 +26,18 @@ public DownloadedCache (string cacheDirectory) public string GetAsFile (string url, string md5 = null) { - if (!Directory.Exists (CacheDirectory)) - Directory.CreateDirectory (CacheDirectory); + Directory.CreateDirectory (CacheDirectory); + var filename = Path.Combine (CacheDirectory, Path.GetFileName (new Uri (url).LocalPath)); - if (File.Exists (filename) && (md5 == null || GetMd5 (filename) == md5)) + lock (locks.GetOrAdd (filename, _ => new object ())) { + if (File.Exists (filename) && (md5 == null || GetMd5 (filename) == md5)) + return filename; + // FIXME: should be clever enough to resolve name conflicts. + new System.Net.WebClient ().DownloadFile (url, filename); + if (md5 != null && GetMd5 (filename) != md5) + throw new InvalidOperationException (string.Format ("The given md5sum doesn't match the actual web resource. '{0}' for '{1}'", md5, url)); return filename; - // FIXME: should be clever enough to resolve name conflicts. - new System.Net.WebClient ().DownloadFile (url, filename); - if (md5 != null && GetMd5 (filename) != md5) - throw new InvalidOperationException (string.Format ("The given md5sum doesn't match the actual web resource. '{0}' for '{1}'", md5, url)); - return filename; + } } string GetMd5 (string filename)