From 5f5257cc52e38b2835c78017f1c24e5e9379dbdb Mon Sep 17 00:00:00 2001 From: iouri-s Date: Mon, 28 Sep 2015 08:45:14 -0700 Subject: [PATCH] Add an option to skip MD5 computation and checking. Also disable MD5 computation when using HTTPS because it is redundant. --- .../Model/UploadParameters.cs | 5 ++++- .../Model/VhdUploaderModel.cs | 2 +- .../StorageServices/AddAzureVhdCommand.cs | 11 ++++++++++- .../Compute/Sync/Upload/BlobCreator.cs | 4 ++-- .../Compute/Sync/Upload/BlobCreatorBase.cs | 6 ++++-- .../Compute/Sync/Upload/BlobSynchronizer.cs | 16 +++++++++++++--- .../Sync/Upload/UploadOperationMetaData.cs | 6 +++--- 7 files changed, 37 insertions(+), 13 deletions(-) diff --git a/src/ServiceManagement/Compute/Commands.ServiceManagement/Model/UploadParameters.cs b/src/ServiceManagement/Compute/Commands.ServiceManagement/Model/UploadParameters.cs index d53e1432694a..4b89c571d8e5 100644 --- a/src/ServiceManagement/Compute/Commands.ServiceManagement/Model/UploadParameters.cs +++ b/src/ServiceManagement/Compute/Commands.ServiceManagement/Model/UploadParameters.cs @@ -21,12 +21,13 @@ namespace Microsoft.WindowsAzure.Commands.ServiceManagement.Model { public class UploadParameters { - public UploadParameters(BlobUri destinationUri, BlobUri baseImageUri, FileInfo localFilePath, bool overWrite, int numberOfUploaderThreads) + public UploadParameters(BlobUri destinationUri, BlobUri baseImageUri, FileInfo localFilePath, bool overWrite, bool skipMd5, int numberOfUploaderThreads) { DestinationUri = destinationUri; BaseImageUri = baseImageUri; LocalFilePath = localFilePath; OverWrite = overWrite; + SkipMd5 = skipMd5; NumberOfUploaderThreads = numberOfUploaderThreads; } @@ -38,6 +39,8 @@ public UploadParameters(BlobUri destinationUri, BlobUri baseImageUri, FileInfo l public bool OverWrite { get; private set; } + public bool SkipMd5 { get; private set; } + public int NumberOfUploaderThreads { get; private set; } public AddAzureVhdCommand Cmdlet { get; set; } diff --git a/src/ServiceManagement/Compute/Commands.ServiceManagement/Model/VhdUploaderModel.cs b/src/ServiceManagement/Compute/Commands.ServiceManagement/Model/VhdUploaderModel.cs index 49ec79d4265e..3b5af363ef43 100644 --- a/src/ServiceManagement/Compute/Commands.ServiceManagement/Model/VhdUploaderModel.cs +++ b/src/ServiceManagement/Compute/Commands.ServiceManagement/Model/VhdUploaderModel.cs @@ -30,7 +30,7 @@ public static VhdUploadContext Upload(UploadParameters uploadParameters) } else { - blobCreator = new BlobCreator(uploadParameters.LocalFilePath, uploadParameters.DestinationUri, uploadParameters.BlobObjectFactory, uploadParameters.OverWrite); + blobCreator = new BlobCreator(uploadParameters.LocalFilePath, uploadParameters.DestinationUri, uploadParameters.BlobObjectFactory, uploadParameters.OverWrite, uploadParameters.SkipMd5); } using (var uploadContext = blobCreator.Create()) diff --git a/src/ServiceManagement/Compute/Commands.ServiceManagement/StorageServices/AddAzureVhdCommand.cs b/src/ServiceManagement/Compute/Commands.ServiceManagement/StorageServices/AddAzureVhdCommand.cs index a2aa33e23440..3deb899f8730 100644 --- a/src/ServiceManagement/Compute/Commands.ServiceManagement/StorageServices/AddAzureVhdCommand.cs +++ b/src/ServiceManagement/Compute/Commands.ServiceManagement/StorageServices/AddAzureVhdCommand.cs @@ -78,6 +78,15 @@ public SwitchParameter OverWrite set; } + [Parameter(Position = 6, Mandatory = false, ValueFromPipelineByPropertyName = true, ParameterSetName = "Vhd", HelpMessage = "Skip computation of MD5 checksum")] + [ValidateNotNullOrEmpty] + [Alias("sm")] + public SwitchParameter SkipMd5 + { + get; + set; + } + public UploadParameters ValidateParameters() { BlobUri destinationUri; @@ -106,7 +115,7 @@ public UploadParameters ValidateParameters() PathIntrinsics currentPath = SessionState.Path; var filePath = new FileInfo(currentPath.GetUnresolvedProviderPathFromPSPath(LocalFilePath.ToString())); - var parameters = new UploadParameters(destinationUri, baseImageUri, filePath, OverWrite.IsPresent, NumberOfUploaderThreads) + var parameters = new UploadParameters(destinationUri, baseImageUri, filePath, OverWrite.IsPresent, SkipMd5.IsPresent, NumberOfUploaderThreads) { Cmdlet = this, BlobObjectFactory = new CloudPageBlobObjectFactory(storageCredentialsFactory, TimeSpan.FromMinutes(1)) diff --git a/src/ServiceManagement/Compute/Sync/Upload/BlobCreator.cs b/src/ServiceManagement/Compute/Sync/Upload/BlobCreator.cs index bd562d84cb56..b3f0f25ff841 100644 --- a/src/ServiceManagement/Compute/Sync/Upload/BlobCreator.cs +++ b/src/ServiceManagement/Compute/Sync/Upload/BlobCreator.cs @@ -19,8 +19,8 @@ namespace Microsoft.WindowsAzure.Commands.Sync.Upload { public class BlobCreator : BlobCreatorBase { - public BlobCreator(FileInfo localVhd, BlobUri destination, ICloudPageBlobObjectFactory blobObjectFactory, bool overWrite) : - base(localVhd, destination, blobObjectFactory, overWrite) + public BlobCreator(FileInfo localVhd, BlobUri destination, ICloudPageBlobObjectFactory blobObjectFactory, bool overWrite, bool skipMd5 = false) : + base(localVhd, destination, blobObjectFactory, overWrite, skipMd5) { } diff --git a/src/ServiceManagement/Compute/Sync/Upload/BlobCreatorBase.cs b/src/ServiceManagement/Compute/Sync/Upload/BlobCreatorBase.cs index f6de6e4dd8f1..20a9139872f6 100644 --- a/src/ServiceManagement/Compute/Sync/Upload/BlobCreatorBase.cs +++ b/src/ServiceManagement/Compute/Sync/Upload/BlobCreatorBase.cs @@ -50,19 +50,21 @@ public abstract class BlobCreatorBase protected CloudPageBlob destinationBlob; protected BlobRequestOptions requestOptions; protected bool overWrite; + protected bool skipMd5; private readonly TimeSpan delayBetweenRetries = TimeSpan.FromSeconds(10); private const int MegaByte = 1024 * 1024; private const int PageSizeInBytes = 2 * MegaByte; private const int MaxBufferSize = 20 * MegaByte; - protected BlobCreatorBase(FileInfo localVhd, BlobUri blobDestination, ICloudPageBlobObjectFactory blobObjectFactory, bool overWrite) + protected BlobCreatorBase(FileInfo localVhd, BlobUri blobDestination, ICloudPageBlobObjectFactory blobObjectFactory, bool overWrite, bool skipMd5 = false) { this.localVhd = localVhd; this.blobObjectFactory = blobObjectFactory; this.destination = new Uri(blobDestination.BlobPath); this.blobDestination = blobDestination; this.overWrite = overWrite; + this.skipMd5 = skipMd5; this.destinationBlob = blobObjectFactory.Create(blobDestination); this.requestOptions = this.blobObjectFactory.CreateRequestOptions(); @@ -79,7 +81,7 @@ public LocalMetaData OperationMetaData { this.operationMetaData = new LocalMetaData { - FileMetaData = FileMetaData.Create(localVhd.FullName), + FileMetaData = FileMetaData.Create(localVhd.FullName, skipMd5), SystemInformation = SystemInformation.Create() }; } diff --git a/src/ServiceManagement/Compute/Sync/Upload/BlobSynchronizer.cs b/src/ServiceManagement/Compute/Sync/Upload/BlobSynchronizer.cs index dc200f7deb33..a7154c44569e 100644 --- a/src/ServiceManagement/Compute/Sync/Upload/BlobSynchronizer.cs +++ b/src/ServiceManagement/Compute/Sync/Upload/BlobSynchronizer.cs @@ -56,10 +56,16 @@ public bool Synchronize() { using (dwr) { - var md5HashOfDataChunk = GetBase64EncodedMd5Hash(dwr.Data, (int) dwr.Range.Length); using (var stream = new MemoryStream(dwr.Data, 0, (int)dwr.Range.Length)) { - b.Properties.ContentMD5 = md5HashOfDataChunk; + // HTTPS provides transport level security that renders + // MD5 checking redundant + if (blob.Uri.Scheme != Uri.UriSchemeHttps) + { + var md5HashOfDataChunk = GetBase64EncodedMd5Hash(dwr.Data, (int)dwr.Range.Length); + b.Properties.ContentMD5 = md5HashOfDataChunk; + } + b.WritePages(stream, dwr.Range.StartIndex); } } @@ -78,7 +84,11 @@ public bool Synchronize() { using(var bdms = new BlobMetaDataScope(new CloudPageBlob(blob.Uri, blob.ServiceClient.Credentials))) { - bdms.Current.SetBlobMd5Hash(md5Hash); + if (this.md5Hash != null && this.md5Hash.Length != 0) + { + bdms.Current.SetBlobMd5Hash(md5Hash); + } + bdms.Current.CleanUpUploadMetaData(); bdms.Complete(); } diff --git a/src/ServiceManagement/Compute/Sync/Upload/UploadOperationMetaData.cs b/src/ServiceManagement/Compute/Sync/Upload/UploadOperationMetaData.cs index 758435d7f71b..d75a7958dc61 100644 --- a/src/ServiceManagement/Compute/Sync/Upload/UploadOperationMetaData.cs +++ b/src/ServiceManagement/Compute/Sync/Upload/UploadOperationMetaData.cs @@ -58,7 +58,7 @@ public DateTime LastModifiedDateUtc set { this.InternalLastModifiedDateUtc = value.ToString(DateTimeFormatInfo.InvariantInfo); } } - public static FileMetaData Create(string filePath) + public static FileMetaData Create(string filePath, bool skipMd5 = false) { var fileInfo = new FileInfo(filePath); if(!fileInfo.Exists) @@ -75,14 +75,14 @@ public static FileMetaData Create(string filePath) LastModifiedDateUtc = DateTime.SpecifyKind(fileInfo.LastWriteTimeUtc, DateTimeKind.Utc), Size = fileInfo.Length, VhdSize = stream.Length, - MD5Hash = CalculateMd5Hash(stream, filePath) + MD5Hash = skipMd5 ? new byte[0] : CalculateMd5Hash(stream, filePath) }; } } private static byte[] CalculateMd5Hash(Stream stream, string filePath) { - using(var md5 = MD5.Create()) + using (var md5 = MD5.Create()) { using (var swrp = new StreamWithReadProgress(stream, TimeSpan.FromSeconds(1))) {