Stream Support Improvements

config/ModulesMapping.jsonc

@@ -13,7 +13,7 @@
     "Files": "^drives\\.|^shares\\.|^users.drive$|^groups.drive$",
     "Financials": "^financials\\.",
     "Groups": "^groups.group$|^groups.directoryObject$|^groups.conversation$|^groups.endpoint$|^groups.extension$|^groups.resourceSpecificPermissionGrant$|^groups.profilePhoto$|^groups.conversationThread$|^groupLifecyclePolicies\\.|^users.group$|^groups.directorySetting$|^groups.Actions$|^groups.Functions$",
-    "Identity.DirectoryManagement": "^administrativeUnits\\.|^contacts\\.|^devices\\.|^domains\\.|^directoryRoles\\.|^directoryRoleTemplates\\.|^directorySettingTemplates\\.|^settings\\.|^subscribedSkus\\.|^contracts\\.|^directory\\.|^users.scopedRoleMembership$|^organization.organization$|^organization.organizationalBranding$|^organization.organizationSettings $|^organization.Actions$|^organization.extension$",
+    "Identity.DirectoryManagement": "^administrativeUnits\\.|^contacts\\.|^devices\\.|^domains\\.|^directoryRoles\\.|^directoryRoleTemplates\\.|^directorySettingTemplates\\.|^settings\\.|^subscribedSkus\\.|^contracts\\.|^directory\\.|^users.scopedRoleMembership$|^organization.organization$|^organization.organizationalBranding$|^organization.organizationSettings$|^organization.Actions$|^organization.extension$",
     "Identity.Governance": "^accessReviews\\.|^businessFlowTemplates\\.|^programs\\.|^programControls\\.|^programControlTypes\\.|^privilegedRoles\\.|^privilegedRoleAssignments\\.|^privilegedRoleAssignmentRequests\\.|^privilegedApproval\\.|^privilegedOperationEvents\\.|^privilegedAccess\\.|^agreements\\.|^users.agreementAcceptance$|^identityGovernance.entitlementManagement$|^identityGovernance.Functions$|^identityGovernance.Actions$",
     "Identity.SignIns": "^organization.certificateBasedAuthConfiguration$|^invitations\\.|^identityProviders\\.|^oauth2PermissionGrants\\.|^riskDetections\\.|^riskyUsers\\.|^dataPolicyOperations\\.|^identity.identityUserFlow$|^trustFramework\\.|^informationProtection\\.|^policies\\.|^users.authentication$|^users.informationProtection$|^identity.conditionalAccessRoot$",
     "Mail": "^users.inferenceClassification$|^users.mailFolder$|^users.message$",
@@ -26,7 +26,7 @@
     "Search": "^search\\.|^external\\.",
     "Security": "^Security\\.",
     "Sites": "^sites.site$|^sites.itemAnalytics$|^sites.columnDefinition$|^sites.contentType$|^sites.drive$|^sites.list$|^sites.sitePage$|^users.site$|^groups.site$|^sites.Functions$|^sites.Actions$",
-    "Teams": "^teams\\.|^chats\\.|^users.chat$|^appCatalogs$|^users.userTeamwork$|^teamwork\\.|^users.team$|^users.userTeamwork$|^groups.team$",
+    "Teams": "^teams\\.|^chats\\.|^users.chat$|^appCatalogs.teamsApp$|^users.userTeamwork$|^teamwork\\.|^users.team$|^groups.team$",
     "Users": "^users.user$|^users.directoryObject$|^users.licenseDetails$|^users.notification$|^users.outlookUser$|^users.profilePhoto$|^users.userSettings$|^users.extension$|^users.oAuth2PermissionGrant$|^users.todo$",
     "Users.Actions": "^users.Actions$",
     "Users.Functions": "^users.Functions$"

src/Applications/Applications/readme.md

@@ -38,6 +38,9 @@ subject-prefix: ''
 
 ``` yaml
 directive:
+# Remove invalid paths.
+  - remove-path-by-operation: onPremisesPublishingProfiles\.(connectors\.memberOf_.*|connectors_GetMemberOf|connectorGroups\.members_.*|connectorGroups_(Get|Create|Update|Delete)Members)
+
 # Remove cmdlets
   - where:
       verb: Test
@@ -69,6 +72,11 @@ directive:
       subject: ^OnPremis(PublishingProfile.*)$
     set:
       subject: OnPremise$1
+# Fix cmdlet name
+  - where:
+      subject: (^OnPremisePublishingProfileConnectorMember$)
+    set:
+      subject: $1Of
 ```
 ### Versioning
 

src/Authentication/Authentication/Common/DiskDataStore.cs

@@ -11,7 +11,7 @@ namespace Microsoft.Graph.PowerShell.Authentication.Common
     /// <summary>
     /// Disk data store based on System.IO APIs.
     /// </summary>
-    internal class DiskDataStore : IDataStore
+    public class DiskDataStore : IDataStore
     {
         /// <summary>
         /// Writes the given contents to the specified file.

src/Authentication/Authentication/Common/ProtectedFileProvider.cs

@@ -22,7 +22,7 @@ public enum FileProtection
     /// The file can be accessed in ReadOnly or ReadWrite mode.
     /// This class MUST be disposed by the caller.
     /// </summary>
-    internal abstract class ProtectedFileProvider : IFileProvider, IDisposable
+    public abstract class ProtectedFileProvider : IFileProvider, IDisposable
     {
         protected Stream _stream;
         public const int MaxTries = 30;

src/Files/Files/readme.md

@@ -38,7 +38,7 @@ subject-prefix: ''
 
 ``` yaml
 directive:
-  - remove-path-by-operation: .*_(Create|Get|Update|Set|Delete)Activities$|.*\.activities.*$
+  - remove-path-by-operation: .*_(Create|Get|Update|Set|Delete)Activities$|.*\.activities.*$|shares\..*_createLink
 ```
 ### Versioning
 

src/readme.graph.md

@@ -6,7 +6,7 @@
 azure: false
 powershell: true
 version: latest
-use: "@autorest/powershell@latest"
+use: "@autorest/powershell@2.1.401"
 metadata:
     authors: Microsoft Corporation
     owners: Microsoft Corporation
@@ -555,6 +555,49 @@ directive:
         return $;
       }
 
+# Modify generated .cs file download cmdlets.
+  - from: source-file-csharp
+    where: $
+    transform: >
+      if (!$documentPath.match(/generated%2Fcmdlets%2FGet\w*\d*.cs/gm))
+      {
+        return $;
+      } else {
+        let outFileParameterRegex = /(^\s*)public\s*global::System\.String\s*OutFile\s*/gmi
+        let streamResponseRegex = /global::System\.Threading\.Tasks\.Task<global::System\.IO\.Stream>\s*response/gmi
+        if($.match(outFileParameterRegex) && $.match(streamResponseRegex)) {
+          // Handle file download.
+          let overrideOnOkCallRegex = /(^\s*)(overrideOnOk\(\s*responseMessage\s*,\s*response\s*,\s*ref\s*_returnNow\s*\);)/gmi
+          $ = $.replace(overrideOnOkCallRegex, '$1$2\n$1using(var stream = await response){ this.WriteToFile(responseMessage, stream, this.GetProviderPath(OutFile, false), _cancellationTokenSource.Token); _returnNow = global::System.Threading.Tasks.Task<bool>.FromResult(true);}\n$1');
+        }
+        return $;
+      }
+
+# Modify generated .cs file upload cmdlets.
+  - from: source-file-csharp
+    where: $
+    transform: >
+      if (!$documentPath.match(/generated%2Fcmdlets%2FSet\w*\d*.cs/gm))
+      {
+        return $;
+      } else {
+        let streamBodyParameterRegex = /(^\s*)public\s*global::System.IO.Stream\s*BodyParameter\s*/gmi
+        if($.match(streamBodyParameterRegex)) {
+          // Replace base class with FileUploadCmdlet.
+          let psBaseClassImplementationRegex = /(\s*:\s*)(global::System.Management.Automation.PSCmdlet)/gmi
+          $ = $.replace(psBaseClassImplementationRegex, '$1Microsoft.Graph.PowerShell.Cmdlets.Custom.FileUploadCmdlet');
+
+          // Set bodyParameter to required to false.
+          let streamBodyParameterAnnotation = /(global::System\.IO\.Stream _bodyParameter;\s*\[global::System\.Management\.Automation\.Parameter\(Mandatory\s*=\s*)(true)/gmi
+          $ = $.replace(streamBodyParameterAnnotation, '$1false');
+
+          // Handle file upload.
+          let processRecordCallRegex = /(^\s*)(asyncCommandRuntime\.Wait\(\s*ProcessRecordAsync\s*\(\))/gmi
+          $ = $.replace(processRecordCallRegex, '$1if (!MyInvocation.BoundParameters.ContainsKey(nameof(BodyParameter))){BodyParameter = GetFileAsStream() ?? BodyParameter;}\n$1$2');
+        }
+        return $;
+      }
+
 # Modify generated runtime TypeConverterExtensions class.
   - from: source-file-csharp
     where: $

tools/Custom/FileUploadCmdlet.cs

@@ -0,0 +1,44 @@
+// ------------------------------------------------------------------------------
+//  Copyright (c) Microsoft Corporation.  All Rights Reserved.  Licensed under the MIT License.  See License in the project root for license information.
+// ------------------------------------------------------------------------------
+namespace Microsoft.Graph.PowerShell.Cmdlets.Custom
+{
+    using Microsoft.Graph.PowerShell.Authentication.Common;
+    using System.IO;
+    using System.Management.Automation;
+
+    public partial class FileUploadCmdlet : PSCmdlet
+    {
+        /// <summary>Backing field for <see cref="InFile" /> property.</summary>
+        private string _inFile;
+
+        /// <summary>The path to the file to upload. This SHOULD include the file name and extension.</summary>
+        [Parameter(Mandatory = true, HelpMessage = "The path to the file to upload. This should include a path and file name. If you omit the path, the current location will be used.")]
+        [Runtime.Info(
+        Required = true,
+        ReadOnly = false,
+        Description = @"The path to the file to upload. This should include a path and file name. If you omit the path, the current location will be used.",
+        PossibleTypes = new[] { typeof(string) })]
+        [ValidateNotNullOrEmpty()]
+        [Category(ParameterCategory.Runtime)]
+        public string InFile { get => this._inFile; set => this._inFile = value; }
+
+        /// <summary>
+        /// Creates a file stream from the provided input file.
+        /// </summary>
+        /// <returns>A file stream.</returns>
+        internal Stream GetFileAsStream()
+        {
+            if (MyInvocation.BoundParameters.ContainsKey(nameof(InFile)))
+            {
+                string resolvedFilePath = this.GetProviderPath(InFile, true);
+                var fileProvider = ProtectedFileProvider.CreateFileProvider(resolvedFilePath, FileProtection.SharedRead, new DiskDataStore());
+                return fileProvider.Stream;
+            }
+            else
+            {
+                return null;
+            }
+        }
+    }
+}

tools/Custom/PSCmdletExtensions.cs

@@ -0,0 +1,121 @@
+// ------------------------------------------------------------------------------
+//  Copyright (c) Microsoft Corporation.  All Rights Reserved.  Licensed under the MIT License.  See License in the project root for license information.
+// ------------------------------------------------------------------------------
+namespace Microsoft.Graph.PowerShell
+{
+    using Microsoft.Graph.PowerShell.Authentication.Common;
+    using System;
+    using System.Collections.ObjectModel;
+    using System.IO;
+    using System.Management.Automation;
+    using System.Net.Http;
+    using System.Threading;
+    using System.Threading.Tasks;
+
+    internal static class PSCmdletExtensions
+    {
+        /// <summary>
+        /// Gets a resolved or unresolved path from PSPath.
+        /// </summary>
+        /// <param name="cmdlet">The calling <see cref="PSCmdlet"/>.</param>
+        /// <param name="filePath">The file path to get a provider path for.</param>
+        /// <param name="isResolvedPath">Determines whether get a resolved or unresolved provider path.</param>
+        /// <returns>The provider path from PSPath.</returns>
+        internal static string GetProviderPath(this PSCmdlet cmdlet, string filePath, bool isResolvedPath)
+        {
+            string providerPath = null;
+            ProviderInfo provider;
+            try
+            {
+                var paths = new Collection<string>();
+                if (isResolvedPath)
+                {
+                    paths = cmdlet.SessionState.Path.GetResolvedProviderPathFromPSPath(filePath, out provider);
+                }
+                else
+                {
+                    paths.Add(cmdlet.SessionState.Path.GetUnresolvedProviderPathFromPSPath(filePath, out provider, out _));
+                }
+
+                if (provider.Name != "FileSystem" || paths.Count == 0)
+                {
+                    cmdlet.ThrowTerminatingError(new ErrorRecord(new Exception("Invalid path."), string.Empty, ErrorCategory.InvalidArgument, filePath));
+                }
+                if (paths.Count > 1)
+                {
+                    cmdlet.ThrowTerminatingError(new ErrorRecord(new Exception("Multiple paths not allowed."), string.Empty, ErrorCategory.InvalidArgument, filePath));
+                }
+                providerPath = paths[0];
+            }
+            catch (Exception)
+            {
+                providerPath = filePath;
+            }
+
+            return providerPath;
+        }
+
+        /// <summary>
+        /// Saves a stream to a file on disk.
+        /// </summary>
+        /// <param name="cmdlet">The calling <see cref="PSCmdlet"/>.</param>
+        /// <param name="response">The HTTP response from the service.</param>
+        /// <param name="inputStream">The stream to write to file.</param>
+        /// <param name="filePath">The path to write the file to. This should include the file name and extension.</param>
+        /// <param name="cancellationToken">A cancellation token that will be used to cancel the operation by the user.</param>
+        internal static void WriteToFile(this PSCmdlet cmdlet, HttpResponseMessage response, Stream inputStream, string filePath, CancellationToken cancellationToken)
+        {
+            using (var fileProvider = ProtectedFileProvider.CreateFileProvider(filePath, FileProtection.ExclusiveWrite, new DiskDataStore()))
+            {
+                string downloadUrl = response?.RequestMessage?.RequestUri.ToString();
+                cmdlet.WriteToStream(inputStream, fileProvider.Stream, downloadUrl, cancellationToken);
+            }
+        }
+
+        /// <summary>
+        /// Writes an input stream to an output stream.
+        /// </summary>
+        /// <param name="cmdlet">The calling <see cref="PSCmdlet"/>.</param>
+        /// <param name="inputStream">The stream to write to an output stream.</param>
+        /// <param name="outputStream">The stream to write the input stream to.</param>
+        /// <param name="cancellationToken">A cancellation token that will be used to cancel the operation by the user.</param>
+        private static void WriteToStream(this PSCmdlet cmdlet, Stream inputStream, Stream outputStream, string downloadUrl, CancellationToken cancellationToken)
+        {
+            Task copyTask = inputStream.CopyToAsync(outputStream);
+            ProgressRecord record = new ProgressRecord(
+                activityId: 0,
+                activity: $"Downloading {downloadUrl ?? "file"}",
+                statusDescription: $"{outputStream.Position} of {outputStream.Length} bytes downloaded.");
+            try
+            {
+                do
+                {
+                    cmdlet.WriteProgress(GetProgress(record, outputStream));
+
+                    Task.Delay(1000, cancellationToken).Wait(cancellationToken);
+                } while (!copyTask.IsCompleted && !cancellationToken.IsCancellationRequested);
+
+                if (copyTask.IsCompleted)
+                {
+                    cmdlet.WriteProgress(GetProgress(record, outputStream));
+                }
+            }
+            catch (OperationCanceledException)
+            {
+            }
+        }
+
+        /// <summary>
+        /// Calculates and updates the progress record of the provided stream.
+        /// </summary>
+        /// <param name="currentProgressRecord">The <see cref="ProgressRecord"/> to update.</param>
+        /// <param name="stream">The stream to calculate its progress.</param>
+        /// <returns>An updated <see cref="ProgressRecord"/>.</returns>
+        private static ProgressRecord GetProgress(ProgressRecord currentProgressRecord, Stream stream)
+        {
+            currentProgressRecord.StatusDescription = $"{stream.Position} of {stream.Length} bytes downloaded.";
+            currentProgressRecord.PercentComplete = (int)Math.Round((double)(100 * stream.Position) / stream.Length);
+            return currentProgressRecord;
+        }
+    }
+}