[CI Example Analyzer] Recover severity for errors

documentation/Debugging-StaticAnalysis-Errors.md

@@ -19,10 +19,10 @@ d:\workspace\powershell\build.proj(511,5): error MSB3073: The command "d:\worksp
 ## Where to find StaticAnalysis reports
 
 The StaticAnalysis reports could show up in two different places in the CI build:
-- On the status page in Jenkins, under the Build Artifacts: the relevant files are `BreakingChangeIssues.csv`, `SignatureIssues.csv`, and/or `HelpIssues.csv`.
-- On the status page in Jenkins, click Build Artifacts then navigate to artifacts.  You will see `BreakingChangeIssues.csv`, `SignatureIssues.csv`, and/or `HelpIssues.csv`.
+- On the status page in Jenkins, under the Build Artifacts: the relevant files are `BreakingChangeIssues.csv`, `SignatureIssues.csv`, `HelpIssues.csv` and/or `ExampleIssues.csv`.
+- On the status page in Jenkins, click Build Artifacts then navigate to artifacts.  You will see `BreakingChangeIssues.csv`, `SignatureIssues.csv`, `HelpIssues.csv` and/or `ExampleIssues.csv`.
 
-Locally, the StaticAnalysis report will show up under Azure-PowerShell/artifacts. You will see `BreakingChangeIssues.csv`, `SignatureIssues.csv`, and/or `HelpIssues.csv`.  You can generate these files by running
+Locally, the StaticAnalysis report will show up under Azure-PowerShell/artifacts. You will see `BreakingChangeIssues.csv`, `SignatureIssues.csv`, `HelpIssues.csv` and/or `ExampleIssues.csv`.  You can generate these files by running
 ```
 msbuild build.proj
 ```
@@ -55,12 +55,12 @@ Signature issues occur when your cmdlets do not follow PowerShell standards.  Pl
 Most help issues that cause StaticAnalysis to fail occur when help has not been added for a particular cmdlet.  If you have not generated help for your new cmdlets, please follow the instructions [here](https://github.com/Azure/azure-powershell/blob/main/documentation/development-docs/help-generation.md). If this is not the issue, follow the steps listed under "Remediation" for each violation listed in HelpIssues.csv.
 
 ### Example Issues
-Example issues occur when your changed markdown files in the `help` folder (_e.g.,_ `src/Accounts/Accounts/help`) violate PowerShell language best practices. Please follow the suggestion displayed in "Remediation" entry for each violation listed in `ExampleIssues.csv`. If you have an issue with severity 0 or 1 that has been approved by the Azure PowerShell team, you can suppress them following these steps:
+Example issues occur when your changed markdown files in the `help` folder (_e.g.,_ `src/Accounts/Accounts/help`) violate PowerShell language best practices. Please follow the suggestion displayed in "Remediation" entry for each violation listed in `ExampleIssues.csv`. Issues with severity 0 or 1 must be addressed, while issues with severity 2 are advisory. To better standardize the writing of documents, please also check the warning issues with severity 2 in log or download the `ExampleIssues.csv` file. If you have an issue with severity 0 or 1 that has been approved by the Azure PowerShell team, you can suppress them following these steps:
 
 - Download the `ExampleIssues.csv` file from the CI pipeline artifacts
 - Open the file using a text editor (such as VS Code) and copy each of the errors you'd like to suppress
 - Paste each of these errors into the `ExampleIssues.csv` file found in their respective [module folder](../tools/StaticAnalysis/Exceptions) (_e.g.,_ if an example issue is being suppressed for Accounts, then you would paste the corresponding line(s) in the `tools/StaticAnalysis/Exceptions/Az.Accounts/ExampleIssue.csv` file) using the same text editor
 - Copy each of the errors you would like to suppress directly from the ExampleIssues.csv file output in the CI pipeline artifacts
 - Push the changes to the .csv file and ensure the errors no longer show up in the `ExampleIssues.csv` file output from the CI pipeline artifacts.
 
-To better standardize the writing of documents, please also check the warning issues with severity 2 by downloading the `ExampleIssues.csv` file.
+If you have unexpected errors, please check whether you have splitted outputs from codes. If outputs cannot be separated from codes, then please add the tag `<!-- Skip: Output cannot be splitted from code -->` to the next line of the example title and in front of the code block.

tools/StaticAnalysis/ExampleAnalyzer/AnalyzeRules/ParameterNameAndValue.psm1

@@ -99,6 +99,19 @@ function Get-RecoveredValueType{
         if ($global:AssignmentLeftAndRight.ContainsKey($CommandElementAst.Extent.Text)){
             $CommandElementAst = $global:AssignmentLeftAndRight.($CommandElementAst.Extent.Text)
         }
+        elseif ($null -ne $CommandElementAst.Left) {
+            if($CommandElementAst.Left -eq $VariableExpressionAst){
+                if($CommandElementAst.Right -eq $VariableExpressionAst){
+                    $CommandElementAst = $null
+                }
+                else{
+                    $CommandElementAst = $CommandElementAst.Right
+                }
+            }
+            else{
+                $CommandElementAst = $CommandElementAst.Left
+            }
+        }
         elseif ($null -ne $CommandElementAst.Expression) {
             if($null -ne $CommandElementAst.Member){
                 $Items += $CommandElementAst.Member
@@ -161,6 +174,9 @@ function Measure-IsTypeMatched{
         [System.Reflection.TypeInfo]$ExpectedType,
         [System.Reflection.TypeInfo]$ActualType
     )
+    if($ActualType -eq $null) {
+        return $false
+    }
     if($ActualType.IsArray) {
         $ActualType = $ActualType.GetElementType()
     }
@@ -219,6 +235,9 @@ function Get-AssignedParameterExpression {
         }
         return $null
     }
+    if($CommandElement_Copy -is [System.Management.Automation.Language.ConvertExpressionAst]){
+        $CommandElement_Copy = $CommandElement_Copy.Type
+    }
     while ($ExpectedType.IsArray) {
         $ExpectedType = $ExpectedType.GetElementType()
     }
@@ -283,7 +302,12 @@ function Get-AssignedParameterExpression {
     elseif($CommandElement_Copy -is [System.Management.Automation.Language.TypeExpressionAst] -or
     $CommandElement_Copy -is [System.Management.Automation.Language.TypeConstraintAst]){
         $ReturnType = $CommandElement_Copy.TypeName.ToString() -as [Type]
-        $ActualType = Get-RecoveredValueType $CommandElement $ReturnType
+        if($null -eq $ReturnType){
+            $ActualType = $null
+        }
+        else{
+            $ActualType = Get-RecoveredValueType $CommandElement $ReturnType
+        }
         if (!(Measure-IsTypeMatched $ExpectedType $ActualType)) {
             # Mismatched_Parameter_Value_Type
             $ExpressionToParameter = "$($CommandElement.Extent.Text)-#-$ExpectedType. Now the type is $ActualType.(Type)"
@@ -293,6 +317,7 @@ function Get-AssignedParameterExpression {
     elseif($CommandElement_Copy -is [System.Management.Automation.Language.ExpressionAst]) {
         # Value is a constant expression                   
         $ConvertedObject = $CommandElement_Copy.Extent.text -as $ExpectedType
+        # Value of Automatic Variable
         if($null -eq $ConvertedObject){
             if($null -ne (Get-Variable | Where-Object {$_.Name -eq $CommandElement_Copy.VariablePath})){
                 $value = (Get-Variable | Where-Object {$_.Name -eq $CommandElement_Copy.VariablePath}).Value
@@ -682,21 +707,21 @@ function Measure-ParameterNameAndValue {
                     $RuleName = [RuleNames]::Invalid_Parameter_Name
                     $Severity = "Error"
                     $RuleSuppressionID = "5011"
-                    $Remediation = "Check validity of the parameter $($CommandParameterPair[$i].ParameterName)."
+                    $Remediation = "Check validity of the parameter -$($CommandParameterPair[$i].ParameterName)."
                 }
                 elseif ($global:CommandParameterPair[$i].ExpressionToParameter -eq "<duplicate>") {
                     $Message = "$($CommandParameterPair[$i].ModuleCmdletExNum)-#@#$($CommandParameterPair[$i].CommandName) -$($CommandParameterPair[$i].ParameterName) appeared more than once."
                     $RuleName = [RuleNames]::Duplicate_Parameter_Name
                     $Severity = "Error"
                     $RuleSuppressionID = "5012"
-                    $Remediation = "Remove redundant parameter $($CommandParameterPair[$i].ParameterName)."
+                    $Remediation = "Remove redundant parameter -$($CommandParameterPair[$i].ParameterName)."
                 }
                 elseif ($null -eq $global:CommandParameterPair[$i].ExpressionToParameter) {
                     $Message = "$($CommandParameterPair[$i].ModuleCmdletExNum)-#@#$($CommandParameterPair[$i].CommandName) -$($CommandParameterPair[$i].ParameterName) must be assigned with a value."
                     $RuleName = [RuleNames]::Unassigned_Parameter
                     $Severity = "Error"
                     $RuleSuppressionID = "5013"
-                    $Remediation = "Assign value for the parameter $($CommandParameterPair[$i].ParameterName)."
+                    $Remediation = "Assign value for the parameter -$($CommandParameterPair[$i].ParameterName)."
                 }
                 elseif ($global:CommandParameterPair[$i].ExpressionToParameter.EndsWith(" is a null-valued parameter value.")) {
                     $Message = "$($CommandParameterPair[$i].ModuleCmdletExNum)-#@#$($CommandParameterPair[$i].CommandName) -$($CommandParameterPair[$i].ParameterName) $($CommandParameterPair[$i].ExpressionToParameter)"

tools/StaticAnalysis/ExampleAnalyzer/ExampleIssue.cs

@@ -41,6 +41,7 @@ public string FormatRecord()
                 Module, Cmdlet, Example, RuleName, ProblemId, Severity, Description, Extent, Remediation);
         }
 
+        // The code that excludes exceptions is in tools/StaticAnalysis/ExampleAnalyzer/utils.ps1 Get-NonExceptionRecord.
         public bool Match(IReportRecord other)
         {
             var result = false;
@@ -50,7 +51,6 @@ public bool Match(IReportRecord other)
                 result = (record.Module == Module)&&
                 (record.Cmdlet == Cmdlet)&&
                 (record.Example == Example)&&
-                (record.ProblemId == ProblemId)&&
                 (record.Description == Description);
             }
             return result;

tools/StaticAnalysis/ExampleAnalyzer/Measure-MarkdownOrScript.ps1

@@ -22,23 +22,16 @@ param (
     [string]$OutputFolder = "$PSScriptRoot\..\..\..\artifacts\StaticAnalysisResults\ExampleAnalysis",
     [switch]$AnalyzeScriptsInFile,
     [switch]$OutputScriptsInFile,
-    [switch]$OutputResultsByModule,
     [switch]$CleanScripts
 )
 
 . $PSScriptRoot\utils.ps1
 
-if ($PSCmdlet.ParameterSetName -eq "Markdown") {
-    $scaleTable = @()
-    $missingTable = @()
-    $deletePromptAndSeparateOutputTable = @()
-}
 $analysisResultsTable = @()
 
 # Clean caches, remove files in "output" folder
 if ($OutputScriptsInFile.IsPresent) {
     Remove-Item $OutputFolder\TempScript.ps1 -ErrorAction SilentlyContinue
-    Remove-Item $OutputFolder\*.csv -Recurse -ErrorAction SilentlyContinue
     Remove-Item $PSScriptRoot\..\..\..\artifacts\StaticAnalysisResults\ExampleIssues.csv -ErrorAction SilentlyContinue
     Remove-Item $OutputFolder -ErrorAction SilentlyContinue
 }
@@ -67,25 +60,12 @@ if ($PSCmdlet.ParameterSetName -eq "Markdown") {
             $result = Measure-SectionMissingAndOutputScript $module $cmdlet $_.FullName `
                 -OutputScriptsInFile:$OutputScriptsInFile.IsPresent `
                 -OutputFolder $OutputFolder
-            $scaleTable += $result.Scale
-            $missingTable += $result.Missing
-            $deletePromptAndSeparateOutputTable += $result.DeletePromptAndSeparateOutput
             $analysisResultsTable += $result.Errors
         }
     }
     if ($AnalyzeScriptsInFile.IsPresent) {
         $ScriptPaths = "$OutputFolder\TempScript.ps1"
     }
-    # Summarize searching results
-    if($scaleTable){
-         $scaleTable | Where-Object {$_ -ne $null} | Export-Csv "$OutputFolder\Scale.csv" -NoTypeInformation
-    }
-    if($missingTable){
-        $missingTable | Where-Object {$_ -ne $null} | Export-Csv "$OutputFolder\Missing.csv" -NoTypeInformation
-    }
-    if($deletePromptAndSeparateOutputTable){
-        $deletePromptAndSeparateOutputTable | Where-Object {$_ -ne $null} | Export-Csv "$OutputFolder\DeletingSeparating.csv" -NoTypeInformation
-    }
 }
 
 # Analyze scripts

tools/StaticAnalysis/ExampleAnalyzer/utils.ps1

@@ -149,7 +149,7 @@ function Get-ExamplesDetailsFromMd {
 }
 <#
     .SYNOPSIS
-    Except the suppressed records
+    Except the suppressed records. It is independent of ExampleIssues.cs.
 #>
 function Get-NonExceptionRecord{
     param(
@@ -190,7 +190,7 @@ function Measure-SectionMissingAndOutputScript {
         [string]$OutputFolder
     )
     $results = @()
-    $missingSeverity = 2
+    $missingSeverity = 1
 
     $fileContent = Get-Content $MarkdownPath -Raw
 
@@ -207,7 +207,6 @@ function Measure-SectionMissingAndOutputScript {
     $missingExampleOutput = 0
     $missingExampleDescription = 0
     $needDeleting = 0
-    $needSplitting = 0
 
     # If Synopsis section exists
     if ($indexOfSynopsis -ne -1) {
@@ -382,7 +381,7 @@ function Measure-SectionMissingAndOutputScript {
             # Output example codes to "TempScript.ps1"
             if ($OutputScriptsInFile.IsPresent) {
                 $cmdletExamplesScriptPath = "$OutputFolder\TempScript.ps1"
-                if($exampleCodes -ne $null){
+                if($null -ne $exampleCodes -and $exampleCodes -ne ""){
                     $exampleCodes = $exampleCodes.Trim()
                     $functionHead = "function $Module-$Cmdlet-$exampleNumber{"
                     Add-Content -Path (Get-Item $cmdletExamplesScriptPath).FullName -Value $functionHead
@@ -394,38 +393,6 @@ function Measure-SectionMissingAndOutputScript {
         }
     }
 
-    # ScaleTable
-    $examples = $examplesDetails.Count
-    $scale = [Scale]@{
-        Module = $module
-        Cmdlet = $cmdlet
-        Examples = $examples
-    }
-
-    # MissingTable
-    if ($missingSynopsis -ne 0 -or $missingDescription -ne 0 -or $missingExampleTitle -ne 0 -or $missingExampleCode -ne 0 -or $missingExampleOutput -ne 0 -or $missingExampleDescription -ne 0) {
-        $missing = [Missing]@{
-            Module = $module
-            Cmdlet = $cmdlet
-            MissingSynopsis = $missingSynopsis
-            MissingDescription = $missingDescription
-            MissingExampleTitle = $missingExampleTitle
-            MissingExampleCode = $missingExampleCode
-            MissingExampleOutput = $missingExampleOutput
-            MissingExampleDescription = $missingExampleDescription
-        }
-    }
-
-    # DeletePromptAndSeparateOutputTable
-    if ($needDeleting -ne 0 -or $needSplitting -ne 0) {
-        $deletePromptAndSeparateOutput = [DeletePromptAndSeparateOutput]@{
-            Module = $module
-            Cmdlet = $cmdlet
-            NeedDeleting = $needDeleting
-            NeedSplitting = $needSplitting
-        }
-    }
-
     # Except the suppressed records
     $results = Get-NonExceptionRecord $results
 
@@ -484,16 +451,16 @@ function Get-ScriptAnalyzerResult {
     $results = @()
     foreach($analysisResult in $analysisResults){
         if($analysisResult.Severity -eq "ParseError"){
-            $Severity = 2
+            $Severity = 1
         }
         elseif($analysisResult.Severity -eq "Error"){
-            $Severity = 2
+            $Severity = 1
         }
         elseif($analysisResult.Severity -eq "Warning"){
-            $Severity = 3
+            $Severity = 2
         }
         elseif($analysisResult.Severity -eq "Information"){
-            $Severity = 4
+            $Severity = 3
         }
         if($analysisResult.RuleSuppressionID -ge 5000 -and $analysisResult.RuleSuppressionID -le 5199){
             $result = [AnalysisOutput]@{

tools/StaticAnalysis/IssueChecker/IssueChecker.cs

@@ -79,7 +79,8 @@ public void Analyze(IEnumerable<string> scopes, IEnumerable<string> modulesToAna
                 {
                     continue;
                 }
-                if (IsSingleExceptionFileHasCriticalIssue(exceptionFilePath, recordTypeName))
+                bool outputWarning = recordTypeName.Equals(typeof(ExampleIssue).FullName);
+                if (IsSingleExceptionFileHasCriticalIssue(exceptionFilePath, recordTypeName, outputWarning))
                 {
                     hasCriticalIssue = true;
                 }
@@ -92,7 +93,7 @@ public void Analyze(IEnumerable<string> scopes, IEnumerable<string> modulesToAna
             }
         }
 
-        private bool IsSingleExceptionFileHasCriticalIssue(string exceptionFilePath, string reportRecordTypeName) 
+        private bool IsSingleExceptionFileHasCriticalIssue(string exceptionFilePath, string reportRecordTypeName, bool outputWarning) 
         {
             bool hasError = false;
             using (var reader = new StreamReader(exceptionFilePath))
@@ -114,6 +115,10 @@ private bool IsSingleExceptionFileHasCriticalIssue(string exceptionFilePath, str
                         hasError = true;
                         errorText.AppendLine(record.FormatRecord());
                     }
+                    else if (record.Severity == 2 && outputWarning)
+                    {
+                        errorText.AppendLine(record.FormatRecord());
+                    }
                 }
                 if (hasError)
                 {