Improve CISA tests

powershell/internal/Get-MtSkippedReason.ps1

@@ -24,7 +24,7 @@ function Get-MtSkippedReason {
         "NotLicensedMdo" { "This test is for tenants that are licensed for Defender for Office 365 Plan 2. See [Microsoft Defender for Office 365 service description](https://learn.microsoft.com/en-us/office365/servicedescriptions/office-365-advanced-threat-protection-service-description)"; break}
         "NotLicensedMdoP2" { "This test is for tenants that are licensed for Defender for Office 365 Plan 2. See [Microsoft Defender for Office 365 service description](https://learn.microsoft.com/en-us/office365/servicedescriptions/office-365-advanced-threat-protection-service-description)"; break}
         "NotLicensedMdoP1" { "This test is for tenants that are licensed for Defender for Office 365 Plan 1. See [Microsoft Defender for Office 365 service description](https://learn.microsoft.com/en-us/office365/servicedescriptions/office-365-advanced-threat-protection-service-description)"; break}
-        "AdvAudit" { "This test is for tenants that are licensed for Advanced Audit. See [Learn about auditing solutions in Microsoft Purview](https://learn.microsoft.com/en-us/purview/audit-solutions-overview#licensing-requirements)"; break}
+        "NotLicensedAdvAudit" { "This test is for tenants that are licensed for Advanced Audit. See [Learn about auditing solutions in Microsoft Purview](https://learn.microsoft.com/en-us/purview/audit-solutions-overview#licensing-requirements)"; break}
         "LicensedEntraIDPremium" { "This test is for tenants that are not licensed for any Entra ID Premium license. See [Entra ID licensing](https://learn.microsoft.com/entra/fundamentals/licensing)"; break}
         "NotSupported" { "This test relies on capabilities not currently available (e.g., cmdlets that are not available on all platforms, Resolve-DnsName)"; break}
         default { $SkippedBecause; break}

powershell/public/Add-MtTestResultDetail.ps1

@@ -66,7 +66,7 @@ function Add-MtTestResultDetail {
         [Parameter(Mandatory = $false)]
         [ValidateSet('NotConnectedAzure', 'NotConnectedExchange', 'NotConnectedGraph', 'NotDotGovDomain', 'NotLicensedEntraIDP1', 'NotConnectedSecurityCompliance', 'NotConnectedTeams',
             'NotLicensedEntraIDP2', 'NotLicensedEntraIDGovernance', 'NotLicensedEntraWorkloadID', 'NotLicensedExoDlp', "LicensedEntraIDPremium", 'NotSupported', 'Custom',
-            'NotLicensedMdo','NotLicensedMdoP2','NotLicensedMdoP1', 'AdvAudit', 'NotLicensedEop'
+            'NotLicensedMdo','NotLicensedMdoP2','NotLicensedMdoP1', 'NotLicensedAdvAudit', 'NotLicensedEop'
         )]
         # Common reasons for why the test was skipped.
         [string] $SkippedBecause,

powershell/public/cisa/entra/Test-MtCisaAppGroupOwnerConsent.ps1

@@ -23,6 +23,8 @@ function Test-MtCisaAppGroupOwnerConsent {
         return $null
     }
 
+    $SettingsApiAvailable = (Invoke-MtGraphRequest -RelativeUri 'settings' -ApiVersion beta).values.name
+
     if ( $SettingsApiAvailable -notcontains 'EnableGroupSpecificConsent' ) {
         Add-MtTestResultDetail -SkippedBecause 'Custom' -SkippedCustomReason 'Settings value is not available. This may be due to the change that this API is no longer available for recently created tenants.'
         return $null

powershell/public/cisa/entra/Test-MtCisaMethodsMigration.ps1

@@ -34,7 +34,7 @@ function Test-MtCisaMethodsMigration {
 
     $migrationState = $result.policyMigrationState
 
-    $testResult = $migrationState -eq "migrationComplete"
+    $testResult = $migrationState -eq "migrationComplete" -or $null -eq $migrationState # Can be 'null' in new tenants that never had legacy settings to migrate from.
 
     if ($testResult) {
         $testResultMarkdown = "Well done. Your tenant has completed the migration to Authentication Methods."

powershell/public/cisa/exchange/Get-MailAuthenticationRecord.ps1

@@ -28,7 +28,7 @@ function Get-MailAuthenticationRecord {
         [string]$DomainName,
 
         # DNS-server to use for lookup.
-        [ipaddress]$DnsServerIpAddress = "1.1.1.1",
+        [ipaddress]$DnsServerIpAddress,
 
         # Selector-name for the DKIM record to retrieve.
         [string]$DkimSelector = "selector1",

powershell/public/cisa/exchange/Get-MtExo.ps1

@@ -87,7 +87,7 @@ function Get-MtExo {
 
     if ($null -eq $__MtSession.ExoCache.$Request) {
         Write-Verbose "$request not in cache, requesting."
-        $response = Invoke-Expression $commands.$Request
+        $response = Invoke-Expression $commands.$Request -ErrorAction Stop
         $__MtSession.ExoCache.$Request = $response
     }
     else {

powershell/public/cisa/exchange/Resolve-SPFRecord.ps1

@@ -27,7 +27,7 @@ function Resolve-SPFRecord {
         [Parameter(Mandatory = $false,
             ValueFromPipelineByPropertyName = $true,
             Position = 2)]
-        [string]$Server = "1.1.1.1",
+        [string]$Server,
 
         # If called nested provide a referrer to build valid objects
         [Parameter(Mandatory = $false)]
@@ -69,7 +69,11 @@ function Resolve-SPFRecord {
         # Query DNS Record
         try {
             if ($isWindows -or $PSVersionTable.PSEdition -eq "Desktop") {
-                $DNSRecords = Resolve-DnsName -Server $Server -Name $Name -Type TXT
+                if ($Server) {
+                    $DNSRecords = Resolve-DnsName -Server $Server -Name $Name -Type TXT -ErrorAction Stop
+                } else {
+                    $DNSRecords = Resolve-DnsName -Name $Name -Type TXT -ErrorAction Stop
+                }
             } else {
                 $cmdletCheck = Get-Command "Resolve-Dns" -ErrorAction SilentlyContinue
                 if ($cmdletCheck) {

powershell/public/cisa/exchange/Test-MtCisaAttachmentFileType.ps1

@@ -24,9 +24,6 @@ function Test-MtCisaAttachmentFileType {
     } elseif (!(Test-MtConnection SecurityCompliance)) {
         Add-MtTestResultDetail -SkippedBecause NotConnectedSecurityCompliance
         return $null
-    } elseif ($null -eq (Get-MtLicenseInformation -Product Eop)) {
-        Add-MtTestResultDetail -SkippedBecause NotLicensedEop
-        return $null
     }
 
     $policies = Get-MtExo -Request MalwareFilterPolicy

powershell/public/cisa/exchange/Test-MtCisaAttachmentFilter.ps1

@@ -25,9 +25,6 @@ function Test-MtCisaAttachmentFilter {
     } elseif (!(Test-MtConnection SecurityCompliance)) {
         Add-MtTestResultDetail -SkippedBecause NotConnectedSecurityCompliance
         return $null
-    } elseif ($null -eq (Get-MtLicenseInformation -Product Eop)) {
-        Add-MtTestResultDetail -SkippedBecause NotLicensedEop
-        return $null
     }
 
     $policies = Get-MtExo -Request MalwareFilterPolicy

powershell/public/cisa/exchange/Test-MtCisaAuditLog.ps1

@@ -33,9 +33,9 @@ function Test-MtCisaAuditLog {
     $portalLink = "https://purview.microsoft.com/audit/auditsearch"
 
     if ($testResult) {
-        $testResultMarkdown = "Well done. Your tenant has [unified audit log enabled]($portalLink).`n`n%TestResult%"
+        $testResultMarkdown = "Well done. Your tenant has [unified audit log enabled]($portalLink)."
     } else {
-        $testResultMarkdown = "Your tenant does not have [unified audit log enabled]($portalLink).`n`n%TestResult%"
+        $testResultMarkdown = "Your tenant does not have [unified audit log enabled]($portalLink)."
     }
 
     Add-MtTestResultDetail -Result $testResultMarkdown

powershell/public/cisa/exchange/Test-MtCisaAuditLogRetention.ps1

@@ -45,27 +45,13 @@ function Test-MtCisaAuditLogRetention {
     $testResult = ($resultPolicies|Measure-Object).Count -ge 1
 
     $portalLink = "https://purview.microsoft.com/audit/auditpolicies"
-    $passResult = "✅ Pass"
-    $failResult = "❌ Fail"
 
     if ($testResult) {
-        $testResultMarkdown = "Well done. Your tenant has [Exchange Online audit retention enabled]($portalLink).`n`n%TestResult%"
+        $testResultMarkdown = "Well done. Your tenant has [Exchange Online audit retention enabled]($portalLink)."
     } else {
-        $testResultMarkdown = "Your tenant does not have [Exchange Online audit retention enabled]($portalLink).`n`n%TestResult%"
+        $testResultMarkdown = "Your tenant does not have [Exchange Online audit retention enabled]($portalLink)."
     }
 
-    $result = "| Policy Result | Policy Name | Record Types | Retention Duration |`n"
-    $result += "| --- | --- | --- | --- |`n"
-    foreach($item in $mailboxes | Sort-Object -Property Name){
-        if($item.Guid -in $resultMailboxes.Guid){
-            $result += "| $passResult | $($item.Name) | $($item.RecordTypes -join ", ") | $($item.RetentionDuration) |`n"
-        }else{
-            $result += "| $failResult | $($item.Name) | $($item.RecordTypes -join ", ") | $($item.RetentionDuration) |`n"
-        }
-    }
-
-    $testResultMarkdown = $testResultMarkdown -replace "%TestResult%", $result
-
     Add-MtTestResultDetail -Result $testResultMarkdown
 
     return $testResult

powershell/public/cisa/exchange/Test-MtCisaBlockExecutable.ps1

@@ -24,9 +24,6 @@ function Test-MtCisaBlockExecutable {
     } elseif (!(Test-MtConnection SecurityCompliance)) {
         Add-MtTestResultDetail -SkippedBecause NotConnectedSecurityCompliance
         return $null
-    } elseif ($null -eq (Get-MtLicenseInformation -Product Eop)) {
-        Add-MtTestResultDetail -SkippedBecause NotLicensedEop
-        return $null
     }
 
     $policies = Get-MtExo -Request MalwareFilterPolicy

powershell/public/cisa/exchange/Test-MtCisaDlp.ps1

@@ -29,10 +29,9 @@ function Test-MtCisaDlp {
         return $null
     }
 
-    $policies = Get-MtExo -Request DlpCompliancePolicy
+    $policies = Get-MtExo -Request DlpCompliancePolicy | Where-Object { $_.ExchangeLocation.DisplayName -contains "All" }
 
     $resultPolicies = $policies | Where-Object {`
-        $_.ExchangeLocation.DisplayName -contains "All" -and `
         $_.Workload -like "*Exchange*" -and `
         -not $_.IsSimulationPolicy -and `
         $_.Enabled
@@ -48,16 +47,18 @@ function Test-MtCisaDlp {
         $testResultMarkdown = "Your tenant does not have [Purview Data Loss Prevention Policies]($portalLink) enabled.`n`n%TestResult%"
     }
 
-    $passResult = "✅ Pass"
-    $failResult = "❌ Fail"
-    $result = "| Name | Status | Description |`n"
-    $result += "| --- | --- | --- |`n"
-    foreach ($item in ($policies | Where-Object {$_.ExchangeLocation.DisplayName -contains "All"}) | Sort-Object -Property name) {
-        $itemResult = $failResult
-        if($item.Guid -in $resultPolicies.Guid){
-            $itemResult = $passResult
+    if ($policies) {
+        $passResult = "✅ Pass"
+        $failResult = "❌ Fail"
+        $result = "| Name | Status | Description |`n"
+        $result += "| --- | --- | --- |`n"
+        foreach ($item in ($policies | Sort-Object -Property name)) {
+            $itemResult = $failResult
+            if($item.Guid -in $resultPolicies.Guid){
+                $itemResult = $passResult
+            }
+            $result += "| $($item.name) | $($itemResult) | $($item.comment) |`n"
         }
-        $result += "| $($item.name) | $($itemResult) | $($item.comment) |`n"
     }
 
     $testResultMarkdown = $testResultMarkdown -replace "%TestResult%", $result

powershell/public/cisa/exchange/Test-MtCisaDlpBaselineRule.ps1

@@ -66,8 +66,6 @@ function Test-MtCisaDlpBaselineRule {
 
     $resultComposite = $resultCcn -and $resultSsn -and $resultItin
 
-    $testResult = ($resultComposite | Measure-Object).Count -ge 1
-
     $portalLink = "https://purview.microsoft.com/datalossprevention/policies"
 
     if ($resultComposite) {
@@ -97,5 +95,5 @@ function Test-MtCisaDlpBaselineRule {
 
     Add-MtTestResultDetail -Result $testResultMarkdown
 
-    return $testResult
+    return $resultComposite
 }

powershell/public/cisa/exchange/Test-MtCisaDlpPii.ps1

@@ -61,16 +61,18 @@ function Test-MtCisaDlpPii {
         $testResultMarkdown = "Your tenant does not have [Purview Data Loss Prevention Policies]($portalLink) enabled with the Sensitive Info Type of All Full Names.`n`n%TestResult%"
     }
 
-    $passResult = "✅ Pass"
-    $failResult = "❌ Fail"
-    $result = "| Status | Policy | Rule |`n"
-    $result += "| --- | --- | --- |`n"
-    foreach ($item in ($rules | Sort-Object -Property ParentPolicyName,Name)) {
-        $itemResult = $failResult
-        if($item.Guid -in $resultRules.Guid){
-            $itemResult = $passResult
+    if ($rules) {
+        $passResult = "✅ Pass"
+        $failResult = "❌ Fail"
+        $result = "| Status | Policy | Rule |`n"
+        $result += "| --- | --- | --- |`n"
+        foreach ($item in ($rules | Sort-Object -Property ParentPolicyName,Name)) {
+            $itemResult = $failResult
+            if($item.Guid -in $resultRules.Guid){
+                $itemResult = $passResult
+            }
+            $result += "| $($itemResult) | $($item.ParentPolicyName) | $($item.Name) |`n"
         }
-        $result += "| $($itemResult) | $($item.ParentPolicyName) | $($item.Name) |`n"
     }
 
     $testResultMarkdown = $testResultMarkdown -replace "%TestResult%", $result

powershell/public/cisa/exchange/Test-MtCisaEmailFilterAlternative.ps1

@@ -8,7 +8,7 @@
 .EXAMPLE
     Test-MtCisaEmailFilterAlternative
 
-    Allways returns null
+    Always returns null
 
 .LINK
     https://maester.dev/docs/commands/Test-MtCisaEmailFilterAlternative
@@ -24,9 +24,6 @@ function Test-MtCisaEmailFilterAlternative {
     } elseif (!(Test-MtConnection SecurityCompliance)) {
         Add-MtTestResultDetail -SkippedBecause NotConnectedSecurityCompliance
         return $null
-    } elseif ($null -eq (Get-MtLicenseInformation -Product Eop)) {
-        Add-MtTestResultDetail -SkippedBecause NotLicensedEop
-        return $null
     } else {
         Add-MtTestResultDetail -SkippedBecause Custom -SkippedCustomReason "Only testing of MDO is supported"
         return $null

powershell/public/cisa/exchange/Test-MtCisaExoAlert.ps1

@@ -24,55 +24,53 @@ function Test-MtCisaExoAlert {
     }elseif(!(Test-MtConnection SecurityCompliance)){
         Add-MtTestResultDetail -SkippedBecause NotConnectedSecurityCompliance
         return $null
-    }elseif($null -eq (Get-MtLicenseInformation -Product Mdo)){
-        Add-MtTestResultDetail -SkippedBecause NotLicensedMdo
+    } elseif ("P1" -notin (Get-MtLicenseInformation -Product MdoV2)) {
+        Add-MtTestResultDetail -SkippedBecause NotLicensedMdoP1
         return $null
     }
 
     $alerts = Get-MtExo -Request ProtectionAlert
 
-    $cisaAlerts = @(
-        "be215649-fba8-4339-9ddd-05991a43b948", #Suspicious email sending patterns detected
-        "8bb9c6c8-dc12-40e1-5bb8-08da05b13393", #Suspicious connector activity
-        "bfd48f06-0865-41a6-85ff-adb746423ebf", #Suspicious Email Forwarding Activity
-        "37a4e852-e711-45ca-b0f4-b076bae3adfd", #Messages have been delayed
-        "5ed2d687-9bd3-49e7-9b56-b7dc0d9af5cb", #Tenant restricted from sending unprovisioned email
-        "a7032ff5-7eee-412b-805b-d1295c7e0932", #Tenant restricted from sending email
-        "a74bb32a-541b-47fb-adfd-f8c62ce3d59b"  #A potentially malicious URL click was detected
-    )
-
-    $filterAlerts = $alerts | Where-Object { `
-        $_.ExchangeObjectId -in $cisaAlerts
+    $cisaAlerts = @{
+        'be215649-fba8-4339-9ddd-05991a43b948' = 'Suspicious email sending patterns detected'
+        '8bb9c6c8-dc12-40e1-5bb8-08da05b13393' = 'Suspicious connector activity'
+        'bfd48f06-0865-41a6-85ff-adb746423ebf' = 'Suspicious Email Forwarding Activity'
+        '37a4e852-e711-45ca-b0f4-b076bae3adfd' = 'Messages have been delayed'
+        '5ed2d687-9bd3-49e7-9b56-b7dc0d9af5cb' = 'Tenant restricted from sending unprovisioned email'
+        'a7032ff5-7eee-412b-805b-d1295c7e0932' = 'Tenant restricted from sending email'
+    }
+    if ("P2" -in (Get-MtLicenseInformation -Product MdoV2)) {
+        $cisaAlerts['a74bb32a-541b-47fb-adfd-f8c62ce3d59b'] = 'A potentially malicious URL click was detected'
     }
 
     $resultAlerts = $alerts | Where-Object { `
-        $_.ExchangeObjectId -in $cisaAlerts -and `
+        $_.ExchangeObjectId -in $cisaAlerts.Keys -and `
         $_.NotificationEnabled
     }
 
-    $testResult = (($resultAlerts|Measure-Object).Count -eq ($cisaAlerts|Measure-Object).Count)
+    $testResult = ($resultAlerts.Count -eq $cisaAlerts.Count)
 
-    $portalLink = "https://security.microsoft.com/alertpoliciesv2"
-    $passResult = "✅ Pass"
-    $failResult = "❌ Fail"
+    $portalLink = 'https://security.microsoft.com/alertpoliciesv2'
+    $passResult = '✅ Pass'
+    $failResult = '❌ Fail'
 
     if ($testResult) {
         $testResultMarkdown = "Well done. Your tenant has [alerts configured]($portalLink).`n`n%TestResult%"
     } else {
-        $testResultMarkdown = "Your tenant does not have [alerts configured]($portalLink).`n`n%TestResult%"
+        $testResultMarkdown = "Your tenant does not have all [alerts configured]($portalLink).`n`n%TestResult%"
     }
 
     $result = "| Alert Name | Alert Result |`n"
     $result += "| --- | --- |`n"
-    foreach($item in $filterAlerts | Sort-Object -Property Identity){
-        if($item.Guid -in $resultAlerts.Guid){
-            $result += "| $($item.Identity) | $passResult |`n"
-        }else{
-            $result += "| $($item.Identity) | $failResult |`n"
+    foreach ($item in $cisaAlerts.GetEnumerator()) {
+        if ($item.Key -in $resultAlerts.Guid) {
+            $result += "| $($item.Value) | $passResult |`n"
+        } else {
+            $result += "| $($item.Value) | $failResult |`n"
         }
     }
 
-    $testResultMarkdown = $testResultMarkdown -replace "%TestResult%", $result
+    $testResultMarkdown = $testResultMarkdown -replace '%TestResult%', $result
 
     Add-MtTestResultDetail -Result $testResultMarkdown
 

powershell/public/cisa/exchange/Test-MtCisaExoAlertSiem.ps1

@@ -24,8 +24,8 @@ function Test-MtCisaExoAlertSiem {
     }elseif(!(Test-MtConnection SecurityCompliance)){
         Add-MtTestResultDetail -SkippedBecause NotConnectedSecurityCompliance
         return $null
-    }elseif($null -eq (Get-MtLicenseInformation -Product Mdo)){
-        Add-MtTestResultDetail -SkippedBecause NotLicensedMdo
+    }elseif("P1" -notin (Get-MtLicenseInformation -Product MdoV2)) {
+        Add-MtTestResultDetail -SkippedBecause NotLicensedMdoP1
         return $null
     }else{
         Add-MtTestResultDetail -SkippedBecause Custom -SkippedCustomReason "Not available for API validation."

powershell/public/cisa/exchange/Test-MtCisaExternalSenderWarning.md

@@ -14,6 +14,7 @@ This feature is only available for Outlook, Outlook for Mac, Outlook on the web,
 2. Enable the feature with the cmdlet `Set-ExternalInOutlook`
 
 ```powershell
+Install-Module -Name ExchangeOnlineManagement
 Connect-ExchangeOnline
 Set-ExternalInOutlook -Enabled $true
 ```

powershell/public/cisa/exchange/Test-MtCisaImpersonation.ps1

@@ -24,8 +24,8 @@ function Test-MtCisaImpersonation {
     } elseif (!(Test-MtConnection SecurityCompliance)) {
         Add-MtTestResultDetail -SkippedBecause NotConnectedSecurityCompliance
         return $null
-    } elseif ($null -eq (Get-MtLicenseInformation -Product Eop)) {
-        Add-MtTestResultDetail -SkippedBecause NotLicensedEop
+    } elseif ("P1" -notin (Get-MtLicenseInformation -Product MdoV2)) {
+        Add-MtTestResultDetail -SkippedBecause NotLicensedMdoP1
         return $null
     }
 

powershell/public/cisa/exchange/Test-MtCisaImpersonationTip.ps1

@@ -24,8 +24,8 @@ function Test-MtCisaImpersonationTip {
     } elseif (!(Test-MtConnection SecurityCompliance)) {
         Add-MtTestResultDetail -SkippedBecause NotConnectedSecurityCompliance
         return $null
-    } elseif ($null -eq (Get-MtLicenseInformation -Product Eop)) {
-        Add-MtTestResultDetail -SkippedBecause NotLicensedEop
+    } elseif ("P1" -notin (Get-MtLicenseInformation -Product MdoV2)) {
+        Add-MtTestResultDetail -SkippedBecause NotLicensedMdoP1
         return $null
     }
 

powershell/public/cisa/exchange/Test-MtCisaMailboxIntelligence.ps1

@@ -24,8 +24,8 @@ function Test-MtCisaMailboxIntelligence {
     } elseif (!(Test-MtConnection SecurityCompliance)) {
         Add-MtTestResultDetail -SkippedBecause NotConnectedSecurityCompliance
         return $null
-    } elseif ($null -eq (Get-MtLicenseInformation -Product Eop)) {
-        Add-MtTestResultDetail -SkippedBecause NotLicensedEop
+    } elseif ("P1" -notin (Get-MtLicenseInformation -Product MdoV2)) {
+        Add-MtTestResultDetail -SkippedBecause NotLicensedMdoP1
         return $null
     }
 

powershell/public/cisa/exchange/Test-MtCisaMalwareAction.ps1

@@ -24,9 +24,6 @@ function Test-MtCisaMalwareAction {
     } elseif (!(Test-MtConnection SecurityCompliance)) {
         Add-MtTestResultDetail -SkippedBecause NotConnectedSecurityCompliance
         return $null
-    } elseif ($null -eq (Get-MtLicenseInformation -Product Eop)) {
-        Add-MtTestResultDetail -SkippedBecause NotLicensedEop
-        return $null
     }
 
     $policies = Get-MtExo -Request MalwareFilterPolicy