Skip to content

Commit c9c380d

Browse files
fflatenfflaten
authored
Ignore excluded/skipped tests and unnecessary setup/teardown using SkipRemainingOnFailure (#2442)
* Refactor GetSkipRemainingOnFailure and ignore excluded and skipped tests * Skip blocks to avoid running BeforeAll/AfterAll * Fix tests * Skip root-level BeforeAll/AfterAll * Fix Skipped container result Skipped-property only exists on Test * Add tests for block-level skip * Cleanup * Cleanup test
1 parent d324f29 commit c9c380d

File tree

4 files changed

+230
-75
lines changed

4 files changed

+230
-75
lines changed

‎src/Pester.RSpec.ps1

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -206,7 +206,7 @@ function PostProcess-RspecTestRun ($TestRun) {
206206
## decorate
207207

208208
# here we add result
209-
$b.result = if ($b.Skipped) {
209+
$b.result = if ($b.Skip) {
210210
"Skipped"
211211
}
212212
elseif ($b.Passed) {

‎src/Pester.Utility.ps1

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -334,7 +334,7 @@ function Fold-Block {
334334
foreach ($b in $Block) {
335335
$Accumulator = & $OnBlock $Block $Accumulator
336336
foreach ($test in $Block.Tests) {
337-
$Accumulator = &$OnTest $test $Accumulator
337+
$Accumulator = & $OnTest $test $Accumulator
338338
}
339339

340340
foreach ($b in $Block.Blocks) {

‎src/functions/Get-SkipRemainingOnFailurePlugin.ps1

Lines changed: 55 additions & 73 deletions
Original file line numberDiff line numberDiff line change
@@ -1,21 +1,46 @@
1-
function New-SkippedTestMessage {
2-
[OutputType([string])]
3-
[CmdletBinding()]
4-
param (
5-
[Parameter(Mandatory)]
6-
[Pester.Test]
7-
$Test
8-
)
9-
"Skipped due to previous failure at '$($Test.ExpandedPath)' and Run.SkipRemainingOnFailure set to '$($PesterPreference.Run.SkipRemainingOnFailure.Value)'"
10-
}
11-
121
function Resolve-SkipRemainingOnFailureConfiguration {
132
$supportedValues = 'None', 'Block', 'Container', 'Run'
143
if ($PesterPreference.Run.SkipRemainingOnFailure.Value -notin $supportedValues) {
154
throw (Get-StringOptionErrorMessage -OptionPath 'Run.SkipRemainingOnFailure' -SupportedValues $supportedValues -Value $PesterPreference.Run.SkipRemainingOnFailure.Value)
165
}
176
}
187

8+
function Set-RemainingAsSkipped {
9+
param(
10+
[Parameter(Mandatory)]
11+
[Pester.Test]
12+
$FailedTest,
13+
14+
[Parameter(Mandatory)]
15+
[Pester.Block]
16+
$Block
17+
)
18+
19+
$errorRecord = [Pester.Factory]::CreateErrorRecord(
20+
'PesterTestSkipped',
21+
"Skipped due to previous failure at '$($FailedTest.ExpandedPath)' and Run.SkipRemainingOnFailure set to '$($PesterPreference.Run.SkipRemainingOnFailure.Value)'",
22+
$null,
23+
$null,
24+
$null,
25+
$false
26+
)
27+
28+
Fold-Block -Block $Block -OnTest {
29+
param ($test)
30+
if ($test.ShouldRun -and -not $test.Skip -and -not $test.Executed) {
31+
# Skipping and counting remaining unexecuted tests
32+
$Context.Configuration.SkipRemainingOnFailureCount += 1
33+
$test.Skip = $true
34+
$test.ErrorRecord.Add($errorRecord)
35+
}
36+
} -OnBlock {
37+
param($block)
38+
if ($block.ShouldRun -and -not $block.Skip -and -not $block.Executed) {
39+
# Marking remaining blocks as Skip to avoid executing BeforeAll/AfterAll
40+
$block.Skip = $true
41+
}
42+
}
43+
}
1944

2045
function Get-SkipRemainingOnFailurePlugin {
2146
# Validate configuration
@@ -29,41 +54,18 @@ function Get-SkipRemainingOnFailurePlugin {
2954
$p.Start = {
3055
param ($Context)
3156

57+
# TODO: Use $Context.GlobalPluginData.SkipRemainingOnFailure.SkippedCount when exposed in $Context
3258
$Context.Configuration.SkipRemainingOnFailureCount = 0
3359
}
3460

3561
if ($PesterPreference.Run.SkipRemainingOnFailure.Value -eq 'Block') {
3662
$p.EachTestTeardownEnd = {
3763
param($Context)
3864

39-
# If test is not marked skipped and failed
40-
# Go through block tests and child tests and mark unexecuted tests as skipped
65+
# If test was not skipped and failed
4166
if (-not $Context.Test.Skipped -and -not $Context.Test.Passed) {
42-
43-
$errorRecord = [Pester.Factory]::CreateErrorRecord(
44-
'PesterTestSkipped',
45-
(New-SkippedTestMessage -Test $Context.Test),
46-
$null,
47-
$null,
48-
$null,
49-
$false
50-
)
51-
52-
foreach ($test in $Context.Block.Tests) {
53-
if (-not $test.Executed) {
54-
$Context.Configuration.SkipRemainingOnFailureCount += 1
55-
$test.Skip = $true
56-
$test.ErrorRecord.Add($errorRecord)
57-
}
58-
}
59-
60-
foreach ($test in ($Context.Block | View-Flat)) {
61-
if (-not $test.Executed) {
62-
$Context.Configuration.SkipRemainingOnFailureCount += 1
63-
$test.Skip = $true
64-
$test.ErrorRecord.Add($errorRecord)
65-
}
66-
}
67+
# Skip all remaining tests in the block recursively
68+
Set-RemainingAsSkipped -FailedTest $Context.Test -Block $Context.Block
6769
}
6870
}
6971
}
@@ -72,57 +74,37 @@ function Get-SkipRemainingOnFailurePlugin {
7274
$p.EachTestTeardownEnd = {
7375
param($Context)
7476

75-
# If test is not marked skipped and failed
76-
# Go through every test in container from block root and marked unexecuted tests as skipped
77+
# If test was not skipped and failed
7778
if (-not $Context.Test.Skipped -and -not $Context.Test.Passed) {
78-
79-
$errorRecord = [Pester.Factory]::CreateErrorRecord(
80-
'PesterTestSkipped',
81-
(New-SkippedTestMessage -Test $Context.Test),
82-
$null,
83-
$null,
84-
$null,
85-
$false
86-
)
87-
88-
foreach ($test in ($Context.Block.Root | View-Flat)) {
89-
if (-not $test.Executed) {
90-
$Context.Configuration.SkipRemainingOnFailureCount += 1
91-
$test.Skip = $true
92-
$test.ErrorRecord.Add($errorRecord)
93-
}
94-
}
79+
# Skip all remaining tests in the container recursively
80+
Set-RemainingAsSkipped -FailedTest $Context.Test -Block $Context.Block.Root
9581
}
9682
}
9783
}
9884

9985
elseif ($PesterPreference.Run.SkipRemainingOnFailure.Value -eq 'Run') {
100-
$p.EachTestSetupStart = {
86+
$p.ContainerRunStart = {
10187
param($Context)
10288

103-
# If a test has failed at some point during the run
104-
# Skip the test before it runs
105-
# This handles skipping tests that failed from different containers in the same run
89+
# If a test failed in a previous container, skip all tests
10690
if ($Context.Configuration.SkipRemainingFailedTest) {
107-
$Context.Configuration.SkipRemainingOnFailureCount += 1
108-
$Context.Test.Skip = $true
109-
110-
$errorRecord = [Pester.Factory]::CreateErrorRecord(
111-
'PesterTestSkipped',
112-
(New-SkippedTestMessage -Test $Context.Configuration.SkipRemainingFailedTest),
113-
$null,
114-
$null,
115-
$null,
116-
$false
117-
)
118-
$Context.Test.ErrorRecord.Add($errorRecord)
91+
# Skip container root block to avoid root-level BeforeAll/AfterAll from running. Only applicable in this mode
92+
$Context.Block.Root.Skip = $true
93+
# Skip all remaining tests in current container
94+
Set-RemainingAsSkipped -FailedTest $Context.Configuration.SkipRemainingFailedTest -Block $Context.Block
11995
}
12096
}
12197

12298
$p.EachTestTeardownEnd = {
12399
param($Context)
124100

101+
# If test was not skipped but failed
125102
if (-not $Context.Test.Skipped -and -not $Context.Test.Passed) {
103+
# Skip all remaining tests in current container
104+
Set-RemainingAsSkipped -FailedTest $Context.Test -Block $Context.Block.Root
105+
106+
# Store failed test so we can skip remaining containers in ContainerRunStart-step
107+
# TODO: Use $Context.GlobalPluginData.SkipRemainingOnFailure.FailedTest when exposed in $Context
126108
$Context.Configuration.SkipRemainingFailedTest = $Context.Test
127109
}
128110
}

‎tst/Pester.RSpec.ts.ps1

Lines changed: 173 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2591,6 +2591,179 @@ i -PassThru:$PassThru {
25912591
$r.Containers[1].Blocks[0].Tests[0].ErrorRecord.FullyQualifiedErrorID | Verify-Equal 'PesterTestSkipped'
25922592
$r.Containers[1].Blocks[0].Tests[0].ErrorRecord.TargetObject.Message | Verify-Equal "Skipped due to previous failure at 'a.b' and Run.SkipRemainingOnFailure set to 'Run'"
25932593
}
2594+
2595+
foreach ($mode in 'Block', 'Container', 'Run') {
2596+
t "Ignore tests with -Skip or excluded by filter in mode '$mode'" {
2597+
$sb1 = {
2598+
Describe 'a' {
2599+
It 'Included - fails' -Tag 'Demo' {
2600+
$false | Should -BeTrue
2601+
}
2602+
It 'Excluded - ignore' {
2603+
$true | Should -BeTrue
2604+
}
2605+
Describe 'b' {
2606+
It 'Included - skip' -Tag 'Demo' {
2607+
$true | Should -BeTrue
2608+
}
2609+
It 'Included but skipped - ignore' -Tag 'Demo' -Skip {
2610+
$true | Should -BeTrue
2611+
}
2612+
}
2613+
}
2614+
Describe 'c' {
2615+
It 'Included - skip on Container and Run' -Tag 'Demo' {
2616+
$true | Should -BeTrue
2617+
}
2618+
}
2619+
}
2620+
2621+
$sb2 = {
2622+
Describe 'd' {
2623+
It 'Included - skip on Run' -Tag 'Demo' {
2624+
$true | Should -BeTrue
2625+
}
2626+
}
2627+
}
2628+
2629+
$c = [PesterConfiguration] @{
2630+
Filter = @{
2631+
Tag = 'Demo'
2632+
}
2633+
Run = @{
2634+
ScriptBlock = $sb1, $sb2
2635+
PassThru = $true
2636+
SkipRemainingOnFailure = $mode
2637+
}
2638+
Output = @{
2639+
CIFormat = 'None'
2640+
}
2641+
}
2642+
2643+
$r = Invoke-Pester -Configuration $c
2644+
2645+
$r.Tests[0].Skipped | Verify-False
2646+
$r.Tests[0].Result | Verify-Equal 'Failed'
2647+
2648+
# Should not mark excluded tests as Skipped
2649+
$r.Tests[1].Skipped | Verify-False
2650+
$r.Tests[1].Result | Verify-Equal 'NotRun'
2651+
2652+
# Should mark included test as Skipped
2653+
$r.Tests[2].Skipped | Verify-True
2654+
$r.Tests[2].Result | Verify-Equal 'Skipped'
2655+
$r.Tests[2].ErrorRecord.TargetObject.Message -match '^Skipped due to previous failure' | Verify-True
2656+
2657+
# Should not modify explicitly skipped tests
2658+
$r.Tests[3].Skipped | Verify-True
2659+
$r.Tests[3].Result | Verify-Equal 'Skipped'
2660+
$r.Tests[3].ErrorRecord | Verify-Null
2661+
2662+
switch ($mode) {
2663+
'Block' { $r.PluginConfiguration.SkipRemainingOnFailureCount | Verify-Equal 1 }
2664+
'Container' { $r.PluginConfiguration.SkipRemainingOnFailureCount | Verify-Equal 2 }
2665+
'Run' { $r.PluginConfiguration.SkipRemainingOnFailureCount | Verify-Equal 3 }
2666+
}
2667+
}
2668+
}
2669+
2670+
foreach ($mode in 'Block', 'Container', 'Run') {
2671+
t "Remaining blocks are skipped in mode '$mode'" {
2672+
$container = [ordered]@{
2673+
RootBeforeAll = 0
2674+
RootAfterAll = 0
2675+
BlockBeforeAll = 0
2676+
BlockAfterAll = 0
2677+
}
2678+
2679+
$sb1 = {
2680+
BeforeAll { $container.RootBeforeAll++ }
2681+
AfterAll { $container.RootAfterAll++ }
2682+
2683+
Describe 'd1' {
2684+
BeforeAll { $container.BlockBeforeAll++ }
2685+
AfterAll { $container.BlockAfterAll++ }
2686+
2687+
It 'Fails' { $false | Should -BeTrue }
2688+
2689+
Context 'c1' {
2690+
BeforeAll { $container.BlockBeforeAll++ }
2691+
AfterAll { $container.BlockAfterAll++ }
2692+
It 'Skipped' { $true | Should -BeTrue }
2693+
}
2694+
}
2695+
Describe 'd2' {
2696+
BeforeAll { $container.BlockBeforeAll++ }
2697+
AfterAll { $container.BlockAfterAll++ }
2698+
2699+
It 'Skipped' { $true | Should -BeTrue }
2700+
}
2701+
}
2702+
2703+
$sb2 = {
2704+
BeforeAll { $container.RootBeforeAll++ }
2705+
AfterAll { $container.RootAfterAll++ }
2706+
2707+
Describe 'd1' {
2708+
BeforeAll { $container.BlockBeforeAll++ }
2709+
AfterAll { $container.BlockAfterAll++ }
2710+
2711+
It 'Skipped' { $true | Should -BeTrue }
2712+
}
2713+
}
2714+
2715+
$c = [PesterConfiguration] @{
2716+
Run = @{
2717+
ScriptBlock = $sb1, $sb2
2718+
PassThru = $true
2719+
SkipRemainingOnFailure = $mode
2720+
}
2721+
Output = @{
2722+
CIFormat = 'None'
2723+
}
2724+
}
2725+
2726+
$r = Invoke-Pester -Configuration $c
2727+
$r.Containers[0].Result | Verify-Equal 'Failed'
2728+
$r.Containers[0].Blocks[0].Result | Verify-Equal 'Failed'
2729+
$r.Containers[0].Blocks[0].Blocks[0].Result | Verify-Equal 'Skipped'
2730+
2731+
# AfterAll should always execute for current and parent blocks of the failure
2732+
# BeforeAll and AfterAll should not be executed for remaining children or siblings
2733+
switch ($mode) {
2734+
'Block' {
2735+
$r.Containers[0].Blocks[1].Result | Verify-Equal 'Passed'
2736+
$r.Containers[1].Result | Verify-Equal 'Passed'
2737+
$r.Containers[1].Blocks[0].Result | Verify-Equal 'Passed'
2738+
2739+
$container.RootBeforeAll | Verify-Equal 2
2740+
$container.RootAfterAll | Verify-Equal 2
2741+
$container.BlockBeforeAll | Verify-Equal 3
2742+
$container.BlockAfterAll | Verify-Equal 3
2743+
}
2744+
'Container' {
2745+
$r.Containers[0].Blocks[1].Result | Verify-Equal 'Skipped'
2746+
$r.Containers[1].Result | Verify-Equal 'Passed'
2747+
$r.Containers[1].Blocks[0].Result | Verify-Equal 'Passed'
2748+
2749+
$container.RootBeforeAll | Verify-Equal 2
2750+
$container.RootAfterAll | Verify-Equal 2
2751+
$container.BlockBeforeAll | Verify-Equal 2
2752+
$container.BlockAfterAll | Verify-Equal 2
2753+
}
2754+
'Run' {
2755+
$r.Containers[0].Blocks[1].Result | Verify-Equal 'Skipped'
2756+
$r.Containers[1].Result | Verify-Equal 'Skipped'
2757+
$r.Containers[1].Blocks[0].Result | Verify-Equal 'Skipped'
2758+
2759+
$container.RootBeforeAll | Verify-Equal 1
2760+
$container.RootAfterAll | Verify-Equal 1
2761+
$container.BlockBeforeAll | Verify-Equal 1
2762+
$container.BlockAfterAll | Verify-Equal 1
2763+
}
2764+
}
2765+
}
2766+
}
25942767
}
25952768

25962769
b 'Changes to CWD are reverted on exit' {

0 commit comments

Comments
 (0)