Skip to content

Commit b9eca3a

Browse files
LordHepipudLordHepipud
committed
Fix background service daemon; fix memory leak
1 parent 0a57bce commit b9eca3a

9 files changed

+171
-36
lines changed

‎doc/31-Changelog.md

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,10 @@ Released closed milestones can be found on [GitHub](https://github.com/Icinga/ic
1919
* [#203](https://github.com/Icinga/icinga-powershell-framework/pull/203) Removes experimental state of the Icinga PowerShell Framework code caching and adds docs on how to use the feature
2020
* [#205](https://github.com/Icinga/icinga-powershell-framework/pull/205) Ensure Icinga for Windows configuration file is opened as read-only for every single task besides actually modifying configuration content
2121

22+
### Bugfixes
23+
24+
* [#206](https://github.com/Icinga/icinga-powershell-framework/pull/206) Fixes background service check daemon for collecting metrics over time which will no longer share data between configured checks which might cause higher CPU load and a possible memory leak
25+
2226
## 1.3.1 (2021-02-04)
2327

2428
[Issue and PRs](https://github.com/Icinga/icinga-powershell-framework/milestone/12?closed=1)

‎lib/core/cache/Get-IcingaCacheData.psm1

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -34,13 +34,13 @@ function Get-IcingaCacheData()
3434

3535
$CacheFile = Join-Path -Path (Join-Path -Path (Join-Path -Path (Get-IcingaCacheDir) -ChildPath $Space) -ChildPath $CacheStore) -ChildPath ([string]::Format('{0}.json', $KeyName));
3636
[string]$Content = '';
37-
$cacheData = @{};
37+
$cacheData = @{ };
3838

3939
if ((Test-Path $CacheFile) -eq $FALSE) {
4040
return $null;
4141
}
4242

43-
$Content = Get-Content -Path $CacheFile;
43+
$Content = Read-IcingaFileContent -File $CacheFile;
4444

4545
if ([string]::IsNullOrEmpty($Content)) {
4646
return $null;

‎lib/core/cache/Set-IcingaCacheData.psm1

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -27,15 +27,15 @@
2727

2828
function Set-IcingaCacheData()
2929
{
30-
param(
30+
param (
3131
[string]$Space,
3232
[string]$CacheStore,
3333
[string]$KeyName,
3434
$Value
3535
);
3636

3737
$CacheFile = Join-Path -Path (Join-Path -Path (Join-Path -Path (Get-IcingaCacheDir) -ChildPath $Space) -ChildPath $CacheStore) -ChildPath ([string]::Format('{0}.json', $KeyName));
38-
$cacheData = @{};
38+
$cacheData = @{ };
3939

4040
if ((Test-Path $CacheFile)) {
4141
$cacheData = Get-IcingaCacheData -Space $Space -CacheStore $CacheStore;

‎lib/core/framework/Clear-IcingaCheckSchedulerCheckData.psm1

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
<#
2+
.SYNOPSIS
3+
Clear all cached values for all check commands executed by this thread.
4+
This is mandatory as we might run into a memory leak otherwise!
5+
.DESCRIPTION
6+
Clear all cached values for all check commands executed by this thread.
7+
This is mandatory as we might run into a memory leak otherwise!
8+
.FUNCTIONALITY
9+
Clear all cached values for all check commands executed by this thread.
10+
This is mandatory as we might run into a memory leak otherwise!
11+
.OUTPUTS
12+
System.Object
13+
.LINK
14+
https://github.com/Icinga/icinga-powershell-framework
15+
#>
16+
17+
function Clear-IcingaCheckSchedulerCheckData()
18+
{
19+
if ($null -eq $global:Icinga) {
20+
return;
21+
}
22+
23+
if ($global:Icinga.ContainsKey('CheckData') -eq $FALSE) {
24+
return;
25+
}
26+
27+
$global:Icinga.CheckData.Clear();
28+
}

‎lib/core/framework/Clear-IcingaCheckSchedulerEnvironment.psm1

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
<#
2+
.SYNOPSIS
3+
Clears the entire check scheduler cache environment and frees memory as
4+
well as cleaning the stack
5+
.DESCRIPTION
6+
Clears the entire check scheduler cache environment and frees memory as
7+
well as cleaning the stack
8+
.FUNCTIONALITY
9+
Clears the entire check scheduler cache environment and frees memory as
10+
well as cleaning the stack
11+
.OUTPUTS
12+
System.Object
13+
.LINK
14+
https://github.com/Icinga/icinga-powershell-framework
15+
#>
16+
17+
function Clear-IcingaCheckSchedulerEnvironment()
18+
{
19+
if ($null -eq $global:Icinga) {
20+
return;
21+
}
22+
23+
Get-IcingaCheckSchedulerPluginOutput | Out-Null;
24+
Get-IcingaCheckSchedulerPerfData | Out-Null;
25+
Clear-IcingaCheckSchedulerCheckData;
26+
}

‎lib/core/framework/Get-IcingaCheckSchedulerCheckData.psm1

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
<#
2+
.SYNOPSIS
3+
Fetch the raw output values for a check command for each single object
4+
processed by New-IcingaCheck
5+
.DESCRIPTION
6+
Fetch the raw output values for a check command for each single object
7+
processed by New-IcingaCheck
8+
.FUNCTIONALITY
9+
Fetch the raw output values for a check command for each single object
10+
processed by New-IcingaCheck
11+
.OUTPUTS
12+
System.Object
13+
.LINK
14+
https://github.com/Icinga/icinga-powershell-framework
15+
#>
16+
17+
function Get-IcingaCheckSchedulerCheckData()
18+
{
19+
if ($null -eq $global:Icinga) {
20+
return $null;
21+
}
22+
23+
if ($global:Icinga.ContainsKey('CheckData') -eq $FALSE) {
24+
return @{ };
25+
}
26+
27+
return $global:Icinga.CheckData;
28+
}

‎lib/core/framework/New-IcingaCheckSchedulerEnvironment.psm1

Lines changed: 52 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,60 @@
1+
<#
2+
.SYNOPSIS
3+
Create a new environment in which we can store check results, performance data
4+
and values over time or executed plugins.
5+
6+
Usage:
7+
8+
Access the string plugin output by calling `Get-IcingaCheckSchedulerPluginOutput`
9+
Access possible performance data with `Get-IcingaCheckSchedulerPerfData`
10+
11+
If you execute check plugins, ensure you read both of these functions to fetch the
12+
result of the plugin call and to clear the stack and memory of the check data.
13+
14+
If you do not require the output, you can write them to Null
15+
16+
Get-IcingaCheckSchedulerPluginOutput | Out-Null;
17+
Get-IcingaCheckSchedulerPerfData | Out-Null;
18+
19+
IMPORTANT:
20+
In addition each value for each object created with `New-IcingaCheck` is stored
21+
with a timestamp for the check command inside a hashtable. If you do not require
22+
these data, you MUST call `Clear-IcingaCheckSchedulerCheckData` to free memory
23+
and clear data from the stack!
24+
25+
If you are finished with all data processing and do not require anything within
26+
memory anyway, you can safely call `Clear-IcingaCheckSchedulerEnvironment` to
27+
do the same thing in one call.
28+
.DESCRIPTION
29+
Fetch the raw output values for a check command for each single object
30+
processed by New-IcingaCheck
31+
.FUNCTIONALITY
32+
Fetch the raw output values for a check command for each single object
33+
processed by New-IcingaCheck
34+
.OUTPUTS
35+
System.Object
36+
.LINK
37+
https://github.com/Icinga/icinga-powershell-framework
38+
#>
39+
140
function New-IcingaCheckSchedulerEnvironment()
241
{
342
# Legacy code
4-
$IcingaDaemonData.IcingaThreadContent.Add('Scheduler', @{ });
43+
if ($IcingaDaemonData.IcingaThreadContent.ContainsKey('Scheduler') -eq $FALSE) {
44+
$IcingaDaemonData.IcingaThreadContent.Add('Scheduler', @{ });
45+
}
546

647
if ($null -eq $global:Icinga) {
7-
$global:Icinga = @{};
48+
$global:Icinga = @{ };
849
}
950

10-
$global:Icinga.Add('CheckResults', @());
11-
$global:Icinga.Add('PerfData', @());
51+
if ($global:Icinga.ContainsKey('CheckResults') -eq $FALSE) {
52+
$global:Icinga.Add('CheckResults', @());
53+
}
54+
if ($global:Icinga.ContainsKey('PerfData') -eq $FALSE) {
55+
$global:Icinga.Add('PerfData', @());
56+
}
57+
if ($global:Icinga.ContainsKey('CheckData') -eq $FALSE) {
58+
$global:Icinga.Add('CheckData', @{ });
59+
}
1260
}

‎lib/daemons/ServiceCheckDaemon/Start-IcingaServiceCheckDaemon.psm1

Lines changed: 22 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,6 @@ function Start-IcingaServiceCheckDaemon()
55

66
Use-Icinga -LibOnly -Daemon;
77

8-
$IcingaDaemonData.BackgroundDaemon.Add('ServiceCheckScheduler', [hashtable]::Synchronized(@{}));
98
$IcingaDaemonData.IcingaThreadPool.Add('ServiceCheckPool', (New-IcingaThreadPool -MaxInstances (Get-IcingaConfigTreeCount -Path 'BackgroundDaemon.RegisteredServices')));
109

1110
while ($TRUE) {
@@ -45,11 +44,16 @@ function Start-IcingaServiceCheckTask()
4544
Use-Icinga -LibOnly -Daemon;
4645
$PassedTime = 0;
4746
$SortedResult = $null;
48-
$OldData = @{};
49-
$PerfCache = @{};
50-
$AverageCalc = @{};
47+
$OldData = @{ };
48+
$PerfCache = @{ };
49+
$AverageCalc = @{ };
5150
[int]$MaxTime = 0;
5251

52+
# Initialise some global variables we use to actually store check result data from
53+
# plugins properly. This is doable from each thread instance as this part isn't
54+
# shared between daemons
55+
New-IcingaCheckSchedulerEnvironment;
56+
5357
foreach ($index in $TimeIndexes) {
5458
# Only allow numeric index values
5559
if ((Test-Numeric $index) -eq $FALSE) {
@@ -73,22 +77,22 @@ function Start-IcingaServiceCheckTask()
7377

7478
[int]$MaxTimeInSeconds = $MaxTime * 60;
7579

76-
if (-Not ($IcingaDaemonData.BackgroundDaemon.ServiceCheckScheduler.ContainsKey($CheckCommand))) {
77-
$IcingaDaemonData.BackgroundDaemon.ServiceCheckScheduler.Add($CheckCommand, [hashtable]::Synchronized(@{}));
78-
$IcingaDaemonData.BackgroundDaemon.ServiceCheckScheduler[$CheckCommand].Add('results', [hashtable]::Synchronized(@{}));
79-
$IcingaDaemonData.BackgroundDaemon.ServiceCheckScheduler[$CheckCommand].Add('average', [hashtable]::Synchronized(@{}));
80+
if (-Not ($global:Icinga.CheckData.ContainsKey($CheckCommand))) {
81+
$global:Icinga.CheckData.Add($CheckCommand, @{ });
82+
$global:Icinga.CheckData[$CheckCommand].Add('results', @{ });
83+
$global:Icinga.CheckData[$CheckCommand].Add('average', @{ });
8084
}
8185

8286
$LoadedCacheData = Get-IcingaCacheData -Space 'sc_daemon' -CacheStore 'checkresult_store' -KeyName $CheckCommand;
8387

8488
if ($null -ne $LoadedCacheData) {
8589
foreach ($entry in $LoadedCacheData.PSObject.Properties) {
86-
$IcingaDaemonData.BackgroundDaemon.ServiceCheckScheduler[$CheckCommand]['results'].Add(
90+
$global:Icinga.CheckData[$CheckCommand]['results'].Add(
8791
$entry.name,
88-
[hashtable]::Synchronized(@{})
92+
@{ }
8993
);
9094
foreach ($item in $entry.Value.PSObject.Properties) {
91-
$IcingaDaemonData.BackgroundDaemon.ServiceCheckScheduler[$CheckCommand]['results'][$entry.name].Add(
95+
$global:Icinga.CheckData[$CheckCommand]['results'][$entry.name].Add(
9296
$item.Name,
9397
$item.Value
9498
);
@@ -106,11 +110,11 @@ function Start-IcingaServiceCheckTask()
106110

107111
$UnixTime = Get-IcingaUnixTime;
108112

109-
foreach ($result in $IcingaDaemonData.BackgroundDaemon.ServiceCheckScheduler[$CheckCommand]['results'].Keys) {
113+
foreach ($result in $global:Icinga.CheckData[$CheckCommand]['results'].Keys) {
110114
[string]$HashIndex = $result;
111-
$SortedResult = $IcingaDaemonData.BackgroundDaemon.ServiceCheckScheduler[$CheckCommand]['results'][$HashIndex].GetEnumerator() | Sort-Object name -Descending;
112-
Add-IcingaHashtableItem -Hashtable $OldData -Key $HashIndex -Value @{} | Out-Null;
113-
Add-IcingaHashtableItem -Hashtable $PerfCache -Key $HashIndex -Value @{} | Out-Null;
115+
$SortedResult = $global:Icinga.CheckData[$CheckCommand]['results'][$HashIndex].GetEnumerator() | Sort-Object name -Descending;
116+
Add-IcingaHashtableItem -Hashtable $OldData -Key $HashIndex -Value @{ } | Out-Null;
117+
Add-IcingaHashtableItem -Hashtable $PerfCache -Key $HashIndex -Value @{ } | Out-Null;
114118

115119
foreach ($timeEntry in $SortedResult) {
116120
foreach ($calc in $AverageCalc.Keys) {
@@ -133,7 +137,7 @@ function Start-IcingaServiceCheckTask()
133137
);
134138

135139
Add-IcingaHashtableItem `
136-
-Hashtable $IcingaDaemonData.BackgroundDaemon.ServiceCheckScheduler[$CheckCommand]['average'] `
140+
-Hashtable $global:Icinga.CheckData[$CheckCommand]['average'] `
137141
-Key $MetricName -Value $AverageValue -Override | Out-Null;
138142

139143
$AverageCalc[$calc].Sum = 0;
@@ -144,11 +148,11 @@ function Start-IcingaServiceCheckTask()
144148
# Flush data we no longer require in our cache to free memory
145149
foreach ($entry in $OldData.Keys) {
146150
foreach ($key in $OldData[$entry].Keys) {
147-
Remove-IcingaHashtableItem -Hashtable $IcingaDaemonData.BackgroundDaemon.ServiceCheckScheduler[$CheckCommand]['results'][$entry] -Key $key.Name
151+
Remove-IcingaHashtableItem -Hashtable $global:Icinga.CheckData[$CheckCommand]['results'][$entry] -Key $key.Name;
148152
}
149153
}
150154

151-
Set-IcingaCacheData -Space 'sc_daemon' -CacheStore 'checkresult' -KeyName $CheckCommand -Value $IcingaDaemonData.BackgroundDaemon.ServiceCheckScheduler[$CheckCommand]['average'];
155+
Set-IcingaCacheData -Space 'sc_daemon' -CacheStore 'checkresult' -KeyName $CheckCommand -Value $global:Icinga.CheckData[$CheckCommand]['average'];
152156
# Write collected metrics to disk in case we reload the daemon. We will load them back into the module after reload then
153157
Set-IcingaCacheData -Space 'sc_daemon' -CacheStore 'checkresult_store' -KeyName $CheckCommand -Value $PerfCache;
154158
} catch {

‎lib/icinga/plugin/New-IcingaCheck.psm1

Lines changed: 7 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -49,22 +49,19 @@ function New-IcingaCheck()
4949
return;
5050
}
5151

52-
if ($global:IcingaDaemonData.ContainsKey('BackgroundDaemon') -eq $FALSE) {
52+
if ($null -eq $global:Icinga -Or $global:Icinga.ContainsKey('CheckData') -eq $FALSE) {
5353
return;
5454
}
5555

56-
if ($global:IcingaDaemonData.BackgroundDaemon.ContainsKey('ServiceCheckScheduler') -eq $FALSE) {
57-
return;
58-
}
59-
60-
if ($global:IcingaDaemonData.BackgroundDaemon.ServiceCheckScheduler.ContainsKey($this.checkcommand)) {
61-
if ($global:IcingaDaemonData.BackgroundDaemon.ServiceCheckScheduler[$this.checkcommand]['results'].ContainsKey($this.name) -eq $FALSE) {
62-
$global:IcingaDaemonData.BackgroundDaemon.ServiceCheckScheduler[$this.checkcommand]['results'].Add(
56+
if ($global:Icinga.CheckData.ContainsKey($this.checkcommand)) {
57+
if ($global:Icinga.CheckData[$this.checkcommand]['results'].ContainsKey($this.name) -eq $FALSE) {
58+
$global:Icinga.CheckData[$this.checkcommand]['results'].Add(
6359
$this.name,
64-
[hashtable]::Synchronized(@{})
60+
@{ }
6561
);
6662
}
67-
$global:IcingaDaemonData.BackgroundDaemon.ServiceCheckScheduler[$this.checkcommand]['results'][$this.name].Add(
63+
64+
$global:Icinga.CheckData[$this.checkcommand]['results'][$this.name].Add(
6865
(Get-IcingaUnixTime),
6966
$this.value
7067
);

0 commit comments

Comments
 (0)