Skip to content

Commit c09df1a

Browse files
kshyjukshyju
authored
Enhancement to memory metrics reporting (#10984)
* Improved memory metrics reporting using CGroup data for Linux consumption. * Minor logging improvements. * Removed OS check as it is not needed since this publisher will be enabled only for Linux. * Log only when IsCGroupMemoryMetricsEnabled property value changes. * Update LinuxContainerLegionMetricsPublisher to use CGroup memory metrics * Reordering the location of property before method, to fix SA 1201 Stylecop warning.
1 parent 93bcd64 commit c09df1a

10 files changed

+188
-53
lines changed

‎release_notes.md

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,4 +2,5 @@
22

33
<!-- Please add your release notes in the following format:
44
- My change description (#PR)
5-
-->
5+
-->
6+
- Improved memory metrics reporting using CGroup data for Linux consumption (#10968)

‎src/WebJobs.Script.WebHost/Metrics/CgroupMemoryUsageHelper.cs

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
// Copyright (c) .NET Foundation. All rights reserved.
2+
// Licensed under the MIT License. See License.txt in the project root for license information.
3+
4+
using System;
5+
using System.IO;
6+
using Microsoft.Extensions.Logging;
7+
8+
namespace Microsoft.Azure.WebJobs.Script.WebHost.Metrics
9+
{
10+
internal static class CgroupMemoryUsageHelper
11+
{
12+
private const string CgroupPathV1 = "/sys/fs/cgroup/memory/memory.usage_in_bytes";
13+
private const string CgroupPathV2 = "/sys/fs/cgroup/memory.current";
14+
15+
/// <summary>
16+
/// Retrieves the memory usage of the control group in bytes, supporting both cgroup v1 and v2.
17+
/// </summary>
18+
/// <param name="logger">An <see cref="ILogger{TCategoryName}"/> instance for logging.</param>
19+
/// <returns>The memory usage in bytes if available; otherwise, 0.</returns>
20+
internal static long GetMemoryUsageInBytes(ILogger logger)
21+
{
22+
try
23+
{
24+
if (TryReadMemoryUsage(CgroupPathV2, out var usageInBytes) || TryReadMemoryUsage(CgroupPathV1, out usageInBytes))
25+
{
26+
return usageInBytes;
27+
}
28+
29+
logger.LogWarning("Memory usage not available from either control group v1 or v2.");
30+
return 0;
31+
}
32+
catch (Exception ex)
33+
{
34+
logger.LogError(ex, "Error reading control group resource usage.");
35+
return 0;
36+
}
37+
}
38+
39+
private static bool TryReadMemoryUsage(string path, out long memoryUsageInBytes)
40+
{
41+
memoryUsageInBytes = 0;
42+
43+
if (!File.Exists(path))
44+
{
45+
return false;
46+
}
47+
48+
return long.TryParse(File.ReadAllText(path), out memoryUsageInBytes);
49+
}
50+
}
51+
}

‎src/WebJobs.Script.WebHost/Metrics/LinuxContainerLegionMetricsPublisher.cs

Lines changed: 34 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
using System.Threading;
88
using System.Threading.Tasks;
99
using Microsoft.Azure.Functions.Platform.Metrics.LinuxConsumption;
10+
using Microsoft.Azure.WebJobs.Script.Config;
1011
using Microsoft.Azure.WebJobs.Script.Diagnostics;
1112
using Microsoft.Azure.WebJobs.Script.WebHost.Configuration;
1213
using Microsoft.Extensions.Logging;
@@ -21,29 +22,34 @@ public sealed class LinuxContainerLegionMetricsPublisher : IMetricsPublisher, ID
2122
private readonly TimeSpan _memorySnapshotInterval = TimeSpan.FromMilliseconds(1000);
2223
private readonly TimeSpan _timerStartDelay = TimeSpan.FromSeconds(2);
2324
private readonly IOptionsMonitor<StandbyOptions> _standbyOptions;
24-
private readonly IDisposable _standbyOptionsOnChangeSubscription;
25+
private readonly IDisposable _standbyOptionsOnChangeListener;
26+
private readonly IDisposable _hostingConfigOptionsOnChangeListener;
2527
private readonly IEnvironment _environment;
2628
private readonly ILogger _logger;
2729
private readonly IScriptHostManager _scriptHostManager;
2830
private readonly string _containerName;
31+
private readonly IOptionsMonitor<FunctionsHostingConfigOptions> _hostingConfigOptions;
2932

3033
private IMetricsLogger _metricsLogger;
3134
private TimeSpan _metricPublishInterval;
3235
private Process _process;
3336
private Timer _processMonitorTimer;
3437
private Timer _metricsPublisherTimer;
3538
private bool _initialized = false;
39+
private bool _isCGroupMemoryMetricsEnabled = false;
3640

3741
public LinuxContainerLegionMetricsPublisher(IEnvironment environment, IOptionsMonitor<StandbyOptions> standbyOptions,
3842
IOptions<LinuxConsumptionLegionMetricsPublisherOptions> options, ILogger<LinuxContainerLegionMetricsPublisher> logger,
3943
IFileSystem fileSystem, ILinuxConsumptionMetricsTracker metricsTracker, IScriptHostManager scriptHostManager,
44+
IOptionsMonitor<FunctionsHostingConfigOptions> functionsHostingConfigOptions,
4045
int? metricsPublishIntervalMS = null)
4146
{
4247
_standbyOptions = standbyOptions ?? throw new ArgumentNullException(nameof(standbyOptions));
4348
_environment = environment ?? throw new ArgumentNullException(nameof(environment));
4449
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
4550
_metricsTracker = metricsTracker ?? throw new ArgumentNullException(nameof(metricsTracker));
4651
_scriptHostManager = scriptHostManager ?? throw new ArgumentNullException(nameof(scriptHostManager));
52+
_hostingConfigOptions = functionsHostingConfigOptions ?? throw new ArgumentNullException(nameof(functionsHostingConfigOptions));
4753
_containerName = options.Value.ContainerName;
4854

4955
// Set this to 15 minutes worth of files
@@ -59,12 +65,14 @@ public LinuxContainerLegionMetricsPublisher(IEnvironment environment, IOptionsMo
5965

6066
if (_standbyOptions.CurrentValue.InStandbyMode)
6167
{
62-
_standbyOptionsOnChangeSubscription = _standbyOptions.OnChange(o => OnStandbyOptionsChange(o));
68+
_standbyOptionsOnChangeListener = _standbyOptions.OnChange(o => OnStandbyOptionsChange(o));
6369
}
6470
else
6571
{
6672
Start();
6773
}
74+
75+
_hostingConfigOptionsOnChangeListener = _hostingConfigOptions.OnChange(OnHostingConfigOptionsChanged);
6876
}
6977

7078
public IMetricsLogger MetricsLogger
@@ -83,6 +91,15 @@ public IMetricsLogger MetricsLogger
8391
}
8492
}
8593

94+
private void OnHostingConfigOptionsChanged(FunctionsHostingConfigOptions newOptions)
95+
{
96+
if (newOptions.IsCGroupMemoryMetricsEnabled != _isCGroupMemoryMetricsEnabled)
97+
{
98+
_logger.LogInformation("CGroup memory metrics enabled: {Enabled}", newOptions.IsCGroupMemoryMetricsEnabled);
99+
_isCGroupMemoryMetricsEnabled = newOptions.IsCGroupMemoryMetricsEnabled;
100+
}
101+
}
102+
86103
public void Initialize()
87104
{
88105
_process = Process.GetCurrentProcess();
@@ -185,11 +202,20 @@ private void OnProcessMonitorTimer(object state)
185202
{
186203
try
187204
{
188-
_process.Refresh();
189-
var commitSizeBytes = _process.WorkingSet64;
190-
if (commitSizeBytes != 0)
205+
long memoryUsageInBytes;
206+
if (_hostingConfigOptions.CurrentValue.IsCGroupMemoryMetricsEnabled)
207+
{
208+
memoryUsageInBytes = CgroupMemoryUsageHelper.GetMemoryUsageInBytes(_logger);
209+
}
210+
else
211+
{
212+
_process.Refresh();
213+
memoryUsageInBytes = _process.WorkingSet64;
214+
}
215+
216+
if (memoryUsageInBytes != 0)
191217
{
192-
AddMemoryActivity(DateTime.UtcNow, commitSizeBytes);
218+
AddMemoryActivity(DateTime.UtcNow, memoryUsageInBytes);
193219
}
194220
}
195221
catch (Exception e)
@@ -244,6 +270,8 @@ public void Dispose()
244270
_metricsPublisherTimer = null;
245271

246272
_metricsTracker.OnDiagnosticEvent -= OnMetricsDiagnosticEvent;
273+
_standbyOptionsOnChangeListener?.Dispose();
274+
_hostingConfigOptionsOnChangeListener?.Dispose();
247275
}
248276

249277
internal class Metrics

‎src/WebJobs.Script.WebHost/Metrics/LinuxContainerMetricsPublisher.cs

Lines changed: 44 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
using System;
55
using System.Collections.Concurrent;
66
using System.Diagnostics;
7+
using System.IO;
78
using System.Linq;
89
using System.Net.Http;
910
using System.Net.Http.Formatting;
@@ -19,7 +20,7 @@
1920

2021
namespace Microsoft.Azure.WebJobs.Script.WebHost.Metrics
2122
{
22-
public class LinuxContainerMetricsPublisher : IMetricsPublisher
23+
public sealed class LinuxContainerMetricsPublisher : IMetricsPublisher, IDisposable
2324
{
2425
public const string PublishMemoryActivityPath = "/memoryactivity";
2526
public const string PublishFunctionActivityPath = "/functionactivity";
@@ -36,10 +37,11 @@ public class LinuxContainerMetricsPublisher : IMetricsPublisher
3637
private readonly TimeSpan _metricPublishInterval = TimeSpan.FromMilliseconds(30 * 1000);
3738
private readonly TimeSpan _timerStartDelay = TimeSpan.FromSeconds(2);
3839
private readonly IOptionsMonitor<StandbyOptions> _standbyOptions;
39-
private readonly IDisposable _standbyOptionsOnChangeSubscription;
40+
private readonly IDisposable _standbyOptionsOnChangeListener;
41+
private readonly IDisposable _hostingConfigOptionsOnChangeListener;
4042
private readonly string _requestUri;
4143
private readonly IEnvironment _environment;
42-
private readonly IOptions<FunctionsHostingConfigOptions> _hostingConfigOptions;
44+
private readonly IOptionsMonitor<FunctionsHostingConfigOptions> _hostingConfigOptions;
4345

4446
// Buffer for all memory activities for this container.
4547
private BlockingCollection<MemoryActivity> _memoryActivities;
@@ -64,8 +66,9 @@ public class LinuxContainerMetricsPublisher : IMetricsPublisher
6466
private int _errorCount = 0;
6567
private string _stampName;
6668
private bool _initialized = false;
69+
private bool _isCGroupMemoryMetricsEnabled = false;
6770

68-
public LinuxContainerMetricsPublisher(IEnvironment environment, IOptionsMonitor<StandbyOptions> standbyOptions, ILogger<LinuxContainerMetricsPublisher> logger, HostNameProvider hostNameProvider, IOptions<FunctionsHostingConfigOptions> functionsHostingConfigOptions, HttpClient httpClient = null)
71+
public LinuxContainerMetricsPublisher(IEnvironment environment, IOptionsMonitor<StandbyOptions> standbyOptions, ILogger<LinuxContainerMetricsPublisher> logger, HostNameProvider hostNameProvider, IOptionsMonitor<FunctionsHostingConfigOptions> functionsHostingConfigOptions, HttpClient httpClient = null)
6972
{
7073
_standbyOptions = standbyOptions ?? throw new ArgumentNullException(nameof(standbyOptions));
7174
_environment = environment ?? throw new ArgumentNullException(nameof(environment));
@@ -86,12 +89,23 @@ public LinuxContainerMetricsPublisher(IEnvironment environment, IOptionsMonitor<
8689
_httpClient = (httpClient != null) ? httpClient : CreateMetricsPublisherHttpClient();
8790
if (_standbyOptions.CurrentValue.InStandbyMode)
8891
{
89-
_standbyOptionsOnChangeSubscription = _standbyOptions.OnChange(o => OnStandbyOptionsChange());
92+
_standbyOptionsOnChangeListener = _standbyOptions.OnChange(o => OnStandbyOptionsChange());
9093
}
9194
else
9295
{
9396
Start();
9497
}
98+
99+
_hostingConfigOptionsOnChangeListener = _hostingConfigOptions.OnChange(OnHostingConfigOptionsChanged);
100+
}
101+
102+
private void OnHostingConfigOptionsChanged(FunctionsHostingConfigOptions newOptions)
103+
{
104+
if (newOptions.IsCGroupMemoryMetricsEnabled != _isCGroupMemoryMetricsEnabled)
105+
{
106+
_logger.LogInformation("CGroup memory metrics enabled: {Enabled}", newOptions.IsCGroupMemoryMetricsEnabled);
107+
_isCGroupMemoryMetricsEnabled = newOptions.IsCGroupMemoryMetricsEnabled;
108+
}
95109
}
96110

97111
private void OnStandbyOptionsChange()
@@ -234,7 +248,6 @@ internal async Task SendRequest<T>(ConcurrentQueue<T> activitiesToPublish, strin
234248
try
235249
{
236250
var request = BuildRequest(HttpMethod.Post, publishPath, activitiesToPublish.ToArray());
237-
238251
HttpResponseMessage response = await _httpClient.SendAsync(request);
239252
response.EnsureSuccessStatusCode();
240253
}
@@ -265,11 +278,20 @@ private void OnProcessMonitorTimer(object state)
265278
{
266279
try
267280
{
268-
_process.Refresh();
269-
var commitSizeBytes = _process.WorkingSet64;
270-
if (commitSizeBytes != 0)
281+
long memoryUsageInBytes;
282+
if (_hostingConfigOptions.CurrentValue.IsCGroupMemoryMetricsEnabled)
283+
{
284+
memoryUsageInBytes = CgroupMemoryUsageHelper.GetMemoryUsageInBytes(_logger);
285+
}
286+
else
287+
{
288+
_process.Refresh();
289+
memoryUsageInBytes = _process.WorkingSet64;
290+
}
291+
292+
if (memoryUsageInBytes != 0)
271293
{
272-
AddMemoryActivity(DateTime.UtcNow, commitSizeBytes);
294+
AddMemoryActivity(DateTime.UtcNow, memoryUsageInBytes);
273295
}
274296
}
275297
catch (Exception e)
@@ -292,7 +314,7 @@ private HttpRequestMessage BuildRequest<TContent>(HttpMethod method, string path
292314
request.Headers.Add(HostNameHeader, _hostNameProvider.Value);
293315
request.Headers.Add(StampNameHeader, _stampName);
294316

295-
if (_hostingConfigOptions.Value.SwtIssuerEnabled)
317+
if (_hostingConfigOptions.CurrentValue.SwtIssuerEnabled)
296318
{
297319
string swtToken = SimpleWebTokenHelper.CreateToken(DateTime.UtcNow.AddMinutes(5));
298320
request.Headers.Add(ScriptConstants.SiteRestrictedTokenHeaderName, swtToken);
@@ -313,5 +335,16 @@ public void OnFunctionCompleted(string functionName, string invocationId)
313335
{
314336
// nothing to do
315337
}
338+
339+
public void Dispose()
340+
{
341+
_standbyOptionsOnChangeListener?.Dispose();
342+
_hostingConfigOptionsOnChangeListener?.Dispose();
343+
_processMonitorTimer?.Dispose();
344+
_metricsPublisherTimer?.Dispose();
345+
_httpClient?.Dispose();
346+
_memoryActivities?.Dispose();
347+
_functionActivities?.Dispose();
348+
}
316349
}
317350
}

‎src/WebJobs.Script.WebHost/WebHostServiceCollectionExtensions.cs

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -305,7 +305,8 @@ private static void AddLinuxContainerServices(this IServiceCollection services)
305305
var metricsTracker = s.GetService<ILinuxConsumptionMetricsTracker>();
306306
var standbyOptions = s.GetService<IOptionsMonitor<StandbyOptions>>();
307307
var scriptHostManager = s.GetService<IScriptHostManager>();
308-
return new LinuxContainerLegionMetricsPublisher(environment, standbyOptions, options, logger, new FileSystem(), metricsTracker, scriptHostManager);
308+
var hostingConfigOptions = s.GetService<IOptionsMonitor<FunctionsHostingConfigOptions>>();
309+
return new LinuxContainerLegionMetricsPublisher(environment, standbyOptions, options, logger, new FileSystem(), metricsTracker, scriptHostManager, hostingConfigOptions);
309310
}
310311
else if (environment.IsFlexConsumptionSku())
311312
{
@@ -320,7 +321,7 @@ private static void AddLinuxContainerServices(this IServiceCollection services)
320321
var logger = s.GetService<ILogger<LinuxContainerMetricsPublisher>>();
321322
var standbyOptions = s.GetService<IOptionsMonitor<StandbyOptions>>();
322323
var hostNameProvider = s.GetService<HostNameProvider>();
323-
var hostingConfigOptions = s.GetService<IOptions<FunctionsHostingConfigOptions>>();
324+
var hostingConfigOptions = s.GetService<IOptionsMonitor<FunctionsHostingConfigOptions>>();
324325
return new LinuxContainerMetricsPublisher(environment, standbyOptions, logger, hostNameProvider, hostingConfigOptions);
325326
}
326327

‎src/WebJobs.Script/Config/FunctionsHostingConfigOptions.cs

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -78,6 +78,22 @@ internal bool SwtIssuerEnabled
7878
}
7979
}
8080

81+
/// <summary>
82+
/// Gets or sets a value indicating whether to use cgroup memory metrics for reporting memory usage.
83+
/// </summary>
84+
internal bool IsCGroupMemoryMetricsEnabled
85+
{
86+
get
87+
{
88+
return GetFeatureAsBooleanOrDefault(ScriptConstants.FeatureFlagEnableCGroupMemoryMetrics, false);
89+
}
90+
91+
set
92+
{
93+
_features[ScriptConstants.FeatureFlagEnableCGroupMemoryMetrics] = value ? "1" : "0";
94+
}
95+
}
96+
8197
/// <summary>
8298
/// Gets or sets a string delimited by '|' that contains a list of admin APIs that are allowed to
8399
/// be invoked internally by platform components.

‎src/WebJobs.Script/ScriptConstants.cs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -142,6 +142,7 @@ public static class ScriptConstants
142142
public const string FeatureFlagDisableOrderedInvocationMessages = "DisableOrderedInvocationMessages";
143143
public const string FeatureFlagEnableAzureMonitorTimeIsoFormat = "EnableAzureMonitorTimeIsoFormat";
144144
public const string FeatureFlagEnableTestDataSuppression = "EnableTestDataSuppression";
145+
public const string FeatureFlagEnableCGroupMemoryMetrics = "EnableCGroupMemoryMetrics";
145146
public const string HostingConfigDisableLinuxAppServiceDetailedExecutionEvents = "DisableLinuxExecutionDetails";
146147
public const string HostingConfigDisableLinuxAppServiceExecutionEventLogBackoff = "DisableLinuxLogBackoff";
147148
public const string FeatureFlagEnableLegacyDurableVersionCheck = "EnableLegacyDurableVersionCheck";

0 commit comments

Comments
 (0)