Skip to content

Commit 811f453

Browse files
yoannmoinetyoannmoinet
authored
Merge pull request #112 from DataDog/yoann/optimize-metrics-aggregation
[perf] Use Set instead of arrays for metrics aggregation
2 parents 08aef49 + fe37d18 commit 811f453

File tree

8 files changed

+114
-116
lines changed

8 files changed

+114
-116
lines changed

‎packages/plugins/telemetry/src/common/aggregator.ts

Lines changed: 55 additions & 58 deletions
Original file line numberDiff line numberDiff line change
@@ -6,10 +6,9 @@ import type { GlobalContext } from '@dd/core/types';
66
import type { Metric, MetricToSend, OptionsDD, Report } from '@dd/telemetry-plugin/types';
77

88
import { getMetric } from './helpers';
9-
import { getPlugins, getLoaders } from './metrics/common';
9+
import { addPluginMetrics, addLoaderMetrics } from './metrics/common';
1010

11-
const getUniversalMetrics = (globalContext: GlobalContext) => {
12-
const metrics: Metric[] = [];
11+
const addUniversalMetrics = (globalContext: GlobalContext, metrics: Set<Metric>) => {
1312
const inputs = globalContext.build.inputs || [];
1413
const outputs = globalContext.build.outputs || [];
1514
const entries = globalContext.build.entries || [];
@@ -48,41 +47,40 @@ const getUniversalMetrics = (globalContext: GlobalContext) => {
4847
}
4948

5049
// Counts
51-
metrics.push(
52-
{
50+
metrics
51+
.add({
5352
metric: 'assets.count',
5453
type: 'count',
5554
value: outputs.length,
5655
tags: [],
57-
},
58-
{
56+
})
57+
.add({
5958
metric: 'entries.count',
6059
type: 'count',
6160
value: entries.length,
6261
tags: [],
63-
},
64-
{
62+
})
63+
.add({
6564
metric: 'errors.count',
6665
type: 'count',
6766
value: nbErrors,
6867
tags: [],
69-
},
70-
{
68+
})
69+
.add({
7170
metric: 'modules.count',
7271
type: 'count',
7372
value: inputs.length,
7473
tags: [],
75-
},
76-
{
74+
})
75+
.add({
7776
metric: 'warnings.count',
7877
type: 'count',
7978
value: nbWarnings,
8079
tags: [],
81-
},
82-
);
80+
});
8381

8482
if (duration) {
85-
metrics.push({
83+
metrics.add({
8684
metric: 'compilation.duration',
8785
type: 'duration',
8886
value: duration,
@@ -106,26 +104,25 @@ const getUniversalMetrics = (globalContext: GlobalContext) => {
106104
...assetsPerInput.get(input.filepath)!.map((assetName) => `assetName:${assetName}`),
107105
);
108106
}
109-
metrics.push(
110-
{
107+
metrics
108+
.add({
111109
metric: 'modules.size',
112110
type: 'size',
113111
value: input.size,
114112
tags,
115-
},
116-
{
113+
})
114+
.add({
117115
metric: 'modules.dependencies',
118116
type: 'count',
119117
value: input.dependencies.size,
120118
tags,
121-
},
122-
{
119+
})
120+
.add({
123121
metric: 'modules.dependents',
124122
type: 'count',
125123
value: input.dependents.size,
126124
tags,
127-
},
128-
);
125+
});
129126
}
130127

131128
// Assets
@@ -139,87 +136,87 @@ const getUniversalMetrics = (globalContext: GlobalContext) => {
139136
.map((entryName) => `entryName:${entryName}`),
140137
);
141138
}
142-
metrics.push(
143-
{
139+
metrics
140+
.add({
144141
metric: 'assets.size',
145142
type: 'size',
146143
value: output.size,
147144
tags,
148-
},
149-
{
145+
})
146+
.add({
150147
metric: 'assets.modules.count',
151148
type: 'count',
152149
value: output.inputs.length,
153150
tags,
154-
},
155-
);
151+
});
156152
}
157153

158154
// Entries
159155
for (const entry of entries) {
160156
const tags = [`entryName:${entry.name}`];
161-
metrics.push(
162-
{
157+
metrics
158+
.add({
163159
metric: 'entries.size',
164160
type: 'size',
165161
value: entry.size,
166162
tags,
167-
},
168-
{
163+
})
164+
.add({
169165
metric: 'entries.modules.count',
170166
type: 'count',
171167
value: entry.inputs.length,
172168
tags,
173-
},
174-
{
169+
})
170+
.add({
175171
metric: 'entries.assets.count',
176172
type: 'count',
177173
value: entry.outputs.length,
178174
tags,
179-
},
180-
);
175+
});
181176
}
182177

183178
return metrics;
184179
};
185180

186-
export const getMetrics = (
181+
export const addMetrics = (
187182
globalContext: GlobalContext,
188183
optionsDD: OptionsDD,
184+
metricsToSend: Set<MetricToSend>,
189185
report?: Report,
190-
): MetricToSend[] => {
191-
const metrics: Metric[] = [];
186+
): void => {
187+
const metrics: Set<Metric> = new Set();
192188

193189
if (report) {
194190
const { timings } = report;
195191

196192
if (timings) {
197193
if (timings.tapables) {
198-
metrics.push(...getPlugins(timings.tapables));
194+
addPluginMetrics(timings.tapables, metrics);
199195
}
200196
if (timings.loaders) {
201-
metrics.push(...getLoaders(timings.loaders));
197+
addLoaderMetrics(timings.loaders, metrics);
202198
}
203199
}
204200
}
205201

206-
metrics.push(...getUniversalMetrics(globalContext));
202+
addUniversalMetrics(globalContext, metrics);
207203

208204
// Format metrics to be DD ready and apply filters
209-
const metricsToSend: MetricToSend[] = metrics
210-
.map((m) => {
211-
let metric: Metric | null = m;
212-
if (optionsDD.filters?.length) {
213-
for (const filter of optionsDD.filters) {
214-
// Could have been filtered out by an early filter.
215-
if (metric) {
216-
metric = filter(metric);
217-
}
205+
for (const metric of metrics) {
206+
if (optionsDD.filters?.length) {
207+
let filteredMetric: Metric | null = metric;
208+
for (const filter of optionsDD.filters) {
209+
// If it's already been filtered out, no need to keep going.
210+
if (!filteredMetric) {
211+
break;
218212
}
213+
filteredMetric = filter(metric);
219214
}
220-
return metric ? getMetric(metric, optionsDD) : null;
221-
})
222-
.filter((m) => m !== null) as MetricToSend[];
223-
224-
return metricsToSend;
215+
if (filteredMetric) {
216+
metricsToSend.add(getMetric(filteredMetric, optionsDD));
217+
}
218+
} else {
219+
metricsToSend.add(getMetric(metric, optionsDD));
220+
}
221+
}
225222
};

‎packages/plugins/telemetry/src/common/metrics/common.ts

Lines changed: 19 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -4,10 +4,8 @@
44

55
import type { TimingsMap, Metric } from '@dd/telemetry-plugin/types';
66

7-
export const getPlugins = (plugins: TimingsMap): Metric[] => {
8-
const metrics: Metric[] = [];
9-
10-
metrics.push({
7+
export const addPluginMetrics = (plugins: TimingsMap, metrics: Set<Metric>): void => {
8+
metrics.add({
119
metric: 'plugins.count',
1210
type: 'count',
1311
value: plugins.size,
@@ -26,67 +24,58 @@ export const getPlugins = (plugins: TimingsMap): Metric[] => {
2624
hookDuration += duration;
2725
pluginDuration += duration;
2826
}
29-
metrics.push(
30-
{
27+
metrics
28+
.add({
3129
metric: 'plugins.hooks.duration',
3230
type: 'duration',
3331
value: hookDuration,
3432
tags: [`pluginName:${plugin.name}`, `hookName:${hook.name}`],
35-
},
36-
{
33+
})
34+
.add({
3735
metric: 'plugins.hooks.increment',
3836
type: 'count',
3937
value: hook.values.length,
4038
tags: [`pluginName:${plugin.name}`, `hookName:${hook.name}`],
41-
},
42-
);
39+
});
4340
}
4441

45-
metrics.push(
46-
{
42+
metrics
43+
.add({
4744
metric: 'plugins.duration',
4845
type: 'duration',
4946
value: pluginDuration,
5047
tags: [`pluginName:${plugin.name}`],
51-
},
52-
{
48+
})
49+
.add({
5350
metric: 'plugins.increment',
5451
type: 'count',
5552
value: pluginCount,
5653
tags: [`pluginName:${plugin.name}`],
57-
},
58-
);
54+
});
5955
}
60-
61-
return metrics;
6256
};
6357

64-
export const getLoaders = (loaders: TimingsMap): Metric[] => {
65-
const metrics: Metric[] = [];
66-
67-
metrics.push({
58+
export const addLoaderMetrics = (loaders: TimingsMap, metrics: Set<Metric>): void => {
59+
metrics.add({
6860
metric: 'loaders.count',
6961
type: 'count',
7062
value: loaders.size,
7163
tags: [],
7264
});
7365

7466
for (const loader of loaders.values()) {
75-
metrics.push(
76-
{
67+
metrics
68+
.add({
7769
metric: 'loaders.duration',
7870
type: 'duration',
7971
value: loader.duration,
8072
tags: [`loaderName:${loader.name}`],
81-
},
82-
{
73+
})
74+
.add({
8375
metric: 'loaders.increment',
8476
type: 'count',
8577
value: loader.increment,
8678
tags: [`loaderName:${loader.name}`],
87-
},
88-
);
79+
});
8980
}
90-
91-
return metrics;
9281
};

‎packages/plugins/telemetry/src/common/output/files.ts

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -13,15 +13,15 @@ type FilesToWrite = {
1313
[key in Files]?: { content: any };
1414
};
1515

16-
export const outputFiles = async (
16+
export const outputFiles: (
1717
data: {
1818
report?: Report;
19-
metrics: MetricToSend[];
19+
metrics: Set<MetricToSend>;
2020
},
2121
outputOptions: OutputOptions,
2222
log: Logger,
2323
cwd: string,
24-
) => {
24+
) => Promise<void> = async (data, outputOptions, log, cwd) => {
2525
// Don't write any file if it's not enabled.
2626
if (typeof outputOptions !== 'string' && typeof outputOptions !== 'object' && !outputOptions) {
2727
return;
@@ -66,8 +66,8 @@ export const outputFiles = async (
6666
};
6767
}
6868

69-
if (metrics && files.metrics) {
70-
filesToWrite.metrics = { content: metrics };
69+
if (files.metrics) {
70+
filesToWrite.metrics = { content: Array.from(metrics) };
7171
}
7272

7373
const proms = Object.entries(filesToWrite).map(async ([filename, file]): Promise<void> => {

‎packages/plugins/telemetry/src/common/sender.ts

Lines changed: 14 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ import type { Logger } from '@dd/core/types';
77
import type { MetricToSend } from '@dd/telemetry-plugin/types';
88

99
export const sendMetrics = (
10-
metrics: MetricToSend[] | undefined,
10+
metrics: Set<MetricToSend>,
1111
auth: { apiKey?: string; endPoint: string },
1212
log: Logger,
1313
) => {
@@ -16,17 +16,25 @@ export const sendMetrics = (
1616
log.warn(`Won't send metrics to Datadog: missing API Key.`);
1717
return;
1818
}
19-
if (!metrics || metrics.length === 0) {
19+
if (!metrics.size) {
2020
log.warn(`No metrics to send.`);
2121
return;
2222
}
2323

24-
const metricsNames = [...new Set(metrics.map((m) => m.metric))]
25-
.sort()
26-
.map((name) => `${name} - ${metrics.filter((m) => m.metric === name).length}`);
24+
const metricIterations: Map<string, number> = new Map();
25+
for (const metric of metrics) {
26+
if (!metricIterations.has(metric.metric)) {
27+
metricIterations.set(metric.metric, 0);
28+
}
29+
metricIterations.set(metric.metric, metricIterations.get(metric.metric)! + 1);
30+
}
31+
32+
const metricsNames = Array.from(metricIterations.entries()).map(
33+
([name, count]) => `${name} - ${count}`,
34+
);
2735

2836
log.debug(`
29-
Sending ${metrics.length} metrics.
37+
Sending ${metrics.size} metrics.
3038
Metrics:
3139
- ${metricsNames.join('\n - ')}`);
3240

0 commit comments

Comments
 (0)