Skip to content

Commit 2f888f4

Browse files
fedetorres93fedetorres93
committed
Add unit tests for UTF-8 content negotiation
Signed-off-by: Federico Torres <[email protected]>
1 parent 5e6f0a8 commit 2f888f4

File tree

7 files changed

+458
-50
lines changed

7 files changed

+458
-50
lines changed

‎prometheus-metrics-core/src/test/java/io/prometheus/metrics/core/metrics/HistogramTest.java

Lines changed: 2 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,12 @@
11
package io.prometheus.metrics.core.metrics;
22

3+
import io.prometheus.metrics.model.snapshots.*;
34
import io.prometheus.metrics.shaded.com_google_protobuf_3_21_7.TextFormat;
45
import io.prometheus.metrics.core.datapoints.DistributionDataPoint;
56
import io.prometheus.metrics.core.exemplars.ExemplarSamplerConfigTestUtil;
67
import io.prometheus.metrics.expositionformats.OpenMetricsTextFormatWriter;
78
import io.prometheus.metrics.expositionformats.PrometheusProtobufWriter;
89
import io.prometheus.metrics.expositionformats.generated.com_google_protobuf_3_21_7.Metrics;
9-
import io.prometheus.metrics.model.snapshots.ClassicHistogramBucket;
10-
import io.prometheus.metrics.model.snapshots.Exemplar;
11-
import io.prometheus.metrics.model.snapshots.Exemplars;
12-
import io.prometheus.metrics.model.snapshots.HistogramSnapshot;
13-
import io.prometheus.metrics.model.snapshots.Labels;
14-
import io.prometheus.metrics.model.snapshots.MetricSnapshots;
1510
import io.prometheus.metrics.tracer.common.SpanContext;
1611
import io.prometheus.metrics.tracer.initializer.SpanContextSupplier;
1712
import org.junit.After;
@@ -723,7 +718,7 @@ public void testDefaults() throws IOException {
723718
// text
724719
ByteArrayOutputStream out = new ByteArrayOutputStream();
725720
OpenMetricsTextFormatWriter writer = new OpenMetricsTextFormatWriter(false, true);
726-
writer.write(out, MetricSnapshots.of(snapshot));
721+
writer.write(out, MetricSnapshots.of(snapshot), EscapingScheme.NO_ESCAPING);
727722
Assert.assertEquals(expectedTextFormat, out.toString());
728723
}
729724

‎prometheus-metrics-core/src/test/java/io/prometheus/metrics/core/metrics/InfoTest.java

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
package io.prometheus.metrics.core.metrics;
22

33
import io.prometheus.metrics.expositionformats.OpenMetricsTextFormatWriter;
4+
import io.prometheus.metrics.model.snapshots.EscapingScheme;
45
import io.prometheus.metrics.model.snapshots.Labels;
56
import io.prometheus.metrics.model.snapshots.MetricSnapshots;
67
import io.prometheus.metrics.shaded.com_google_protobuf_3_21_7.TextFormat;
@@ -98,7 +99,7 @@ public void testConstLabelsDuplicate2() {
9899
private void assertTextFormat(String expected, Info info) throws IOException {
99100
OpenMetricsTextFormatWriter writer = new OpenMetricsTextFormatWriter(true, true);
100101
ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
101-
writer.write(outputStream, MetricSnapshots.of(info.collect()));
102+
writer.write(outputStream, MetricSnapshots.of(info.collect()), EscapingScheme.NO_ESCAPING);
102103
String result = outputStream.toString(StandardCharsets.UTF_8.name());
103104
if (!result.contains(expected)) {
104105
throw new AssertionError(expected + " is not contained in the following output:\n" + result);

‎prometheus-metrics-exposition-formats/src/test/java/io/prometheus/metrics/expositionformats/ExpositionFormatsTest.java

Lines changed: 112 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,9 @@
1212

1313
import java.io.ByteArrayOutputStream;
1414
import java.io.IOException;
15+
import java.util.concurrent.atomic.AtomicInteger;
16+
17+
import static io.prometheus.metrics.model.snapshots.PrometheusNaming.nameEscapingScheme;
1518

1619
public class ExpositionFormatsTest {
1720

@@ -373,6 +376,8 @@ public void testGaugeUTF8() throws IOException {
373376
.build())
374377
.build();
375378
assertPrometheusText(prometheusText, gauge);
379+
380+
PrometheusNaming.nameValidationScheme = ValidationScheme.LEGACY_VALIDATION;
376381
}
377382

378383
@Test
@@ -1837,31 +1842,134 @@ public void testLabelValueEscape() throws IOException {
18371842
assertPrometheusText(prometheus, counter);
18381843
}
18391844

1845+
@Test
1846+
public void testFindWriter() {
1847+
EscapingScheme oldDefault = nameEscapingScheme;
1848+
nameEscapingScheme = EscapingScheme.UNDERSCORE_ESCAPING;
1849+
ExpositionFormats expositionFormats = ExpositionFormats.init();
1850+
1851+
// delimited format
1852+
String acceptHeaderValue = "application/vnd.google.protobuf;proto=io.prometheus.client.MetricFamily;encoding=delimited";
1853+
String expectedFmt = "application/vnd.google.protobuf; proto=io.prometheus.client.MetricFamily; encoding=delimited; escaping=underscores";
1854+
EscapingScheme escapingScheme = EscapingScheme.fromAcceptHeader(acceptHeaderValue);
1855+
ExpositionFormatWriter writer = expositionFormats.findWriter(acceptHeaderValue);
1856+
Assert.assertEquals(expectedFmt, writer.getContentType() + escapingScheme.toHeaderFormat());
1857+
1858+
// plain text format
1859+
acceptHeaderValue = "text/plain;version=0.0.4";
1860+
expectedFmt = "text/plain; version=0.0.4; charset=utf-8; escaping=underscores";
1861+
escapingScheme = EscapingScheme.fromAcceptHeader(acceptHeaderValue);
1862+
writer = expositionFormats.findWriter(acceptHeaderValue);
1863+
Assert.assertEquals(expectedFmt, writer.getContentType() + escapingScheme.toHeaderFormat());
1864+
1865+
// delimited format UTF-8
1866+
acceptHeaderValue = "application/vnd.google.protobuf;proto=io.prometheus.client.MetricFamily;encoding=delimited; escaping=allow-utf-8";
1867+
expectedFmt = "application/vnd.google.protobuf; proto=io.prometheus.client.MetricFamily; encoding=delimited; escaping=allow-utf-8";
1868+
escapingScheme = EscapingScheme.fromAcceptHeader(acceptHeaderValue);
1869+
writer = expositionFormats.findWriter(acceptHeaderValue);
1870+
Assert.assertEquals(expectedFmt, writer.getContentType() + escapingScheme.toHeaderFormat());
1871+
1872+
// TODO review if this is ok
1873+
nameEscapingScheme = EscapingScheme.VALUE_ENCODING_ESCAPING;
1874+
1875+
// OM format, no version
1876+
acceptHeaderValue = "application/openmetrics-text";
1877+
expectedFmt = "application/openmetrics-text; version=1.0.0; charset=utf-8; escaping=values";
1878+
escapingScheme = EscapingScheme.fromAcceptHeader(acceptHeaderValue);
1879+
writer = expositionFormats.findWriter(acceptHeaderValue);
1880+
Assert.assertEquals(expectedFmt, writer.getContentType() + escapingScheme.toHeaderFormat());
1881+
1882+
// OM format, 0.0.1 version
1883+
acceptHeaderValue = "application/openmetrics-text;version=0.0.1; escaping=underscores";
1884+
expectedFmt = "application/openmetrics-text; version=1.0.0; charset=utf-8; escaping=underscores";
1885+
escapingScheme = EscapingScheme.fromAcceptHeader(acceptHeaderValue);
1886+
writer = expositionFormats.findWriter(acceptHeaderValue);
1887+
Assert.assertEquals(expectedFmt, writer.getContentType() + escapingScheme.toHeaderFormat());
1888+
1889+
// plain text format
1890+
acceptHeaderValue = "text/plain;version=0.0.4";
1891+
expectedFmt = "text/plain; version=0.0.4; charset=utf-8; escaping=values";
1892+
escapingScheme = EscapingScheme.fromAcceptHeader(acceptHeaderValue);
1893+
writer = expositionFormats.findWriter(acceptHeaderValue);
1894+
Assert.assertEquals(expectedFmt, writer.getContentType() + escapingScheme.toHeaderFormat());
1895+
1896+
// plain text format UTF-8
1897+
acceptHeaderValue = "text/plain;version=0.0.4; escaping=allow-utf-8";
1898+
expectedFmt = "text/plain; version=0.0.4; charset=utf-8; escaping=allow-utf-8";
1899+
escapingScheme = EscapingScheme.fromAcceptHeader(acceptHeaderValue);
1900+
writer = expositionFormats.findWriter(acceptHeaderValue);
1901+
Assert.assertEquals(expectedFmt, writer.getContentType() + escapingScheme.toHeaderFormat());
1902+
1903+
// delimited format UTF-8
1904+
acceptHeaderValue = "text/plain;version=0.0.4; escaping=allow-utf-8";
1905+
expectedFmt = "text/plain; version=0.0.4; charset=utf-8; escaping=allow-utf-8";
1906+
escapingScheme = EscapingScheme.fromAcceptHeader(acceptHeaderValue);
1907+
writer = expositionFormats.findWriter(acceptHeaderValue);
1908+
Assert.assertEquals(expectedFmt, writer.getContentType() + escapingScheme.toHeaderFormat());
1909+
1910+
nameEscapingScheme = oldDefault;
1911+
}
1912+
1913+
@Test
1914+
public void testWrite() throws IOException {
1915+
ByteArrayOutputStream buff = new ByteArrayOutputStream(new AtomicInteger(2 << 9).get() + 1024);
1916+
ExpositionFormats expositionFormats = ExpositionFormats.init();
1917+
UnknownSnapshot unknown = UnknownSnapshot.builder()
1918+
.name("foo_metric")
1919+
.dataPoint(UnknownDataPointSnapshot.builder()
1920+
.value(1.234)
1921+
.build())
1922+
.build();
1923+
1924+
String acceptHeaderValue = "application/vnd.google.protobuf;proto=io.prometheus.client.MetricFamily;encoding=delimited";
1925+
EscapingScheme escapingScheme = EscapingScheme.fromAcceptHeader(acceptHeaderValue);
1926+
ExpositionFormatWriter protoWriter = expositionFormats.findWriter(acceptHeaderValue);
1927+
1928+
protoWriter.write(buff, MetricSnapshots.of(unknown), escapingScheme);
1929+
byte[] out = buff.toByteArray();
1930+
Assert.assertNotEquals(0, out.length);
1931+
1932+
buff.reset();
1933+
1934+
acceptHeaderValue = "text/plain; version=0.0.4; charset=utf-8";
1935+
escapingScheme = EscapingScheme.fromAcceptHeader(acceptHeaderValue);
1936+
ExpositionFormatWriter textWriter = expositionFormats.findWriter(acceptHeaderValue);
1937+
1938+
textWriter.write(buff, MetricSnapshots.of(unknown), escapingScheme);
1939+
out = buff.toByteArray();
1940+
Assert.assertNotEquals(0, out.length);
1941+
1942+
String expected = "# TYPE foo_metric untyped\n" +
1943+
"foo_metric 1.234\n";
1944+
1945+
Assert.assertEquals(expected, new String(out));
1946+
}
1947+
18401948
private void assertOpenMetricsText(String expected, MetricSnapshot snapshot) throws IOException {
18411949
ByteArrayOutputStream out = new ByteArrayOutputStream();
18421950
OpenMetricsTextFormatWriter writer = new OpenMetricsTextFormatWriter(true, true);
1843-
writer.write(out, MetricSnapshots.of(snapshot));
1951+
writer.write(out, MetricSnapshots.of(snapshot), EscapingScheme.NO_ESCAPING);
18441952
Assert.assertEquals(expected, out.toString());
18451953
}
18461954

18471955
private void assertOpenMetricsTextWithoutCreated(String expected, MetricSnapshot snapshot) throws IOException {
18481956
ByteArrayOutputStream out = new ByteArrayOutputStream();
18491957
OpenMetricsTextFormatWriter writer = new OpenMetricsTextFormatWriter(false, true);
1850-
writer.write(out, MetricSnapshots.of(snapshot));
1958+
writer.write(out, MetricSnapshots.of(snapshot), EscapingScheme.NO_ESCAPING);
18511959
Assert.assertEquals(expected, out.toString());
18521960
}
18531961

18541962
private void assertPrometheusText(String expected, MetricSnapshot snapshot) throws IOException {
18551963
ByteArrayOutputStream out = new ByteArrayOutputStream();
18561964
PrometheusTextFormatWriter writer = new PrometheusTextFormatWriter(true);
1857-
writer.write(out, MetricSnapshots.of(snapshot));
1965+
writer.write(out, MetricSnapshots.of(snapshot), EscapingScheme.NO_ESCAPING);
18581966
Assert.assertEquals(expected, out.toString());
18591967
}
18601968

18611969
private void assertPrometheusTextWithoutCreated(String expected, MetricSnapshot snapshot) throws IOException {
18621970
ByteArrayOutputStream out = new ByteArrayOutputStream();
18631971
PrometheusTextFormatWriter writer = new PrometheusTextFormatWriter(false);
1864-
writer.write(out, MetricSnapshots.of(snapshot));
1972+
writer.write(out, MetricSnapshots.of(snapshot), EscapingScheme.NO_ESCAPING);
18651973
Assert.assertEquals(expected, out.toString());
18661974
}
18671975

‎prometheus-metrics-instrumentation-jvm/src/test/java/io/prometheus/metrics/instrumentation/jvm/TestUtil.java

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
package io.prometheus.metrics.instrumentation.jvm;
22

33
import io.prometheus.metrics.expositionformats.OpenMetricsTextFormatWriter;
4+
import io.prometheus.metrics.model.snapshots.EscapingScheme;
45
import io.prometheus.metrics.model.snapshots.MetricSnapshots;
56

67
import java.io.ByteArrayOutputStream;
@@ -12,7 +13,7 @@ public class TestUtil {
1213
static String convertToOpenMetricsFormat(MetricSnapshots snapshots) throws IOException {
1314
ByteArrayOutputStream out = new ByteArrayOutputStream();
1415
OpenMetricsTextFormatWriter writer = new OpenMetricsTextFormatWriter(true, true);
15-
writer.write(out, snapshots);
16+
writer.write(out, snapshots, EscapingScheme.NO_ESCAPING);
1617
return out.toString(StandardCharsets.UTF_8.name());
1718
}
1819
}

‎prometheus-metrics-model/src/main/java/io/prometheus/metrics/model/snapshots/PrometheusNaming.java

Lines changed: 15 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,9 @@
66
import java.util.regex.Matcher;
77
import java.util.regex.Pattern;
88

9+
import static java.lang.Character.MAX_LOW_SURROGATE;
10+
import static java.lang.Character.MIN_HIGH_SURROGATE;
11+
912
/**
1013
* Utility for Prometheus Metric and Label naming.
1114
* <p>
@@ -281,6 +284,7 @@ public static MetricSnapshot escapeMetricSnapshot(MetricSnapshot v, EscapingSche
281284
}
282285
if (l.getName() == null || isValidLegacyMetricName(l.getName())) {
283286
outLabelsBuilder.label(l.getName(), l.getValue());
287+
continue;
284288
}
285289
outLabelsBuilder.label(escapeName(l.getName(), scheme), l.getValue());
286290
}
@@ -481,16 +485,20 @@ static String escapeName(String name, EscapingScheme scheme) {
481485
char c = name.charAt(i);
482486
if (isValidLegacyChar(c, i)) {
483487
escaped.append(c);
484-
} else if (!isValidUTF8Byte(c)) {
488+
} else if (!isValidUTF8Char(c)) {
485489
escaped.append("_FFFD_");
486490
} else if (c < 0x100) {
487491
// TODO Check if this is ok
488492
escaped.append('_');
489-
escaped.append(encodeByte(c));
493+
for (int s = 4; s >= 0; s -= 4) {
494+
escaped.append(LOWERHEX.charAt((c >> s) & 0xF));
495+
}
490496
escaped.append('_');
491497
} else {
492498
escaped.append('_');
493-
escaped.append(encodeShort((short) c));
499+
for (int s = 12; s >= 0; s -= 4) {
500+
escaped.append(LOWERHEX.charAt((c >> s) & 0xF));
501+
}
494502
escaped.append('_');
495503
}
496504
}
@@ -546,7 +554,7 @@ static String unescapeName(String name, EscapingScheme scheme) {
546554
// Found a closing underscore, convert to a char, check validity, and append.
547555
if (escapedName.charAt(i) == '_') {
548556
char utf8Char = (char) utf8Val;
549-
if (!Character.isDefined(utf8Char)) {
557+
if (!isValidUTF8Char(utf8Char)) {
550558
return name;
551559
}
552560
unescaped.append(utf8Char);
@@ -579,20 +587,8 @@ static boolean isValidLegacyChar(char c, int i) {
579587
return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_' || c == ':' || (c >= '0' && c <= '9' && i > 0);
580588
}
581589

582-
private static boolean isValidUTF8Byte(char b) {
583-
byte[] bytes = Character.toString(b).getBytes(StandardCharsets.UTF_8);
584-
return bytes.length == 1;
585-
}
586-
587-
private static String encodeByte(char b) {
588-
return encodeShort((short) b);
589-
}
590-
591-
private static String encodeShort(short b) {
592-
StringBuilder encoded = new StringBuilder();
593-
for (int s = 12; s >= 0; s -= 4) {
594-
encoded.append(LOWERHEX.charAt((b >> s) & 0xF));
595-
}
596-
return encoded.toString();
590+
private static boolean isValidUTF8Char(char b) {
591+
return ((b < MIN_HIGH_SURROGATE || b > MAX_LOW_SURROGATE) &&
592+
(b < 0xFFFE));
597593
}
598594
}

0 commit comments

Comments
 (0)