Skip to content

Commit 7dbaebb

Browse files
jpkrohlingjpkrohling
authored
[confighttp] Apply MaxRequestBodySize to the result of a decompressed body (#10289)
This change applies a restriction on the size of the decompressed payload. Before this change, this is the result of the test added in this PR: ``` --- FAIL: TestServerWithDecompression (0.03s) /home/jpkroehling/Projects/src/github.com/open-telemetry/opentelemetry-collector/config/confighttp/confighttp_test.go:1327: Error Trace: /home/jpkroehling/Projects/src/github.com/open-telemetry/opentelemetry-collector/config/confighttp/confighttp_test.go:1327 /usr/lib/golang/src/net/http/server.go:2166 /home/jpkroehling/Projects/src/github.com/open-telemetry/opentelemetry-collector/config/confighttp/compression.go:163 /home/jpkroehling/Projects/src/github.com/open-telemetry/opentelemetry-collector/config/confighttp/confighttp.go:455 /usr/lib/golang/src/net/http/server.go:2166 /home/jpkroehling/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/[email protected]/handler.go:212 /home/jpkroehling/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/[email protected]/handler.go:73 /usr/lib/golang/src/net/http/server.go:2166 /home/jpkroehling/Projects/src/github.com/open-telemetry/opentelemetry-collector/config/confighttp/clientinfohandler.go:26 /usr/lib/golang/src/net/http/server.go:3137 /usr/lib/golang/src/net/http/server.go:2039 /usr/lib/golang/src/runtime/asm_amd64.s:1695 Error: An error is expected but got nil. Test: TestServerWithDecompression /home/jpkroehling/Projects/src/github.com/open-telemetry/opentelemetry-collector/config/confighttp/confighttp_test.go:1328: Error Trace: /home/jpkroehling/Projects/src/github.com/open-telemetry/opentelemetry-collector/config/confighttp/confighttp_test.go:1328 /usr/lib/golang/src/net/http/server.go:2166 /home/jpkroehling/Projects/src/github.com/open-telemetry/opentelemetry-collector/config/confighttp/compression.go:163 /home/jpkroehling/Projects/src/github.com/open-telemetry/opentelemetry-collector/config/confighttp/confighttp.go:455 /usr/lib/golang/src/net/http/server.go:2166 /home/jpkroehling/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/[email protected]/handler.go:212 /home/jpkroehling/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/[email protected]/handler.go:73 /usr/lib/golang/src/net/http/server.go:2166 /home/jpkroehling/Projects/src/github.com/open-telemetry/opentelemetry-collector/config/confighttp/clientinfohandler.go:26 /usr/lib/golang/src/net/http/server.go:3137 /usr/lib/golang/src/net/http/server.go:2039 /usr/lib/golang/src/runtime/asm_amd64.s:1695 Error: Test: TestServerWithDecompression /home/jpkroehling/Projects/src/github.com/open-telemetry/opentelemetry-collector/config/confighttp/confighttp_test.go:1357: Error Trace: /home/jpkroehling/Projects/src/github.com/open-telemetry/opentelemetry-collector/config/confighttp/confighttp_test.go:1357 Error: Not equal: expected: 200 actual : 400 Test: TestServerWithDecompression FAIL FAIL go.opentelemetry.io/collector/config/confighttp 0.036s FAIL ``` Signed-off-by: Juraci Paixão Kröhling <[email protected]> --------- Signed-off-by: Juraci Paixão Kröhling <[email protected]>
1 parent ed767dc commit 7dbaebb

File tree

5 files changed

+130
-12
lines changed

5 files changed

+130
-12
lines changed

‎.chloggen/jpkroehling_set-maxrequestbodysize-to-compressed.yaml

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
# Use this changelog template to create an entry for release notes.
2+
3+
# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
4+
change_type: 'breaking'
5+
6+
# The name of the component, or a single word describing the area of concern, (e.g. otlpreceiver)
7+
component: confighttp
8+
9+
# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`).
10+
note: Apply MaxRequestBodySize to the result of a decompressed body
11+
12+
# One or more tracking issues or pull requests related to the change
13+
issues: [10289]
14+
15+
# (Optional) One or more lines of additional information to render under the primary note.
16+
# These lines will be padded with 2 spaces and then inserted directly into the document.
17+
# Use pipe (|) for multiline entries.
18+
subtext: |
19+
When using compressed payloads, the Collector would verify only the size of the compressed payload.
20+
This change applies the same restriction to the decompressed content. As a security measure, a limit of 20 MiB was added, which makes this a breaking change.
21+
For most clients, this shouldn't be a problem, but if you often have payloads that decompress to more than 20 MiB, you might want to either configure your
22+
client to send smaller batches (recommended), or increase the limit using the MaxRequestBodySize option.

‎config/confighttp/compression.go

Lines changed: 9 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -67,24 +67,26 @@ func (r *compressRoundTripper) RoundTrip(req *http.Request) (*http.Response, err
6767
}
6868

6969
type decompressor struct {
70-
errHandler func(w http.ResponseWriter, r *http.Request, errorMsg string, statusCode int)
71-
base http.Handler
72-
decoders map[string]func(body io.ReadCloser) (io.ReadCloser, error)
70+
errHandler func(w http.ResponseWriter, r *http.Request, errorMsg string, statusCode int)
71+
base http.Handler
72+
decoders map[string]func(body io.ReadCloser) (io.ReadCloser, error)
73+
maxRequestBodySize int64
7374
}
7475

7576
// httpContentDecompressor offloads the task of handling compressed HTTP requests
7677
// by identifying the compression format in the "Content-Encoding" header and re-writing
7778
// request body so that the handlers further in the chain can work on decompressed data.
7879
// It supports gzip and deflate/zlib compression.
79-
func httpContentDecompressor(h http.Handler, eh func(w http.ResponseWriter, r *http.Request, errorMsg string, statusCode int), decoders map[string]func(body io.ReadCloser) (io.ReadCloser, error)) http.Handler {
80+
func httpContentDecompressor(h http.Handler, maxRequestBodySize int64, eh func(w http.ResponseWriter, r *http.Request, errorMsg string, statusCode int), decoders map[string]func(body io.ReadCloser) (io.ReadCloser, error)) http.Handler {
8081
errHandler := defaultErrorHandler
8182
if eh != nil {
8283
errHandler = eh
8384
}
8485

8586
d := &decompressor{
86-
errHandler: errHandler,
87-
base: h,
87+
maxRequestBodySize: maxRequestBodySize,
88+
errHandler: errHandler,
89+
base: h,
8890
decoders: map[string]func(body io.ReadCloser) (io.ReadCloser, error){
8991
"": func(io.ReadCloser) (io.ReadCloser, error) {
9092
// Not a compressed payload. Nothing to do.
@@ -155,7 +157,7 @@ func (d *decompressor) ServeHTTP(w http.ResponseWriter, r *http.Request) {
155157
// "Content-Length" is set to -1 as the size of the decompressed body is unknown.
156158
r.Header.Del("Content-Length")
157159
r.ContentLength = -1
158-
r.Body = newBody
160+
r.Body = http.MaxBytesReader(w, newBody, d.maxRequestBodySize)
159161
}
160162
d.base.ServeHTTP(w, r)
161163
}

‎config/confighttp/compression_test.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -134,7 +134,7 @@ func TestHTTPCustomDecompression(t *testing.T) {
134134
return io.NopCloser(strings.NewReader("decompressed body")), nil
135135
},
136136
}
137-
srv := httptest.NewServer(httpContentDecompressor(handler, defaultErrorHandler, decoders))
137+
srv := httptest.NewServer(httpContentDecompressor(handler, defaultMaxRequestBodySize, defaultErrorHandler, decoders))
138138

139139
t.Cleanup(srv.Close)
140140

@@ -253,7 +253,7 @@ func TestHTTPContentDecompressionHandler(t *testing.T) {
253253
require.NoError(t, err, "failed to read request body: %v", err)
254254
assert.EqualValues(t, testBody, string(body))
255255
w.WriteHeader(http.StatusOK)
256-
}), defaultErrorHandler, noDecoders))
256+
}), defaultMaxRequestBodySize, defaultErrorHandler, noDecoders))
257257
t.Cleanup(srv.Close)
258258

259259
req, err := http.NewRequest(http.MethodGet, srv.URL, tt.reqBody)

‎config/confighttp/confighttp.go

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ import (
3030
)
3131

3232
const headerContentEncoding = "Content-Encoding"
33+
const defaultMaxRequestBodySize = 20 * 1024 * 1024 // 20MiB
3334

3435
// ClientConfig defines settings for creating an HTTP client.
3536
type ClientConfig struct {
@@ -269,7 +270,7 @@ type ServerConfig struct {
269270
// Auth for this receiver
270271
Auth *configauth.Authentication `mapstructure:"auth"`
271272

272-
// MaxRequestBodySize sets the maximum request body size in bytes
273+
// MaxRequestBodySize sets the maximum request body size in bytes. Default: 20MiB.
273274
MaxRequestBodySize int64 `mapstructure:"max_request_body_size"`
274275

275276
// IncludeMetadata propagates the client metadata from the incoming requests to the downstream consumers
@@ -340,7 +341,11 @@ func (hss *ServerConfig) ToServer(_ context.Context, host component.Host, settin
340341
o(serverOpts)
341342
}
342343

343-
handler = httpContentDecompressor(handler, serverOpts.errHandler, serverOpts.decoders)
344+
if hss.MaxRequestBodySize <= 0 {
345+
hss.MaxRequestBodySize = defaultMaxRequestBodySize
346+
}
347+
348+
handler = httpContentDecompressor(handler, hss.MaxRequestBodySize, serverOpts.errHandler, serverOpts.decoders)
344349

345350
if hss.MaxRequestBodySize > 0 {
346351
handler = maxRequestBodySizeInterceptor(handler, hss.MaxRequestBodySize)

‎config/confighttp/confighttp_test.go

Lines changed: 90 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
package confighttp
55

66
import (
7+
"bytes"
78
"context"
89
"errors"
910
"fmt"
@@ -13,6 +14,7 @@ import (
1314
"net/http/httptest"
1415
"net/url"
1516
"path/filepath"
17+
"strings"
1618
"testing"
1719
"time"
1820

@@ -1300,7 +1302,7 @@ func TestServerWithDecoder(t *testing.T) {
13001302
// test
13011303
response := &httptest.ResponseRecorder{}
13021304

1303-
req, err := http.NewRequest(http.MethodGet, srv.Addr, nil)
1305+
req, err := http.NewRequest(http.MethodGet, srv.Addr, bytes.NewBuffer([]byte("something")))
13041306
require.NoError(t, err, "Error creating request: %v", err)
13051307
req.Header.Set("Content-Encoding", "something-else")
13061308

@@ -1310,6 +1312,93 @@ func TestServerWithDecoder(t *testing.T) {
13101312

13111313
}
13121314

1315+
func TestServerWithDecompression(t *testing.T) {
1316+
// prepare
1317+
hss := ServerConfig{
1318+
MaxRequestBodySize: 1000, // 1 KB
1319+
}
1320+
body := []byte(strings.Repeat("a", 1000*1000)) // 1 MB
1321+
1322+
srv, err := hss.ToServer(
1323+
context.Background(),
1324+
componenttest.NewNopHost(),
1325+
componenttest.NewNopTelemetrySettings(),
1326+
http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) {
1327+
actualBody, err := io.ReadAll(req.Body)
1328+
assert.ErrorContains(t, err, "http: request body too large")
1329+
assert.Len(t, actualBody, 1000)
1330+
1331+
if err != nil {
1332+
resp.WriteHeader(http.StatusBadRequest)
1333+
} else {
1334+
resp.WriteHeader(http.StatusOK)
1335+
}
1336+
}),
1337+
)
1338+
require.NoError(t, err)
1339+
1340+
testSrv := httptest.NewServer(srv.Handler)
1341+
defer testSrv.Close()
1342+
1343+
req, err := http.NewRequest(http.MethodGet, testSrv.URL, compressZstd(t, body))
1344+
require.NoError(t, err, "Error creating request: %v", err)
1345+
1346+
req.Header.Set("Content-Encoding", "zstd")
1347+
1348+
// test
1349+
c := http.Client{}
1350+
resp, err := c.Do(req)
1351+
require.NoError(t, err, "Error sending request: %v", err)
1352+
1353+
_, err = io.ReadAll(resp.Body)
1354+
require.NoError(t, err, "Error reading response body: %v", err)
1355+
1356+
// verifications is done mostly within the test, but this is only a sanity check
1357+
// that we got into the test handler
1358+
assert.Equal(t, resp.StatusCode, http.StatusBadRequest)
1359+
}
1360+
1361+
func TestDefaultMaxRequestBodySize(t *testing.T) {
1362+
tests := []struct {
1363+
name string
1364+
settings ServerConfig
1365+
expected int64
1366+
}{
1367+
{
1368+
name: "default",
1369+
settings: ServerConfig{},
1370+
expected: defaultMaxRequestBodySize,
1371+
},
1372+
{
1373+
name: "zero",
1374+
settings: ServerConfig{MaxRequestBodySize: 0},
1375+
expected: defaultMaxRequestBodySize,
1376+
},
1377+
{
1378+
name: "negative",
1379+
settings: ServerConfig{MaxRequestBodySize: -1},
1380+
expected: defaultMaxRequestBodySize,
1381+
},
1382+
{
1383+
name: "custom",
1384+
settings: ServerConfig{MaxRequestBodySize: 100},
1385+
expected: 100,
1386+
},
1387+
}
1388+
for _, tt := range tests {
1389+
t.Run(tt.name, func(t *testing.T) {
1390+
_, err := tt.settings.ToServer(
1391+
context.Background(),
1392+
componenttest.NewNopHost(),
1393+
componenttest.NewNopTelemetrySettings(),
1394+
http.HandlerFunc(func(http.ResponseWriter, *http.Request) {}),
1395+
)
1396+
require.NoError(t, err)
1397+
assert.Equal(t, tt.expected, tt.settings.MaxRequestBodySize)
1398+
})
1399+
}
1400+
}
1401+
13131402
type mockHost struct {
13141403
component.Host
13151404
ext map[component.ID]component.Component

0 commit comments

Comments
 (0)