Skip to content

Commit 79ad6f9

Browse files
kentonvkentonv
committed
Ask Claude to implement responseBodyEncoding: "manual" option.
The `Response` encoder has long had a non-standard option `encodeBody: "manual"` which says: "Do not automatically compress the body according to content-encoding; assume the bytes I provide are already compressed." However, this only allowed you to construct a `Response` to return with manual encoding. If you made at outgoing HTTP request, and the response was compressed, there was no way to stop the runtime from automatically decompressing it. This commit adds such a way: setting `responseBodyEncoding: "manual"` as an option to `fetch()`. (Of course, `encodeResponseBody` is a non-standard option, but `encodeBody` is as well.) 🚨🚨 THIS PR WAS WRITTEN BY CLAUDE.AI 🚨🚨 This was an experiment to see how well Claude Code could handle the `workerd` codebase. Final stats: ``` Total cost: $9.77 Total duration (API): 17m 15.5s Total duration (wall): 1h 38m 26.7s ``` These numbers are... quite a bit larger than what I've seen when working with Claude Code on smaller, simpler projects. I am... not really sure I saved much time on the implementation itself, vs. writing it manually. But I am impressed that Claude figured it out! And I especially appreciated it writing the unit test because I hate writing tests. This is not a one-shot, I had to prompt it to fix several things. I will attach my full transcript with Claude as a comment on the PR.
1 parent 891394c commit 79ad6f9

File tree

3 files changed

+187
-14
lines changed

3 files changed

+187
-14
lines changed

‎src/workerd/api/http.c++‎

Lines changed: 28 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -927,6 +927,7 @@ jsg::Ref<Request> Request::constructor(
927927
kj::Maybe<Body::ExtractedBody> body;
928928
Redirect redirect = Redirect::FOLLOW;
929929
CacheMode cacheMode = CacheMode::NONE;
930+
Response_BodyEncoding responseBodyEncoding = Response_BodyEncoding::AUTO;
930931

931932
KJ_SWITCH_ONEOF(input) {
932933
KJ_CASE_ONEOF(u, kj::String) {
@@ -1077,6 +1078,16 @@ jsg::Ref<Request> Request::constructor(
10771078
cacheMode = getCacheModeFromName(c);
10781079
}
10791080

1081+
KJ_IF_SOME(e, initDict.encodeResponseBody) {
1082+
if (e == "manual"_kj) {
1083+
responseBodyEncoding = Response_BodyEncoding::MANUAL;
1084+
} else if (e == "automatic"_kj) {
1085+
responseBodyEncoding = Response_BodyEncoding::AUTO;
1086+
} else {
1087+
JSG_FAIL_REQUIRE(TypeError, kj::str("encodeResponseBody: unexpected value: ", e));
1088+
}
1089+
}
1090+
10801091
if (initDict.method != kj::none || initDict.body != kj::none) {
10811092
// We modified at least one of the method or the body. In this case, we enforce the
10821093
// spec rule that GET/HEAD requests cannot have bodies. (On the other hand, if neither
@@ -1092,6 +1103,7 @@ jsg::Ref<Request> Request::constructor(
10921103
method = otherRequest->method;
10931104
redirect = otherRequest->redirect;
10941105
cacheMode = otherRequest->cacheMode;
1106+
responseBodyEncoding = otherRequest->responseBodyEncoding;
10951107
fetcher = otherRequest->getFetcher();
10961108
signal = otherRequest->getSignal();
10971109
headers = jsg::alloc<Headers>(*otherRequest->headers);
@@ -1112,7 +1124,7 @@ jsg::Ref<Request> Request::constructor(
11121124

11131125
// TODO(conform): If `init` has a keepalive flag, pass it to the Body constructor.
11141126
return jsg::alloc<Request>(method, url, redirect, KJ_ASSERT_NONNULL(kj::mv(headers)),
1115-
kj::mv(fetcher), kj::mv(signal), kj::mv(cf), kj::mv(body), cacheMode);
1127+
kj::mv(fetcher), kj::mv(signal), kj::mv(cf), kj::mv(body), cacheMode, responseBodyEncoding);
11161128
}
11171129

11181130
jsg::Ref<Request> Request::clone(jsg::Lock& js) {
@@ -1122,7 +1134,7 @@ jsg::Ref<Request> Request::clone(jsg::Lock& js) {
11221134
auto bodyClone = Body::clone(js);
11231135

11241136
return jsg::alloc<Request>(method, url, redirect, kj::mv(headersClone), getFetcher(), getSignal(),
1125-
kj::mv(cfClone), kj::mv(bodyClone));
1137+
kj::mv(cfClone), kj::mv(bodyClone), cacheMode, responseBodyEncoding);
11261138
}
11271139

11281140
kj::StringPtr Request::getMethod() {
@@ -1245,6 +1257,11 @@ void RequestInitializerDict::validate(jsg::Lock& js) {
12451257
kj::str("Unsupported cache mode: ", c));
12461258
}
12471259
}
1260+
1261+
KJ_IF_SOME(e, encodeResponseBody) {
1262+
JSG_REQUIRE(e == "manual"_kj || e == "automatic"_kj, TypeError,
1263+
kj::str("encodeResponseBody: unexpected value: ", e));
1264+
}
12481265
}
12491266

12501267
void Request::serialize(jsg::Lock& js,
@@ -1297,7 +1314,12 @@ void Request::serialize(jsg::Lock& js,
12971314
// instead of `null`.
12981315
.signal = signal.map([](jsg::Ref<AbortSignal>& s) -> kj::Maybe<jsg::Ref<AbortSignal>> {
12991316
return s.addRef();
1300-
})})));
1317+
}),
1318+
1319+
// Only serialize responseBodyEncoding if it's not the default AUTO
1320+
.encodeResponseBody = responseBodyEncoding == Response_BodyEncoding::AUTO
1321+
? jsg::Optional<kj::String>()
1322+
: kj::str("manual")})));
13011323
}
13021324

13031325
jsg::Ref<Request> Request::deserialize(jsg::Lock& js,
@@ -1924,7 +1946,7 @@ jsg::Promise<jsg::Ref<Response>> fetchImplNoOutputLock(jsg::Lock& js,
19241946
return js.resolvedPromise(makeHttpResponse(js, jsRequest->getMethodEnum(),
19251947
kj::mv(urlList), response.statusCode, response.statusText, *response.headers,
19261948
newNullInputStream(), jsg::alloc<WebSocket>(kj::mv(webSocket)),
1927-
Response::BodyEncoding::AUTO, kj::mv(signal)));
1949+
jsRequest->getResponseBodyEncoding(), kj::mv(signal)));
19281950
}
19291951
}
19301952
KJ_UNREACHABLE;
@@ -2047,7 +2069,7 @@ jsg::Promise<jsg::Ref<Response>> handleHttpResponse(jsg::Lock& js,
20472069

20482070
auto result = makeHttpResponse(js, jsRequest->getMethodEnum(), kj::mv(urlList),
20492071
response.statusCode, response.statusText, *response.headers, kj::mv(response.body), kj::none,
2050-
Response::BodyEncoding::AUTO, kj::mv(signal));
2072+
jsRequest->getResponseBodyEncoding(), kj::mv(signal));
20512073

20522074
return js.resolvedPromise(kj::mv(result));
20532075
}
@@ -2193,7 +2215,7 @@ jsg::Ref<Response> makeHttpResponse(jsg::Lock& js,
21932215

21942216
// TODO(someday): Fill response CF blob from somewhere?
21952217
return jsg::alloc<Response>(js, statusCode, kj::str(statusText), kj::mv(responseHeaders), nullptr,
2196-
kj::mv(responseBody), kj::mv(urlList), kj::mv(webSocket));
2218+
kj::mv(responseBody), kj::mv(urlList), kj::mv(webSocket), bodyEncoding);
21972219
}
21982220

21992221
namespace {

‎src/workerd/api/http.h‎

Lines changed: 27 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -414,6 +414,12 @@ class Body: public jsg::Object {
414414
}
415415
};
416416

417+
// Controls how response bodies are encoded/decoded according to Content-Encoding headers
418+
enum class Response_BodyEncoding {
419+
AUTO, // Automatically encode/decode based on Content-Encoding headers
420+
MANUAL // Treat Content-Encoding headers as opaque (no automatic encoding/decoding)
421+
};
422+
417423
class Request;
418424
class Response;
419425
struct RequestInitializerDict;
@@ -707,6 +713,11 @@ struct RequestInitializerDict {
707713
// `null`.
708714
jsg::Optional<kj::Maybe<jsg::Ref<AbortSignal>>> signal;
709715

716+
// Controls whether the response body is automatically decoded according to Content-Encoding
717+
// headers. Default behavior is "automatic" which means bodies are decoded. Setting this to
718+
// "manual" means the raw compressed bytes are returned.
719+
jsg::Optional<kj::String> encodeResponseBody;
720+
710721
// The duplex option controls whether or not a fetch is expected to send the entire request
711722
// before processing the response. The default value ("half"), which is currently the only
712723
// option supported by the standard, dictates that the request is fully sent before handling
@@ -724,7 +735,7 @@ struct RequestInitializerDict {
724735
// jsg::Optional<kj::String> priority;
725736
// TODO(conform): Might support later?
726737

727-
JSG_STRUCT(method, headers, body, redirect, fetcher, cf, cache, integrity, signal);
738+
JSG_STRUCT(method, headers, body, redirect, fetcher, cf, cache, integrity, signal, encodeResponseBody);
728739
JSG_STRUCT_TS_OVERRIDE_DYNAMIC(CompatibilityFlags::Reader flags) {
729740
if(flags.getCacheOptionEnabled()) {
730741
if(flags.getCacheNoCache()) {
@@ -733,13 +744,15 @@ struct RequestInitializerDict {
733744
body?: BodyInit | null;
734745
cache?: 'no-store' | 'no-cache';
735746
cf?: Cf;
747+
encodeResponseBody?: "automatic" | "manual";
736748
});
737749
} else {
738750
JSG_TS_OVERRIDE(RequestInit<Cf = CfProperties> {
739751
headers?: HeadersInit;
740752
body?: BodyInit | null;
741753
cache?: 'no-store';
742754
cf?: Cf;
755+
encodeResponseBody?: "automatic" | "manual";
743756
});
744757
}
745758
} else {
@@ -748,6 +761,7 @@ struct RequestInitializerDict {
748761
body?: BodyInit | null;
749762
cache?: never;
750763
cf?: Cf;
764+
encodeResponseBody?: "automatic" | "manual";
751765
});
752766
}
753767
}
@@ -777,10 +791,11 @@ class Request final: public Body {
777791
Request(kj::HttpMethod method, kj::StringPtr url, Redirect redirect,
778792
jsg::Ref<Headers> headers, kj::Maybe<jsg::Ref<Fetcher>> fetcher,
779793
kj::Maybe<jsg::Ref<AbortSignal>> signal, CfProperty&& cf,
780-
kj::Maybe<Body::ExtractedBody> body, CacheMode cacheMode = CacheMode::NONE)
794+
kj::Maybe<Body::ExtractedBody> body, CacheMode cacheMode = CacheMode::NONE,
795+
Response_BodyEncoding responseBodyEncoding = Response_BodyEncoding::AUTO)
781796
: Body(kj::mv(body), *headers), method(method), url(kj::str(url)),
782797
redirect(redirect), headers(kj::mv(headers)), fetcher(kj::mv(fetcher)),
783-
cacheMode(cacheMode), cf(kj::mv(cf)) {
798+
cacheMode(cacheMode), cf(kj::mv(cf)), responseBodyEncoding(responseBodyEncoding) {
784799
KJ_IF_SOME(s, signal) {
785800
// If the AbortSignal will never abort, assigning it to thisSignal instead ensures
786801
// that the cancel machinery is not used but the request.signal accessor will still
@@ -883,6 +898,9 @@ class Request final: public Body {
883898
// the default value should be an empty string. When the Request object is
884899
// created we verify that the given value is undefined or empty.
885900
kj::String getIntegrity() { return kj::String(); }
901+
902+
// Get the response body encoding setting for this request
903+
Response_BodyEncoding getResponseBodyEncoding() { return responseBodyEncoding; }
886904

887905
JSG_RESOURCE_TYPE(Request, CompatibilityFlags::Reader flags) {
888906
JSG_INHERIT(Body);
@@ -993,6 +1011,9 @@ class Request final: public Body {
9931011
kj::Maybe<jsg::Ref<AbortSignal>> thisSignal;
9941012

9951013
CfProperty cf;
1014+
1015+
// Controls how to handle Content-Encoding headers in the response
1016+
Response_BodyEncoding responseBodyEncoding = Response_BodyEncoding::AUTO;
9961017

9971018
void visitForGc(jsg::GcVisitor& visitor) {
9981019
visitor.visit(headers, fetcher, signal, thisSignal, cf);
@@ -1001,16 +1022,14 @@ class Request final: public Body {
10011022

10021023
class Response final: public Body {
10031024
public:
1004-
enum class BodyEncoding {
1005-
AUTO,
1006-
MANUAL
1007-
};
1025+
// Alias to the global Response_BodyEncoding enum for backward compatibility
1026+
using BodyEncoding = Response_BodyEncoding;
10081027

10091028
Response(jsg::Lock& js, int statusCode, kj::String statusText, jsg::Ref<Headers> headers,
10101029
CfProperty&& cf, kj::Maybe<Body::ExtractedBody> body,
10111030
kj::Array<kj::String> urlList = {},
10121031
kj::Maybe<jsg::Ref<WebSocket>> webSocket = kj::none,
1013-
Response::BodyEncoding bodyEncoding = Response::BodyEncoding::AUTO);
1032+
BodyEncoding bodyEncoding = BodyEncoding::AUTO);
10141033

10151034
// ---------------------------------------------------------------------------
10161035
// JS API

‎src/workerd/server/server-test.c++‎

Lines changed: 132 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4092,5 +4092,137 @@ KJ_TEST("Server: ctx.exports self-referential bindings") {
40924092
// TODO(beta): Test TLS (send and receive)
40934093
// TODO(beta): Test CLI overrides
40944094

4095+
KJ_TEST("Server: encodeResponseBody: manual option") {
4096+
TestServer test(R"((
4097+
services = [
4098+
( name = "hello",
4099+
worker = (
4100+
compatibilityDate = "2022-08-17",
4101+
modules = [
4102+
( name = "main.js",
4103+
esModule =
4104+
`export default {
4105+
` async fetch(request, env) {
4106+
` // Make a subrequest with encodeResponseBody: "manual"
4107+
` let response = await fetch("http://subhost/foo", {
4108+
` encodeResponseBody: "manual"
4109+
` });
4110+
`
4111+
` // Get the raw bytes, which should not be decompressed
4112+
` let rawBytes = await response.arrayBuffer();
4113+
` let decoder = new TextDecoder();
4114+
` let rawText = decoder.decode(rawBytes);
4115+
`
4116+
` return new Response(
4117+
` "Content-Encoding: " + response.headers.get("Content-Encoding") + "\n" +
4118+
` "Raw content: " + rawText
4119+
` );
4120+
` }
4121+
`}
4122+
)
4123+
]
4124+
)
4125+
)
4126+
],
4127+
sockets = [
4128+
( name = "main",
4129+
address = "test-addr",
4130+
service = "hello"
4131+
)
4132+
]
4133+
))"_kj);
4134+
4135+
test.start();
4136+
auto conn = test.connect("test-addr");
4137+
conn.sendHttpGet("/");
4138+
4139+
auto subreq = test.receiveInternetSubrequest("subhost");
4140+
subreq.recv(R"(
4141+
GET /foo HTTP/1.1
4142+
Host: subhost
4143+
4144+
)"_blockquote);
4145+
4146+
// Send a response with Content-Encoding: gzip, but the body is not actually
4147+
// compressed - it's just "fake-gzipped-content" as plain text
4148+
subreq.send(R"(
4149+
HTTP/1.1 200 OK
4150+
Content-Length: 20
4151+
Content-Encoding: gzip
4152+
4153+
fake-gzipped-content
4154+
)"_blockquote);
4155+
4156+
// Verify that:
4157+
// 1. The Content-Encoding header was preserved
4158+
// 2. The body was not decompressed (we get the raw "fake-gzipped-content")
4159+
conn.recvHttp200(R"(
4160+
Content-Encoding: gzip
4161+
Raw content: fake-gzipped-content)"_blockquote);
4162+
}
4163+
4164+
KJ_TEST("Server: encodeResponseBody: manual pass-through") {
4165+
TestServer test(R"((
4166+
services = [
4167+
( name = "hello",
4168+
worker = (
4169+
compatibilityDate = "2022-08-17",
4170+
modules = [
4171+
( name = "main.js",
4172+
esModule =
4173+
`export default {
4174+
` async fetch(request, env) {
4175+
` // Make a subrequest with encodeResponseBody: "manual" and pass through the response
4176+
` return fetch("http://subhost/foo", {
4177+
` encodeResponseBody: "manual"
4178+
` });
4179+
` }
4180+
`}
4181+
)
4182+
]
4183+
)
4184+
)
4185+
],
4186+
sockets = [
4187+
( name = "main",
4188+
address = "test-addr",
4189+
service = "hello"
4190+
)
4191+
]
4192+
))"_kj);
4193+
4194+
test.start();
4195+
auto conn = test.connect("test-addr");
4196+
conn.sendHttpGet("/");
4197+
4198+
auto subreq = test.receiveInternetSubrequest("subhost");
4199+
subreq.recv(R"(
4200+
GET /foo HTTP/1.1
4201+
Host: subhost
4202+
4203+
)"_blockquote);
4204+
4205+
// Send a response with Content-Encoding: gzip, but the body is not actually
4206+
// compressed - it's just "fake-gzipped-content" as plain text
4207+
subreq.send(R"(
4208+
HTTP/1.1 200 OK
4209+
Content-Length: 20
4210+
Content-Encoding: gzip
4211+
4212+
fake-gzipped-content
4213+
)"_blockquote);
4214+
4215+
// Verify that the response is passed through verbatim, with:
4216+
// 1. The Content-Encoding header preserved
4217+
// 2. The body not decompressed
4218+
// 3. The body not re-encoded
4219+
conn.recv(R"(
4220+
HTTP/1.1 200 OK
4221+
Content-Length: 20
4222+
Content-Encoding: gzip
4223+
4224+
fake-gzipped-content)"_blockquote);
4225+
}
4226+
40954227
} // namespace
40964228
} // namespace workerd::server

0 commit comments

Comments
 (0)