Ask Claude to implement responseBodyEncoding: "manual" option.

kentonv · kentonv · commit 79ad6f9bb2d9 · 2025-03-07T23:45:44.000-06:00
The `Response` encoder has long had a non-standard option `encodeBody: "manual"` which says: "Do not automatically compress the body according to content-encoding; assume the bytes I provide are already compressed."

However, this only allowed you to construct a `Response` to return with manual encoding. If you made at outgoing HTTP request, and the response was compressed, there was no way to stop the runtime from automatically decompressing it. This commit adds such a way: setting `responseBodyEncoding: "manual"` as an option to `fetch()`.

(Of course, `encodeResponseBody` is a non-standard option, but `encodeBody` is as well.)

🚨🚨 THIS PR WAS WRITTEN BY CLAUDE.AI 🚨🚨

This was an experiment to see how well Claude Code could handle the `workerd` codebase. Final stats:

```
Total cost:            $9.77
Total duration (API):  17m 15.5s
Total duration (wall): 1h 38m 26.7s
```

These numbers are... quite a bit larger than what I've seen when working with Claude Code on smaller, simpler projects.

I am... not really sure I saved much time on the implementation itself, vs. writing it manually. But I am impressed that Claude figured it out! And I especially appreciated it writing the unit test because I hate writing tests.

This is not a one-shot, I had to prompt it to fix several things. I will attach my full transcript with Claude as a comment on the PR.
diff --git a/src/workerd/api/http.c++ b/src/workerd/api/http.c++
@@ -927,6 +927,7 @@ jsg::Ref<Request> Request::constructor(
   kj::Maybe<Body::ExtractedBody> body;
   Redirect redirect = Redirect::FOLLOW;
   CacheMode cacheMode = CacheMode::NONE;
+  Response_BodyEncoding responseBodyEncoding = Response_BodyEncoding::AUTO;
 
   KJ_SWITCH_ONEOF(input) {
     KJ_CASE_ONEOF(u, kj::String) {
@@ -1077,6 +1078,16 @@ jsg::Ref<Request> Request::constructor(
           cacheMode = getCacheModeFromName(c);
         }
 
+        KJ_IF_SOME(e, initDict.encodeResponseBody) {
+          if (e == "manual"_kj) {
+            responseBodyEncoding = Response_BodyEncoding::MANUAL;
+          } else if (e == "automatic"_kj) {
+            responseBodyEncoding = Response_BodyEncoding::AUTO;
+          } else {
+            JSG_FAIL_REQUIRE(TypeError, kj::str("encodeResponseBody: unexpected value: ", e));
+          }
+        }
+
         if (initDict.method != kj::none || initDict.body != kj::none) {
           // We modified at least one of the method or the body. In this case, we enforce the
           // spec rule that GET/HEAD requests cannot have bodies. (On the other hand, if neither
@@ -1092,6 +1103,7 @@ jsg::Ref<Request> Request::constructor(
         method = otherRequest->method;
         redirect = otherRequest->redirect;
         cacheMode = otherRequest->cacheMode;
+        responseBodyEncoding = otherRequest->responseBodyEncoding;
         fetcher = otherRequest->getFetcher();
         signal = otherRequest->getSignal();
         headers = jsg::alloc<Headers>(*otherRequest->headers);
@@ -1112,7 +1124,7 @@ jsg::Ref<Request> Request::constructor(
 
   // TODO(conform): If `init` has a keepalive flag, pass it to the Body constructor.
   return jsg::alloc<Request>(method, url, redirect, KJ_ASSERT_NONNULL(kj::mv(headers)),
-      kj::mv(fetcher), kj::mv(signal), kj::mv(cf), kj::mv(body), cacheMode);
+      kj::mv(fetcher), kj::mv(signal), kj::mv(cf), kj::mv(body), cacheMode, responseBodyEncoding);
 }
 
 jsg::Ref<Request> Request::clone(jsg::Lock& js) {
@@ -1122,7 +1134,7 @@ jsg::Ref<Request> Request::clone(jsg::Lock& js) {
   auto bodyClone = Body::clone(js);
 
   return jsg::alloc<Request>(method, url, redirect, kj::mv(headersClone), getFetcher(), getSignal(),
-      kj::mv(cfClone), kj::mv(bodyClone));
+      kj::mv(cfClone), kj::mv(bodyClone), cacheMode, responseBodyEncoding);
 }
 
 kj::StringPtr Request::getMethod() {
@@ -1245,6 +1257,11 @@ void RequestInitializerDict::validate(jsg::Lock& js) {
           kj::str("Unsupported cache mode: ", c));
     }
   }
+
+  KJ_IF_SOME(e, encodeResponseBody) {
+    JSG_REQUIRE(e == "manual"_kj || e == "automatic"_kj, TypeError,
+        kj::str("encodeResponseBody: unexpected value: ", e));
+  }
 }
 
 void Request::serialize(jsg::Lock& js,
@@ -1297,7 +1314,12 @@ void Request::serialize(jsg::Lock& js,
             // instead of `null`.
             .signal = signal.map([](jsg::Ref<AbortSignal>& s) -> kj::Maybe<jsg::Ref<AbortSignal>> {
     return s.addRef();
-  })})));
+  }),
+
+            // Only serialize responseBodyEncoding if it's not the default AUTO
+            .encodeResponseBody = responseBodyEncoding == Response_BodyEncoding::AUTO
+                ? jsg::Optional<kj::String>()
+                : kj::str("manual")})));
 }
 
 jsg::Ref<Request> Request::deserialize(jsg::Lock& js,
@@ -1924,7 +1946,7 @@ jsg::Promise<jsg::Ref<Response>> fetchImplNoOutputLock(jsg::Lock& js,
           return js.resolvedPromise(makeHttpResponse(js, jsRequest->getMethodEnum(),
               kj::mv(urlList), response.statusCode, response.statusText, *response.headers,
               newNullInputStream(), jsg::alloc<WebSocket>(kj::mv(webSocket)),
-              Response::BodyEncoding::AUTO, kj::mv(signal)));
+              jsRequest->getResponseBodyEncoding(), kj::mv(signal)));
         }
       }
       KJ_UNREACHABLE;
@@ -2047,7 +2069,7 @@ jsg::Promise<jsg::Ref<Response>> handleHttpResponse(jsg::Lock& js,
 
   auto result = makeHttpResponse(js, jsRequest->getMethodEnum(), kj::mv(urlList),
       response.statusCode, response.statusText, *response.headers, kj::mv(response.body), kj::none,
-      Response::BodyEncoding::AUTO, kj::mv(signal));
+      jsRequest->getResponseBodyEncoding(), kj::mv(signal));
 
   return js.resolvedPromise(kj::mv(result));
 }
@@ -2193,7 +2215,7 @@ jsg::Ref<Response> makeHttpResponse(jsg::Lock& js,
 
   // TODO(someday): Fill response CF blob from somewhere?
   return jsg::alloc<Response>(js, statusCode, kj::str(statusText), kj::mv(responseHeaders), nullptr,
-      kj::mv(responseBody), kj::mv(urlList), kj::mv(webSocket));
+      kj::mv(responseBody), kj::mv(urlList), kj::mv(webSocket), bodyEncoding);
 }
 
 namespace {
diff --git a/src/workerd/api/http.h b/src/workerd/api/http.h
@@ -414,6 +414,12 @@ class Body: public jsg::Object {
   }
 };
 
+// Controls how response bodies are encoded/decoded according to Content-Encoding headers
+enum class Response_BodyEncoding {
+  AUTO,    // Automatically encode/decode based on Content-Encoding headers
+  MANUAL   // Treat Content-Encoding headers as opaque (no automatic encoding/decoding)
+};
+
 class Request;
 class Response;
 struct RequestInitializerDict;
@@ -707,6 +713,11 @@ struct RequestInitializerDict {
   // `null`.
   jsg::Optional<kj::Maybe<jsg::Ref<AbortSignal>>> signal;
 
+  // Controls whether the response body is automatically decoded according to Content-Encoding
+  // headers. Default behavior is "automatic" which means bodies are decoded. Setting this to
+  // "manual" means the raw compressed bytes are returned.
+  jsg::Optional<kj::String> encodeResponseBody;
+
   // The duplex option controls whether or not a fetch is expected to send the entire request
   // before processing the response. The default value ("half"), which is currently the only
   // option supported by the standard, dictates that the request is fully sent before handling
@@ -724,7 +735,7 @@ struct RequestInitializerDict {
   // jsg::Optional<kj::String> priority;
   // TODO(conform): Might support later?
 
-  JSG_STRUCT(method, headers, body, redirect, fetcher, cf, cache, integrity, signal);
+  JSG_STRUCT(method, headers, body, redirect, fetcher, cf, cache, integrity, signal, encodeResponseBody);
   JSG_STRUCT_TS_OVERRIDE_DYNAMIC(CompatibilityFlags::Reader flags) {
     if(flags.getCacheOptionEnabled()) {
       if(flags.getCacheNoCache()) {
@@ -733,13 +744,15 @@ struct RequestInitializerDict {
           body?: BodyInit | null;
           cache?: 'no-store' | 'no-cache';
           cf?: Cf;
+          encodeResponseBody?: "automatic" | "manual";
         });
       } else {
         JSG_TS_OVERRIDE(RequestInit<Cf = CfProperties> {
           headers?: HeadersInit;
           body?: BodyInit | null;
           cache?: 'no-store';
           cf?: Cf;
+          encodeResponseBody?: "automatic" | "manual";
         });
       }
     } else {
@@ -748,6 +761,7 @@ struct RequestInitializerDict {
         body?: BodyInit | null;
         cache?: never;
         cf?: Cf;
+        encodeResponseBody?: "automatic" | "manual";
       });
     }
   }
@@ -777,10 +791,11 @@ class Request final: public Body {
   Request(kj::HttpMethod method, kj::StringPtr url, Redirect redirect,
           jsg::Ref<Headers> headers, kj::Maybe<jsg::Ref<Fetcher>> fetcher,
           kj::Maybe<jsg::Ref<AbortSignal>> signal, CfProperty&& cf,
-          kj::Maybe<Body::ExtractedBody> body, CacheMode cacheMode = CacheMode::NONE)
+          kj::Maybe<Body::ExtractedBody> body, CacheMode cacheMode = CacheMode::NONE,
+          Response_BodyEncoding responseBodyEncoding = Response_BodyEncoding::AUTO)
     : Body(kj::mv(body), *headers), method(method), url(kj::str(url)),
       redirect(redirect), headers(kj::mv(headers)), fetcher(kj::mv(fetcher)),
-      cacheMode(cacheMode), cf(kj::mv(cf)) {
+      cacheMode(cacheMode), cf(kj::mv(cf)), responseBodyEncoding(responseBodyEncoding) {
     KJ_IF_SOME(s, signal) {
       // If the AbortSignal will never abort, assigning it to thisSignal instead ensures
       // that the cancel machinery is not used but the request.signal accessor will still
@@ -883,6 +898,9 @@ class Request final: public Body {
   // the default value should be an empty string. When the Request object is
   // created we verify that the given value is undefined or empty.
   kj::String getIntegrity() { return kj::String(); }
+  
+  // Get the response body encoding setting for this request  
+  Response_BodyEncoding getResponseBodyEncoding() { return responseBodyEncoding; }
 
   JSG_RESOURCE_TYPE(Request, CompatibilityFlags::Reader flags) {
     JSG_INHERIT(Body);
@@ -993,6 +1011,9 @@ class Request final: public Body {
   kj::Maybe<jsg::Ref<AbortSignal>> thisSignal;
 
   CfProperty cf;
+  
+  // Controls how to handle Content-Encoding headers in the response
+  Response_BodyEncoding responseBodyEncoding = Response_BodyEncoding::AUTO;
 
   void visitForGc(jsg::GcVisitor& visitor) {
     visitor.visit(headers, fetcher, signal, thisSignal, cf);
@@ -1001,16 +1022,14 @@ class Request final: public Body {
 
 class Response final: public Body {
 public:
-  enum class BodyEncoding {
-    AUTO,
-    MANUAL
-  };
+  // Alias to the global Response_BodyEncoding enum for backward compatibility
+  using BodyEncoding = Response_BodyEncoding;
 
   Response(jsg::Lock& js, int statusCode, kj::String statusText, jsg::Ref<Headers> headers,
            CfProperty&& cf, kj::Maybe<Body::ExtractedBody> body,
            kj::Array<kj::String> urlList = {},
            kj::Maybe<jsg::Ref<WebSocket>> webSocket = kj::none,
-           Response::BodyEncoding bodyEncoding = Response::BodyEncoding::AUTO);
+           BodyEncoding bodyEncoding = BodyEncoding::AUTO);
 
   // ---------------------------------------------------------------------------
   // JS API
diff --git a/src/workerd/server/server-test.c++ b/src/workerd/server/server-test.c++
@@ -4092,5 +4092,137 @@ KJ_TEST("Server: ctx.exports self-referential bindings") {
 // TODO(beta): Test TLS (send and receive)
 // TODO(beta): Test CLI overrides
 
+KJ_TEST("Server: encodeResponseBody: manual option") {
+  TestServer test(R"((
+    services = [
+      ( name = "hello",
+        worker = (
+          compatibilityDate = "2022-08-17",
+          modules = [
+            ( name = "main.js",
+              esModule =
+                `export default {
+                `  async fetch(request, env) {
+                `    // Make a subrequest with encodeResponseBody: "manual"
+                `    let response = await fetch("http://subhost/foo", {
+                `      encodeResponseBody: "manual"
+                `    });
+                `    
+                `    // Get the raw bytes, which should not be decompressed
+                `    let rawBytes = await response.arrayBuffer();
+                `    let decoder = new TextDecoder();
+                `    let rawText = decoder.decode(rawBytes);
+                `    
+                `    return new Response(
+                `      "Content-Encoding: " + response.headers.get("Content-Encoding") + "\n" +
+                `      "Raw content: " + rawText
+                `    );
+                `  }
+                `}
+            )
+          ]
+        )
+      )
+    ],
+    sockets = [
+      ( name = "main",
+        address = "test-addr",
+        service = "hello"
+      )
+    ]
+  ))"_kj);
+
+  test.start();
+  auto conn = test.connect("test-addr");
+  conn.sendHttpGet("/");
+
+  auto subreq = test.receiveInternetSubrequest("subhost");
+  subreq.recv(R"(
+    GET /foo HTTP/1.1
+    Host: subhost
+
+  )"_blockquote);
+
+  // Send a response with Content-Encoding: gzip, but the body is not actually
+  // compressed - it's just "fake-gzipped-content" as plain text
+  subreq.send(R"(
+    HTTP/1.1 200 OK
+    Content-Length: 20
+    Content-Encoding: gzip
+
+    fake-gzipped-content
+  )"_blockquote);
+
+  // Verify that:
+  // 1. The Content-Encoding header was preserved
+  // 2. The body was not decompressed (we get the raw "fake-gzipped-content")
+  conn.recvHttp200(R"(
+    Content-Encoding: gzip
+    Raw content: fake-gzipped-content)"_blockquote);
+}
+
+KJ_TEST("Server: encodeResponseBody: manual pass-through") {
+  TestServer test(R"((
+    services = [
+      ( name = "hello",
+        worker = (
+          compatibilityDate = "2022-08-17",
+          modules = [
+            ( name = "main.js",
+              esModule =
+                `export default {
+                `  async fetch(request, env) {
+                `    // Make a subrequest with encodeResponseBody: "manual" and pass through the response
+                `    return fetch("http://subhost/foo", {
+                `      encodeResponseBody: "manual"
+                `    });
+                `  }
+                `}
+            )
+          ]
+        )
+      )
+    ],
+    sockets = [
+      ( name = "main",
+        address = "test-addr",
+        service = "hello"
+      )
+    ]
+  ))"_kj);
+
+  test.start();
+  auto conn = test.connect("test-addr");
+  conn.sendHttpGet("/");
+
+  auto subreq = test.receiveInternetSubrequest("subhost");
+  subreq.recv(R"(
+    GET /foo HTTP/1.1
+    Host: subhost
+
+  )"_blockquote);
+
+  // Send a response with Content-Encoding: gzip, but the body is not actually
+  // compressed - it's just "fake-gzipped-content" as plain text
+  subreq.send(R"(
+    HTTP/1.1 200 OK
+    Content-Length: 20
+    Content-Encoding: gzip
+
+    fake-gzipped-content
+  )"_blockquote);
+
+  // Verify that the response is passed through verbatim, with:
+  // 1. The Content-Encoding header preserved
+  // 2. The body not decompressed
+  // 3. The body not re-encoded
+  conn.recv(R"(
+    HTTP/1.1 200 OK
+    Content-Length: 20
+    Content-Encoding: gzip
+
+    fake-gzipped-content)"_blockquote);
+}
+
 }  // namespace
 }  // namespace workerd::server
