From 0ef0b0e9a7ba8c92c8d3eda1b672c1a2fec2e341 Mon Sep 17 00:00:00 2001 From: jake champion Date: Fri, 22 Nov 2024 22:21:36 +0000 Subject: [PATCH 1/6] fix: use version of htmlrewriter which does not make use of asyncify, which looks to have a potential memory leak under high load we noticed the memory issue with Netlify's CSP plugin which used the same htmlrewriter library. We've built a new htmlrewriter library which uses the latest version of lol-html and removes the ability to use async-handlers, which is what required asyncify to be included. --- deno.json | 3 ++- edge-runtime/lib/response.ts | 6 ++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/deno.json b/deno.json index 28e8ea6c3c..561c910ed5 100644 --- a/deno.json +++ b/deno.json @@ -6,5 +6,6 @@ }, "imports": { "@netlify/edge-functions": "https://edge.netlify.com/v1/index.ts" - } + }, + "importMap": "./edge-runtime/vendor/import_map.json" } diff --git a/edge-runtime/lib/response.ts b/edge-runtime/lib/response.ts index 3fd82c5ab0..8c5ddea80e 100644 --- a/edge-runtime/lib/response.ts +++ b/edge-runtime/lib/response.ts @@ -1,5 +1,5 @@ import type { Context } from '@netlify/edge-functions' -import { HTMLRewriter } from '../vendor/deno.land/x/html_rewriter@v0.1.0-pre.17/index.ts' +import { HTMLRewriter, init, type TextChunk } from 'https://deno.land/x/htmlrewriter@v1.0.0/src/index.ts' import { updateModifiedHeaders } from './headers.ts' import type { StructuredLogger } from './logging.ts' @@ -13,6 +13,8 @@ import { relativizeURL, } from './util.ts' +await init(); + export interface FetchEventResult { response: Response waitUntil: Promise @@ -79,7 +81,7 @@ export const buildResponse = async ({ if (response.dataTransforms.length > 0) { rewriter.on('script[id="__NEXT_DATA__"]', { - text(textChunk) { + text(textChunk: TextChunk) { // Grab all the chunks in the Next data script tag buffer += textChunk.text if (textChunk.lastInTextNode) { From 8526938301648f8923aabbe1eb4c9360b384e092 Mon Sep 17 00:00:00 2001 From: pieh Date: Mon, 9 Dec 2024 16:39:02 +0100 Subject: [PATCH 2/6] chore: vendor updated htmlrewriter --- edge-runtime/lib/response.ts | 8 ++++++-- edge-runtime/vendor.ts | 2 +- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/edge-runtime/lib/response.ts b/edge-runtime/lib/response.ts index 8c5ddea80e..b09ea896a9 100644 --- a/edge-runtime/lib/response.ts +++ b/edge-runtime/lib/response.ts @@ -1,5 +1,9 @@ import type { Context } from '@netlify/edge-functions' -import { HTMLRewriter, init, type TextChunk } from 'https://deno.land/x/htmlrewriter@v1.0.0/src/index.ts' +import { + HTMLRewriter, + init, + type TextChunk, +} from '../vendor/deno.land/x/htmlrewriter@v1.0.0/src/index.ts' import { updateModifiedHeaders } from './headers.ts' import type { StructuredLogger } from './logging.ts' @@ -13,7 +17,7 @@ import { relativizeURL, } from './util.ts' -await init(); +await init() export interface FetchEventResult { response: Response diff --git a/edge-runtime/vendor.ts b/edge-runtime/vendor.ts index e8c6720d4d..df153912c3 100644 --- a/edge-runtime/vendor.ts +++ b/edge-runtime/vendor.ts @@ -12,6 +12,6 @@ import 'https://deno.land/std@0.175.0/node/util.ts' import 'https://deno.land/std@0.175.0/path/mod.ts' import 'https://deno.land/x/path_to_regexp@v6.2.1/index.ts' -import 'https://deno.land/x/html_rewriter@v0.1.0-pre.17/index.ts' +import 'https://deno.land/x/htmlrewriter@v1.0.0/src/index.ts' import 'https://v1-7-0--edge-utils.netlify.app/logger/mod.ts' From a860d2b0c6530e7301a80e162fa1ea1bf7c57abe Mon Sep 17 00:00:00 2001 From: pieh Date: Mon, 9 Dec 2024 17:00:02 +0100 Subject: [PATCH 3/6] fix: update remaining htmlrewriter import --- edge-runtime/lib/middleware.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/edge-runtime/lib/middleware.ts b/edge-runtime/lib/middleware.ts index 46bbef6888..2ae25c1d34 100644 --- a/edge-runtime/lib/middleware.ts +++ b/edge-runtime/lib/middleware.ts @@ -1,6 +1,6 @@ import type { Context } from '@netlify/edge-functions' -import { ElementHandlers } from '../vendor/deno.land/x/html_rewriter@v0.1.0-pre.17/index.ts' +import type { ElementHandlers } from '../vendor/deno.land/x/htmlrewriter@v1.0.0/src/index.ts' type NextDataTransform = (data: T) => T From a804602c53570fa1e562f90c1cfe6ee4d7300b16 Mon Sep 17 00:00:00 2001 From: pieh Date: Tue, 17 Dec 2024 19:31:27 +0100 Subject: [PATCH 4/6] fix: workaround deno vendor limitation (not pulling static wasm files) --- tools/build.js | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/tools/build.js b/tools/build.js index 5db351d152..4dd84ea492 100644 --- a/tools/build.js +++ b/tools/build.js @@ -1,5 +1,8 @@ +import { createWriteStream } from 'node:fs' import { cp, rm } from 'node:fs/promises' -import { resolve, join } from 'node:path' +import { join, resolve } from 'node:path' +import { Readable } from 'stream' +import { finished } from 'stream/promises' import { build, context } from 'esbuild' import { execaCommand } from 'execa' @@ -94,6 +97,22 @@ async function vendorDeno() { console.log(`📦 Vendoring Deno modules into '${vendorDest}'...`) await execaCommand(`deno vendor ${vendorSource} --output=${vendorDest} --force`) + + // htmlrewriter contains wasm files and those don't currently work great with vendoring + // see https://github.com/denoland/deno/issues/14123 + // to workaround this we copy the wasm files manually + const filesToDownload = ['https://deno.land/x/htmlrewriter@v1.0.0/pkg/htmlrewriter_bg.wasm'] + await Promise.all( + filesToDownload.map(async (urlString) => { + const url = new URL(urlString) + + const destination = join(vendorDest, url.hostname, url.pathname) + + const res = await fetch(url) + const fileStream = createWriteStream(destination, { flags: 'wx' }) + await finished(Readable.fromWeb(res.body).pipe(fileStream)) + }), + ) } const args = new Set(process.argv.slice(2)) From f2462476f8684c58ca3f7e04f01c67cedad7b04c Mon Sep 17 00:00:00 2001 From: pieh Date: Tue, 17 Dec 2024 20:09:15 +0100 Subject: [PATCH 5/6] fix: inline htmlrewriter wasm blob to workaround bundling problems --- edge-runtime/lib/response.ts | 3 --- src/build/functions/edge.ts | 14 ++++++++++++++ 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/edge-runtime/lib/response.ts b/edge-runtime/lib/response.ts index b09ea896a9..8faf0d906a 100644 --- a/edge-runtime/lib/response.ts +++ b/edge-runtime/lib/response.ts @@ -1,7 +1,6 @@ import type { Context } from '@netlify/edge-functions' import { HTMLRewriter, - init, type TextChunk, } from '../vendor/deno.land/x/htmlrewriter@v1.0.0/src/index.ts' @@ -17,8 +16,6 @@ import { relativizeURL, } from './util.ts' -await init() - export interface FetchEventResult { response: Response waitUntil: Promise diff --git a/src/build/functions/edge.ts b/src/build/functions/edge.ts index f6d331ddb6..0bb21e5471 100644 --- a/src/build/functions/edge.ts +++ b/src/build/functions/edge.ts @@ -85,13 +85,27 @@ const writeHandlerFile = async (ctx: PluginContext, { matchers, name }: NextDefi JSON.stringify(minimalNextConfig), ) + const htmlRewriterWasm = await readFile( + join( + ctx.pluginDir, + 'edge-runtime/vendor/deno.land/x/htmlrewriter@v1.0.0/pkg/htmlrewriter_bg.wasm', + ), + ) + // Writing the function entry file. It wraps the middleware code with the // compatibility layer mentioned above. await writeFile( join(handlerDirectory, `${handlerName}.js`), ` + import { decode as _base64Decode } from './edge-runtime/vendor/deno.land/std@0.175.0/encoding/base64.ts'; + import { init as htmlRewriterInit } from './edge-runtime/vendor/deno.land/x/htmlrewriter@v1.0.0/src/index.ts' import {handleMiddleware} from './edge-runtime/middleware.ts'; import handler from './server/${name}.js'; + + await htmlRewriterInit({ module_or_path: _base64Decode(${JSON.stringify( + htmlRewriterWasm.toString('base64'), + )}).buffer }); + export default (req, context) => handleMiddleware(req, context, handler); `, ) From fdb11ed10ac52929af99f52b22ec7160eda3a6af Mon Sep 17 00:00:00 2001 From: Michal Piechowiak Date: Tue, 17 Dec 2024 22:50:01 +0100 Subject: [PATCH 6/6] Update tools/build.js Co-authored-by: Philippe Serhal --- tools/build.js | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/build.js b/tools/build.js index 4dd84ea492..301304f5de 100644 --- a/tools/build.js +++ b/tools/build.js @@ -109,6 +109,7 @@ async function vendorDeno() { const destination = join(vendorDest, url.hostname, url.pathname) const res = await fetch(url) + if (!res.ok) throw new Error('Failed to fetch .wasm file to vendor', { cause: err }) const fileStream = createWriteStream(destination, { flags: 'wx' }) await finished(Readable.fromWeb(res.body).pipe(fileStream)) }),