fix: use version of htmlrewriter which does not make use of asyncify, which looks to have a potential memory leak under high load (#2721)

* fix: use version of htmlrewriter which does not make use of asyncify, which looks to have a potential memory leak under high load

we noticed the memory issue with Netlify's CSP plugin which used the same htmlrewriter library. We've built a new htmlrewriter library which uses the latest version of lol-html and removes the ability to use async-handlers, which is what required asyncify to be included.

* chore: vendor updated htmlrewriter

* fix: update remaining htmlrewriter import

* fix: workaround deno vendor limitation (not pulling static wasm files)

* fix: inline htmlrewriter wasm blob to workaround bundling problems

* Update tools/build.js

Co-authored-by: Philippe Serhal <[email protected]>

---------

Co-authored-by: jake champion <[email protected]>
Co-authored-by: Philippe Serhal <[email protected]>

Author: 3 people

deno.json

@@ -6,5 +6,6 @@
   },
   "imports": {
     "@netlify/edge-functions": "https://edge.netlify.com/v1/index.ts"
-  }
+  },
+  "importMap": "./edge-runtime/vendor/import_map.json"
 }

edge-runtime/lib/middleware.ts

@@ -1,6 +1,6 @@
 import type { Context } from '@netlify/edge-functions'
 
-import { ElementHandlers } from '../vendor/deno.land/x/[email protected]/index.ts'
+import type { ElementHandlers } from '../vendor/deno.land/x/[email protected]/src/index.ts'
 
 type NextDataTransform = <T>(data: T) => T
 

edge-runtime/lib/response.ts

@@ -1,5 +1,8 @@
 import type { Context } from '@netlify/edge-functions'
-import { HTMLRewriter } from '../vendor/deno.land/x/[email protected]/index.ts'
+import {
+  HTMLRewriter,
+  type TextChunk,
+} from '../vendor/deno.land/x/[email protected]/src/index.ts'
 
 import { updateModifiedHeaders } from './headers.ts'
 import type { StructuredLogger } from './logging.ts'
@@ -79,7 +82,7 @@ export const buildResponse = async ({
 
     if (response.dataTransforms.length > 0) {
       rewriter.on('script[id="__NEXT_DATA__"]', {
-        text(textChunk) {
+        text(textChunk: TextChunk) {
           // Grab all the chunks in the Next data script tag
           buffer += textChunk.text
           if (textChunk.lastInTextNode) {

edge-runtime/vendor.ts

@@ -12,6 +12,6 @@ import 'https://deno.land/[email protected]/node/util.ts'
 import 'https://deno.land/[email protected]/path/mod.ts'
 
 import 'https://deno.land/x/[email protected]/index.ts'
-import 'https://deno.land/x/[email protected]/index.ts'
+import 'https://deno.land/x/[email protected]/src/index.ts'
 
 import 'https://v1-7-0--edge-utils.netlify.app/logger/mod.ts'

src/build/functions/edge.ts

@@ -85,13 +85,27 @@ const writeHandlerFile = async (ctx: PluginContext, { matchers, name }: NextDefi
     JSON.stringify(minimalNextConfig),
   )
 
+  const htmlRewriterWasm = await readFile(
+    join(
+      ctx.pluginDir,
+      'edge-runtime/vendor/deno.land/x/[email protected]/pkg/htmlrewriter_bg.wasm',
+    ),
+  )
+
   // Writing the function entry file. It wraps the middleware code with the
   // compatibility layer mentioned above.
   await writeFile(
     join(handlerDirectory, `${handlerName}.js`),
     `
+    import { decode as _base64Decode } from './edge-runtime/vendor/deno.land/[email protected]/encoding/base64.ts';
+    import { init as htmlRewriterInit } from './edge-runtime/vendor/deno.land/x/[email protected]/src/index.ts'
     import {handleMiddleware} from './edge-runtime/middleware.ts';
     import handler from './server/${name}.js';
+
+    await htmlRewriterInit({ module_or_path: _base64Decode(${JSON.stringify(
+      htmlRewriterWasm.toString('base64'),
+    )}).buffer });
+
     export default (req, context) => handleMiddleware(req, context, handler);
     `,
   )

tools/build.js

@@ -1,5 +1,8 @@
+import { createWriteStream } from 'node:fs'
 import { cp, rm } from 'node:fs/promises'
-import { resolve, join } from 'node:path'
+import { join, resolve } from 'node:path'
+import { Readable } from 'stream'
+import { finished } from 'stream/promises'
 
 import { build, context } from 'esbuild'
 import { execaCommand } from 'execa'
@@ -94,6 +97,23 @@ async function vendorDeno() {
   console.log(`📦 Vendoring Deno modules into '${vendorDest}'...`)
 
   await execaCommand(`deno vendor ${vendorSource} --output=${vendorDest} --force`)
+
+  // htmlrewriter contains wasm files and those don't currently work great with vendoring
+  // see https://github.com/denoland/deno/issues/14123
+  // to workaround this we copy the wasm files manually
+  const filesToDownload = ['https://deno.land/x/[email protected]/pkg/htmlrewriter_bg.wasm']
+  await Promise.all(
+    filesToDownload.map(async (urlString) => {
+      const url = new URL(urlString)
+
+      const destination = join(vendorDest, url.hostname, url.pathname)
+
+      const res = await fetch(url)
+      if (!res.ok) throw new Error('Failed to fetch .wasm file to vendor', { cause: err })
+      const fileStream = createWriteStream(destination, { flags: 'wx' })
+      await finished(Readable.fromWeb(res.body).pipe(fileStream))
+    }),
+  )
 }
 
 const args = new Set(process.argv.slice(2))