fix: don't permamently cache fallback html

Author: pieh

Diff for: src/build/content/prerendered.ts

@@ -9,6 +9,8 @@ import { glob } from 'fast-glob'
 import pLimit from 'p-limit'
 import { satisfies } from 'semver'
 
+import { FS_BLOBS_MANIFEST } from '../../run/constants.js'
+import { type FSBlobsManifest } from '../../run/next.cjs'
 import { encodeBlobKey } from '../../shared/blobkey.js'
 import type {
   CachedFetchValueForMultipleVersions,
@@ -158,6 +160,11 @@ export const copyPrerenderedContent = async (ctx: PluginContext): Promise<void>
           })
         : false
 
+      const fsBlobsManifest: FSBlobsManifest = {
+        fallbackPaths: [],
+        outputRoot: ctx.distDir,
+      }
+
       await Promise.all([
         ...Object.entries(manifest.routes).map(
           ([route, meta]): Promise<void> =>
@@ -237,6 +244,8 @@ export const copyPrerenderedContent = async (ctx: PluginContext): Promise<void>
                 }
 
                 await writeCacheEntry(key, value, lastModified, ctx)
+
+                fsBlobsManifest.fallbackPaths.push(`${key}.html`)
               }
             })
           }
@@ -254,6 +263,10 @@ export const copyPrerenderedContent = async (ctx: PluginContext): Promise<void>
         )
         await writeCacheEntry(key, value, lastModified, ctx)
       }
+      await writeFile(
+        join(ctx.serverHandlerDir, FS_BLOBS_MANIFEST),
+        JSON.stringify(fsBlobsManifest),
+      )
     } catch (error) {
       ctx.failBuild('Failed assembling prerendered content for upload', error)
     }

Diff for: src/run/constants.ts

@@ -5,3 +5,4 @@ export const MODULE_DIR = fileURLToPath(new URL('.', import.meta.url))
 export const PLUGIN_DIR = resolve(`${MODULE_DIR}../../..`)
 // a file where we store the required-server-files config object in to access during runtime
 export const RUN_CONFIG = 'run-config.json'
+export const FS_BLOBS_MANIFEST = 'fs-blobs-manifest.json'

Diff for: src/run/handlers/request-context.cts

@@ -11,7 +11,7 @@ export type RequestContext = {
   responseCacheGetLastModified?: number
   responseCacheKey?: string
   responseCacheTags?: string[]
-  usedFsRead?: boolean
+  usedFsReadForNonFallback?: boolean
   didPagesRouterOnDemandRevalidate?: boolean
   serverTiming?: string
   routeHandlerRevalidate?: NetlifyCachedRouteValue['revalidate']

Diff for: src/run/headers.test.ts

@@ -321,7 +321,7 @@ describe('headers', () => {
       })
     })
 
-    test('should not set any headers if "cache-control" is not set and "requestContext.usedFsRead" is not truthy', () => {
+    test('should not set any headers if "cache-control" is not set and "requestContext.usedFsReadForNonFallback" is not truthy', () => {
       const headers = new Headers()
       const request = new Request(defaultUrl)
       vi.spyOn(headers, 'set')
@@ -331,15 +331,15 @@ describe('headers', () => {
       expect(headers.set).toHaveBeenCalledTimes(0)
     })
 
-    test('should set permanent, durable "netlify-cdn-cache-control" if "cache-control" is not set and "requestContext.usedFsRead" is truthy', () => {
+    test('should set permanent, durable "netlify-cdn-cache-control" if "cache-control" is not set and "requestContext.usedFsReadForNonFallback" is truthy', () => {
       const headers = new Headers()
       const request = new Request(defaultUrl)
       vi.spyOn(headers, 'set')
 
       const requestContext = createRequestContext()
-      requestContext.usedFsRead = true
+      requestContext.usedFsReadForNonFallback = true
 
-      setCacheControlHeaders(headers, request, requestContext, true)
+      setCacheControlHeaders(headers, request, requestContext)
 
       expect(headers.set).toHaveBeenNthCalledWith(
         1,

Diff for: src/run/headers.ts

@@ -263,7 +263,7 @@ export const setCacheControlHeaders = (
     cacheControl === null &&
     !headers.has('cdn-cache-control') &&
     !headers.has('netlify-cdn-cache-control') &&
-    requestContext.usedFsRead
+    requestContext.usedFsReadForNonFallback
   ) {
     // handle CDN Cache Control on static files
     headers.set('cache-control', 'public, max-age=0, must-revalidate')
@@ -272,7 +272,10 @@ export const setCacheControlHeaders = (
 }
 
 export const setCacheTagsHeaders = (headers: Headers, requestContext: RequestContext) => {
-  if (requestContext.responseCacheTags) {
+  if (
+    requestContext.responseCacheTags &&
+    (headers.has('cache-control') || headers.has('netlify-cdn-cache-control'))
+  ) {
     headers.set('netlify-cache-tag', requestContext.responseCacheTags.join(','))
   }
 }

Diff for: src/run/next.cts

@@ -1,5 +1,5 @@
-import fs from 'fs/promises'
-import { relative, resolve } from 'path'
+import fs, { readFile } from 'fs/promises'
+import { join, relative, resolve } from 'path'
 
 // @ts-expect-error no types installed
 import { patchFs } from 'fs-monkey'
@@ -80,6 +80,27 @@ console.timeEnd('import next server')
 
 type FS = typeof import('fs')
 
+export type FSBlobsManifest = {
+  fallbackPaths: string[]
+  outputRoot: string
+}
+
+function normalizeStaticAssetPath(path: string) {
+  return path.startsWith('/') ? path : `/${path}`
+}
+
+let fsBlobsManifestPromise: Promise<FSBlobsManifest> | undefined
+const getFSBlobsManifest = (): Promise<FSBlobsManifest> => {
+  if (!fsBlobsManifestPromise) {
+    fsBlobsManifestPromise = (async () => {
+      const { FS_BLOBS_MANIFEST, PLUGIN_DIR } = await import('./constants.js')
+      return JSON.parse(await readFile(resolve(PLUGIN_DIR, FS_BLOBS_MANIFEST), 'utf-8'))
+    })()
+  }
+
+  return fsBlobsManifestPromise
+}
+
 export async function getMockedRequestHandlers(...args: Parameters<typeof getRequestHandlers>) {
   const tracer = getTracer()
   return tracer.withActiveSpan('mocked request handler', async () => {
@@ -96,13 +117,17 @@ export async function getMockedRequestHandlers(...args: Parameters<typeof getReq
       } catch (error) {
         // only try to get .html files from the blob store
         if (typeof path === 'string' && path.endsWith('.html')) {
+          const fsBlobsManifest = await getFSBlobsManifest()
+
           const store = getRegionalBlobStore()
-          const relPath = relative(resolve('.next/server/pages'), path)
+          const relPath = relative(resolve(join(fsBlobsManifest.outputRoot, '/server/pages')), path)
           const file = await store.get(await encodeBlobKey(relPath))
           if (file !== null) {
-            const requestContext = getRequestContext()
-            if (requestContext) {
-              requestContext.usedFsRead = true
+            if (!fsBlobsManifest.fallbackPaths.includes(normalizeStaticAssetPath(relPath))) {
+              const requestContext = getRequestContext()
+              if (requestContext) {
+                requestContext.usedFsReadForNonFallback = true
+              }
             }
 
             return file