Skip to content

Commit a0c93ca

Browse files
orinokaiorinokai
authored
feat: write tags manifest at build time to simplify request time response tagging (#94)
* feat: initial tags manifest wip * feat: augment prerender manifest * feat: refactor code * fix: tidy up * feat: async copying again * chore: consistency * chore: error check * chore: update test name for clarity * fix: move tag manifest * chore: remove unnecessary cache tag tests * fix: update page router cache tag prefix
1 parent 90bb11b commit a0c93ca

File tree

14 files changed

+132
-197
lines changed

14 files changed

+132
-197
lines changed

‎.github/workflows/test-e2e.yml

+2-2
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
name: run e2e tests
1+
name: Run next.js tests
22

33
on:
44
workflow_dispatch:
@@ -23,7 +23,7 @@ env:
2323
NEXT_TEST_CONTINUE_ON_ERROR: 1
2424

2525
jobs:
26-
test-e2e:
26+
e2e:
2727
if:
2828
${{ github.event_name == 'workflow_dispatch' ||
2929
contains(github.event.pull_request.labels.*.name, 'run-e2e-tests') }}

‎src/build/constants.ts

+2-2
Original file line numberDiff line numberDiff line change
@@ -14,9 +14,9 @@ export const STATIC_DIR = '.netlify/static'
1414
export const TEMP_DIR = '.netlify/temp'
1515

1616
export const SERVER_FUNCTIONS_DIR = '.netlify/functions-internal'
17-
export const SERVER_HANDLER_NAME = '_netlify-server-handler'
17+
export const SERVER_HANDLER_NAME = '___netlify-server-handler'
1818
export const SERVER_HANDLER_DIR = join(SERVER_FUNCTIONS_DIR, SERVER_HANDLER_NAME)
1919

2020
export const EDGE_FUNCTIONS_DIR = '.netlify/edge-functions'
21-
export const EDGE_HANDLER_NAME = '_netlify-edge-handler'
21+
export const EDGE_HANDLER_NAME = '___netlify-edge-handler'
2222
export const EDGE_HANDLER_DIR = join(EDGE_FUNCTIONS_DIR, EDGE_HANDLER_NAME)

‎src/build/content/server.ts

+59-4
Original file line numberDiff line numberDiff line change
@@ -1,30 +1,85 @@
1+
import { NetlifyPluginOptions } from '@netlify/build'
12
import glob from 'fast-glob'
2-
import { cp } from 'node:fs/promises'
3-
import { join } from 'node:path'
3+
import { cp, readFile, writeFile } from 'node:fs/promises'
4+
import { join, resolve } from 'node:path'
5+
import { getPrerenderManifest } from '../config.js'
6+
import { SERVER_HANDLER_DIR } from '../constants.js'
47

58
/**
69
* Copy App/Pages Router Javascript needed by the server handler
710
*/
8-
export const copyServerContent = async (src: string, dest: string): Promise<void> => {
11+
export const copyNextServerCode = async ({
12+
constants: { PUBLISH_DIR },
13+
}: Pick<NetlifyPluginOptions, 'constants'>): Promise<void> => {
14+
const src = resolve(PUBLISH_DIR, 'standalone/.next')
15+
const dest = resolve(SERVER_HANDLER_DIR, '.next')
16+
917
const paths = await glob([`*`, `server/*`, `server/chunks/*`, `server/+(app|pages)/**/*.js`], {
1018
cwd: src,
1119
extglob: true,
1220
})
21+
1322
await Promise.all(
1423
paths.map(async (path: string) => {
1524
await cp(join(src, path), join(dest, path), { recursive: true })
1625
}),
1726
)
1827
}
1928

20-
export const copyServerDependencies = async (src: string, dest: string): Promise<void> => {
29+
export const copyNextDependencies = async ({
30+
constants: { PUBLISH_DIR },
31+
}: Pick<NetlifyPluginOptions, 'constants'>): Promise<void> => {
32+
const src = resolve(PUBLISH_DIR, 'standalone/node_modules')
33+
const dest = resolve(SERVER_HANDLER_DIR, 'node_modules')
34+
2135
const paths = await glob([`**`], {
2236
cwd: src,
2337
extglob: true,
2438
})
39+
2540
await Promise.all(
2641
paths.map(async (path: string) => {
2742
await cp(join(src, path), join(dest, path), { recursive: true })
2843
}),
2944
)
3045
}
46+
47+
export const writeTagsManifest = async ({
48+
constants: { PUBLISH_DIR },
49+
}: Pick<NetlifyPluginOptions, 'constants'>): Promise<void> => {
50+
const manifest = await getPrerenderManifest({ PUBLISH_DIR })
51+
52+
const routes = Object.entries(manifest.routes).map(async ([route, definition]) => {
53+
let tags
54+
55+
// app router
56+
if (definition.dataRoute?.endsWith('.rsc')) {
57+
const path = resolve(PUBLISH_DIR, `server/app/${route === '/' ? '/index' : route}.meta`)
58+
try {
59+
const file = await readFile(path, 'utf-8')
60+
const meta = JSON.parse(file)
61+
tags = meta.headers['x-next-cache-tags']
62+
} catch (error) {
63+
console.log(`Unable to read cache tags for: ${path}`)
64+
}
65+
}
66+
67+
// pages router
68+
if (definition.dataRoute?.endsWith('.json')) {
69+
tags = `_N_T_${route}`
70+
}
71+
72+
// route handler
73+
if (definition.dataRoute === null) {
74+
tags = definition.initialHeaders?.['x-next-cache-tags']
75+
}
76+
77+
return [route, tags]
78+
})
79+
80+
await writeFile(
81+
resolve(resolve(SERVER_HANDLER_DIR, '.netlify/tags-manifest.json')),
82+
JSON.stringify(Object.fromEntries(await Promise.all(routes))),
83+
'utf-8',
84+
)
85+
}

‎src/build/functions/server.ts

+33-28
Original file line numberDiff line numberDiff line change
@@ -1,25 +1,18 @@
11
import { NetlifyPluginOptions } from '@netlify/build'
22
import { nodeFileTrace } from '@vercel/nft'
3-
import { cp, rm, writeFile } from 'fs/promises'
3+
import { cp, mkdir, rm, writeFile } from 'fs/promises'
44
import { basename, join, relative, resolve } from 'node:path'
55
import {
66
PLUGIN_DIR,
77
PLUGIN_NAME,
88
PLUGIN_VERSION,
9+
SERVER_FUNCTIONS_DIR,
910
SERVER_HANDLER_DIR,
1011
SERVER_HANDLER_NAME,
1112
} from '../constants.js'
12-
import { copyServerContent, copyServerDependencies } from '../content/server.js'
13-
14-
/**
15-
* Create a Netlify function to run the Next.js server
16-
*/
17-
export const createServerHandler = async ({
18-
constants: { PUBLISH_DIR },
19-
}: Pick<NetlifyPluginOptions, 'constants'>) => {
20-
// reset the handler directory
21-
await rm(resolve(SERVER_HANDLER_DIR), { recursive: true, force: true })
13+
import { copyNextDependencies, copyNextServerCode, writeTagsManifest } from '../content/server.js'
2214

15+
const copyHandlerDependencies = async () => {
2316
// trace the handler dependencies
2417
const { fileList } = await nodeFileTrace(
2518
[
@@ -52,24 +45,13 @@ export const createServerHandler = async ({
5245
// resolve it with the plugin directory like `<abs-path>/node_modules/@netlify/next-runtime`
5346
// if it is a node_module resolve it with the process working directory.
5447
const relPath = relative(path.includes(PLUGIN_NAME) ? PLUGIN_DIR : cwd, absPath)
55-
await cp(absPath, resolve(SERVER_HANDLER_DIR, relPath), {
56-
recursive: true,
57-
})
48+
await cp(absPath, resolve(SERVER_HANDLER_DIR, relPath), { recursive: true })
5849
}),
5950
)
51+
}
6052

61-
// copy the next.js standalone build output to the handler directory
62-
await copyServerContent(
63-
resolve(PUBLISH_DIR, 'standalone/.next'),
64-
resolve(SERVER_HANDLER_DIR, '.next'),
65-
)
66-
await copyServerDependencies(
67-
resolve(PUBLISH_DIR, 'standalone/node_modules'),
68-
resolve(SERVER_HANDLER_DIR, 'node_modules'),
69-
)
70-
71-
// create the handler metadata file
72-
await writeFile(
53+
const writeHandlerManifest = () => {
54+
return writeFile(
7355
resolve(SERVER_HANDLER_DIR, `${SERVER_HANDLER_NAME}.json`),
7456
JSON.stringify({
7557
config: {
@@ -81,6 +63,7 @@ export const createServerHandler = async ({
8163
'package.json',
8264
'dist/**',
8365
'.next/**',
66+
'.netlify/**',
8467
'node_modules/**',
8568
],
8669
includedFilesBasePath: resolve(SERVER_HANDLER_DIR),
@@ -89,13 +72,35 @@ export const createServerHandler = async ({
8972
}),
9073
'utf-8',
9174
)
75+
}
9276

93-
// configure ESM
77+
const writePackageMetadata = async () => {
9478
await writeFile(resolve(SERVER_HANDLER_DIR, 'package.json'), JSON.stringify({ type: 'module' }))
79+
}
9580

96-
// write the root handler file
81+
const writeHandlerFile = async () => {
9782
await writeFile(
9883
resolve(SERVER_HANDLER_DIR, `${SERVER_HANDLER_NAME}.js`),
9984
`import handler from './dist/run/handlers/server.js';export default handler`,
10085
)
10186
}
87+
88+
/**
89+
* Create a Netlify function to run the Next.js server
90+
*/
91+
export const createServerHandler = async ({
92+
constants,
93+
}: Pick<NetlifyPluginOptions, 'constants'>) => {
94+
await rm(resolve(SERVER_FUNCTIONS_DIR), { recursive: true, force: true })
95+
await mkdir(resolve(SERVER_HANDLER_DIR, '.netlify'), { recursive: true })
96+
97+
await Promise.all([
98+
copyNextServerCode({ constants }),
99+
copyNextDependencies({ constants }),
100+
writeTagsManifest({ constants }),
101+
copyHandlerDependencies(),
102+
writeHandlerManifest(),
103+
writePackageMetadata(),
104+
writeHandlerFile(),
105+
])
106+
}

‎src/index.ts

+3-3
Original file line numberDiff line numberDiff line change
@@ -12,14 +12,14 @@ import { createServerHandler } from './build/functions/server.js'
1212
import { createEdgeHandlers } from './build/functions/edge.js'
1313

1414
export const onPreBuild = async ({ constants, utils }: NetlifyPluginOptions) => {
15-
setPreBuildConfig()
1615
await restoreBuildCache({ constants, utils })
16+
setPreBuildConfig()
1717
}
1818

1919
export const onBuild = async ({ constants, utils }: NetlifyPluginOptions) => {
2020
await saveBuildCache({ constants, utils })
2121

22-
return Promise.all([
22+
await Promise.all([
2323
copyStaticAssets({ constants }),
2424
uploadStaticContent({ constants }),
2525
uploadPrerenderedContent({ constants }),
@@ -29,8 +29,8 @@ export const onBuild = async ({ constants, utils }: NetlifyPluginOptions) => {
2929
}
3030

3131
export const onPostBuild = async ({ constants, netlifyConfig }: NetlifyPluginOptions) => {
32-
setPostBuildConfig({ netlifyConfig })
3332
await publishStaticDir({ constants })
33+
setPostBuildConfig({ netlifyConfig })
3434
}
3535

3636
export const onEnd = async ({ constants }: NetlifyPluginOptions) => {

‎src/run/config.ts

+9-5
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,13 @@
11
import type { NextConfigComplete } from 'next/dist/server/config-shared.js'
22
import { readFile } from 'node:fs/promises'
3-
import { PLUGIN_DIR, RUN_DIR } from './constants.js'
3+
import { resolve } from 'node:path'
4+
import { PLUGIN_DIR } from './constants.js'
45

56
/**
67
* Get Next.js config from the build output
78
*/
89
export const getRunConfig = async () => {
9-
// get config from the build output
10-
const file = await readFile(`${RUN_DIR}/.next/required-server-files.json`, 'utf-8')
11-
const json = JSON.parse(file)
12-
return json.config
10+
return JSON.parse(await readFile(resolve('.next/required-server-files.json'), 'utf-8')).config
1311
}
1412

1513
/**
@@ -25,3 +23,9 @@ export const setRunConfig = (config: NextConfigComplete) => {
2523
// set config
2624
process.env.__NEXT_PRIVATE_STANDALONE_CONFIG = JSON.stringify(config)
2725
}
26+
27+
export type TagsManifest = Record<string, string>
28+
29+
export const getTagsManifest = async (): Promise<TagsManifest> => {
30+
return JSON.parse(await readFile(resolve('.netlify/tags-manifest.json'), 'utf-8'))
31+
}

‎src/run/constants.ts

-3
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,3 @@ import { fileURLToPath } from 'node:url'
33

44
export const MODULE_DIR = fileURLToPath(new URL('.', import.meta.url))
55
export const PLUGIN_DIR = resolve(`${MODULE_DIR}../..`)
6-
export const WORKING_DIR = process.cwd()
7-
8-
export const RUN_DIR = WORKING_DIR

‎src/run/handlers/cache.cts

+10-32
Original file line numberDiff line numberDiff line change
@@ -3,25 +3,26 @@
33
//
44
import { getDeployStore } from '@netlify/blobs'
55
import { purgeCache } from '@netlify/functions'
6-
import { readFileSync } from 'node:fs'
6+
import type { PrerenderManifest } from 'next/dist/build/index.js'
7+
8+
import { NEXT_CACHE_TAGS_HEADER } from 'next/dist/lib/constants.js'
79
import type {
810
CacheHandler,
911
CacheHandlerContext,
1012
IncrementalCache,
1113
} from 'next/dist/server/lib/incremental-cache/index.js'
12-
import { NEXT_CACHE_TAGS_HEADER } from 'next/dist/lib/constants.js'
14+
import { readFileSync } from 'node:fs'
1315
import { join } from 'node:path/posix'
1416
// @ts-expect-error This is a type only import
1517
import type { CacheEntryValue } from '../../build/content/prerendered.js'
16-
import type { PrerenderManifest } from 'next/dist/build/index.js'
1718

1819
type TagManifest = { revalidatedAt: number }
1920

2021
const tagsManifestPath = '.netlify/cache/tags'
2122
export const blobStore = getDeployStore()
2223

2324
// load the prerender manifest
24-
const prerenderManifest = JSON.parse(
25+
const prerenderManifest: PrerenderManifest = JSON.parse(
2526
readFileSync(join(process.cwd(), '.next/prerender-manifest.json'), 'utf-8'),
2627
)
2728

@@ -30,27 +31,6 @@ function toRoute(pathname: string): string {
3031
return pathname.replace(/\/$/, '').replace(/\/index$/, '') || '/'
3132
}
3233

33-
// borrowed from https://github.com/vercel/next.js/blob/canary/packages/next/src/shared/lib/page-path/normalize-page-path.ts#L14
34-
function ensureLeadingSlash(path: string): string {
35-
return path.startsWith('/') ? path : `/${path}`
36-
}
37-
38-
function isDynamicRoute(path: string): void | boolean {
39-
const dynamicRoutes: PrerenderManifest['dynamicRoutes'] = prerenderManifest.dynamicRoutes
40-
Object.values(dynamicRoutes).find((route) => {
41-
return new RegExp(route.routeRegex).test(path)
42-
})
43-
}
44-
45-
function normalizePath(path: string): string {
46-
// If there is a page that is '/index' the first statement ensures that it will be '/index/index'
47-
return /^\/index(\/|$)/.test(path) && !isDynamicRoute(path)
48-
? `/index${path}`
49-
: path === '/'
50-
? '/index'
51-
: ensureLeadingSlash(path)
52-
}
53-
5434
module.exports = class NetlifyCacheHandler implements CacheHandler {
5535
options: CacheHandlerContext
5636
revalidatedTags: string[]
@@ -66,7 +46,7 @@ module.exports = class NetlifyCacheHandler implements CacheHandler {
6646
async get(...args: Parameters<CacheHandler['get']>): ReturnType<CacheHandler['get']> {
6747
const [cacheKey, ctx = {}] = args
6848
console.debug(`[NetlifyCacheHandler.get]: ${cacheKey}`)
69-
const blob = await this.getBlobKey(cacheKey, ctx.fetchCache)
49+
const blob = await this.getBlobByKey(cacheKey, ctx.fetchCache)
7050

7151
// if blob is null then we don't have a cache entry
7252
if (!blob) {
@@ -169,7 +149,7 @@ module.exports = class NetlifyCacheHandler implements CacheHandler {
169149
* @param fetch If it is a FETCH request or not
170150
* @returns the parsed data from the cache or null if not
171151
*/
172-
private async getBlobKey(
152+
private async getBlobByKey(
173153
key: string,
174154
fetch?: boolean,
175155
): Promise<
@@ -179,10 +159,8 @@ module.exports = class NetlifyCacheHandler implements CacheHandler {
179159
isAppPath: boolean
180160
} & CacheEntryValue)
181161
> {
182-
const normalizedKey = normalizePath(key)
183-
// Want to avoid normalizaing if '/index' is being passed as a key.
184-
const appKey = join('server/app', key === '/index' ? key : normalizedKey)
185-
const pagesKey = join('server/pages', key === '/index' ? key : normalizedKey)
162+
const appKey = join('server/app', key)
163+
const pagesKey = join('server/pages', key)
186164
const fetchKey = join('cache/fetch-cache', key)
187165

188166
if (fetch) {
@@ -248,7 +226,7 @@ module.exports = class NetlifyCacheHandler implements CacheHandler {
248226

249227
const isStale = cacheTags.some((tag) => {
250228
// TODO: test for this case
251-
if (tag && this.revalidatedTags?.includes(tag)) {
229+
if (this.revalidatedTags?.includes(tag)) {
252230
return true
253231
}
254232

0 commit comments

Comments
 (0)