Skip to content

Commit 2576f81

Browse files
orinokaiorinokai
authored
feat: symlink for speed and to avoid clobbering user files (#56)
* chore: swap globby for fast-glob * chore: consistent object params * feat: don't move output, but use cache * feat: add getPrerenderManifest function * chore: tidy up constants * feat: symlink files instead of copy * feat: move blob store and file handling out * fix: add tests and fixes * fix: fix tests * chore: remove logs and onlys
1 parent d21b4a0 commit 2576f81

19 files changed

+443
-488
lines changed

‎package-lock.json‎

Lines changed: 1 addition & 2 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎package.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,7 @@
4242
"@netlify/functions": "^2.0.1",
4343
"@vercel/nft": "^0.24.3",
4444
"fs-monkey": "^1.0.5",
45-
"globby": "^13.2.2",
45+
"fast-glob": "^3.3.2",
4646
"os": "^0.1.2",
4747
"outdent": "^0.8.0",
4848
"p-limit": "^4.0.0"

‎src/build/blob.ts‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
import { getDeployStore } from '@netlify/blobs'
2+
import type { NetlifyPluginConstants } from '@netlify/build'
3+
4+
export const getBlobStore = ({
5+
NETLIFY_API_TOKEN,
6+
NETLIFY_API_HOST,
7+
SITE_ID,
8+
}: Pick<NetlifyPluginConstants, 'NETLIFY_API_TOKEN' | 'NETLIFY_API_HOST' | 'SITE_ID'>): ReturnType<
9+
typeof getDeployStore
10+
> => {
11+
return getDeployStore({
12+
deployID: process.env.DEPLOY_ID,
13+
siteID: SITE_ID,
14+
token: NETLIFY_API_TOKEN,
15+
apiURL: `https://${NETLIFY_API_HOST}`,
16+
})
17+
}

‎src/build/cache.ts‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
import { NetlifyPluginOptions } from '@netlify/build'
2+
import { resolve } from 'node:path'
3+
4+
export const saveBuildCache = async ({
5+
constants: { PUBLISH_DIR },
6+
utils: { cache },
7+
}: Pick<NetlifyPluginOptions, 'constants' | 'utils'>) => {
8+
if (await cache.save(resolve(PUBLISH_DIR, 'cache'))) {
9+
console.log('Next.js cache saved.')
10+
} else {
11+
console.log('No Next.js cache to save.')
12+
}
13+
}
14+
15+
export const restoreBuildCache = async ({
16+
constants: { PUBLISH_DIR },
17+
utils: { cache },
18+
}: Pick<NetlifyPluginOptions, 'constants' | 'utils'>) => {
19+
if (await cache.restore(resolve(PUBLISH_DIR, 'cache'))) {
20+
console.log('Next.js cache restored.')
21+
} else {
22+
console.log('No Next.js cache to restore.')
23+
}
24+
}

‎src/build/config.ts‎

Lines changed: 16 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,24 @@
1-
import { NetlifyConfig } from '@netlify/build'
1+
import type { NetlifyPluginConstants, NetlifyPluginOptions } from '@netlify/build'
2+
import type { PrerenderManifest } from 'next/dist/build/index.js'
3+
import { readFile } from 'node:fs/promises'
4+
import { resolve } from 'node:path'
5+
import { STATIC_DIR } from './constants.js'
6+
7+
export const getPrerenderManifest = async ({
8+
PUBLISH_DIR,
9+
}: Pick<NetlifyPluginConstants, 'PUBLISH_DIR'>): Promise<PrerenderManifest> => {
10+
return JSON.parse(await readFile(resolve(PUBLISH_DIR, 'prerender-manifest.json'), 'utf-8'))
11+
}
212

313
/**
414
* Enable Next.js standalone mode at build time
515
*/
6-
export const setBuildConfig = (netlifyConfig: NetlifyConfig) => {
16+
export const setBuildConfig = () => {
717
process.env.NEXT_PRIVATE_STANDALONE = 'true'
18+
}
19+
20+
export const setDeployConfig = ({ netlifyConfig }: Pick<NetlifyPluginOptions, 'netlifyConfig'>) => {
21+
netlifyConfig.build.publish = STATIC_DIR
822
netlifyConfig.redirects ||= []
923
netlifyConfig.redirects.push({
1024
from: '/*',

‎src/build/constants.ts‎

Lines changed: 11 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,21 @@
1+
import { readFileSync } from 'node:fs'
12
import { join } from 'node:path'
23
import { fileURLToPath } from 'node:url'
34

45
export const MODULE_DIR = fileURLToPath(new URL('.', import.meta.url))
56
export const PLUGIN_DIR = join(MODULE_DIR, '../..')
67

7-
/** A relative path where we store the build output */
8-
export const BUILD_DIR = '.netlify'
8+
const pkg = JSON.parse(readFileSync(join(PLUGIN_DIR, 'package.json'), 'utf-8'))
99

10-
export const SERVER_FUNCTIONS_DIR = join(BUILD_DIR, 'functions-internal')
11-
export const SERVER_HANDLER_NAME = '___netlify-server-handler'
10+
export const PLUGIN_NAME = pkg.name
11+
export const PLUGIN_VERSION = pkg.version
12+
13+
export const STATIC_DIR = '.netlify/static'
14+
15+
export const SERVER_FUNCTIONS_DIR = '.netlify/functions-internal'
16+
export const SERVER_HANDLER_NAME = '_netlify-server-handler'
1217
export const SERVER_HANDLER_DIR = join(SERVER_FUNCTIONS_DIR, SERVER_HANDLER_NAME)
1318

14-
export const EDGE_FUNCTIONS_DIR = join(BUILD_DIR, 'edge-functions')
15-
export const EDGE_HANDLER_NAME = '___netlify-edge-handler'
19+
export const EDGE_FUNCTIONS_DIR = '.netlify/edge-functions'
20+
export const EDGE_HANDLER_NAME = '_netlify-edge-handler'
1621
export const EDGE_HANDLER_DIR = join(EDGE_FUNCTIONS_DIR, EDGE_HANDLER_NAME)

‎src/build/content/prerendered.ts‎

Lines changed: 82 additions & 87 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,13 @@
1-
import { getDeployStore } from '@netlify/blobs'
2-
import { NetlifyPluginConstants } from '@netlify/build'
3-
import { globby } from 'globby'
1+
import { NetlifyPluginOptions } from '@netlify/build'
2+
import glob from 'fast-glob'
3+
import type { PrerenderManifest } from 'next/dist/build/index.js'
44
import { readFile } from 'node:fs/promises'
5-
import { join } from 'node:path'
5+
import { basename, dirname, extname, resolve } from 'node:path'
6+
import { join as joinPosix } from 'node:path/posix'
67
import { cpus } from 'os'
78
import pLimit from 'p-limit'
8-
import { parse, ParsedPath } from 'path'
9-
import { BUILD_DIR } from '../constants.js'
9+
import { getBlobStore } from '../blob.js'
10+
import { getPrerenderManifest } from '../config.js'
1011

1112
export type CacheEntry = {
1213
key: string
@@ -45,107 +46,101 @@ type FetchCacheValue = {
4546
}
4647

4748
// static prerendered pages content with JSON data
48-
const isPage = ({ dir, name, ext }: ParsedPath, paths: string[]) => {
49-
return dir.startsWith('server/pages') && ext === '.html' && paths.includes(`${dir}/${name}.json`)
49+
const isPage = (key: string, routes: string[]) => {
50+
return key.startsWith('server/pages') && routes.includes(key.replace(/^server\/pages/, ''))
5051
}
5152
// static prerendered app content with RSC data
52-
const isApp = ({ dir, ext }: ParsedPath) => {
53-
return dir.startsWith('server/app') && ext === '.html'
53+
const isApp = (path: string) => {
54+
return path.startsWith('server/app') && extname(path) === '.html'
5455
}
5556
// static prerendered app route handler
56-
const isRoute = ({ dir, ext }: ParsedPath) => {
57-
return dir.startsWith('server/app') && ext === '.body'
57+
const isRoute = (path: string) => {
58+
return path.startsWith('server/app') && extname(path) === '.body'
5859
}
59-
// fetch cache data
60-
const isFetch = ({ dir }: ParsedPath) => {
61-
return dir.startsWith('cache/fetch-cache')
60+
// fetch cache data (excluding tags manifest)
61+
const isFetch = (path: string) => {
62+
return path.startsWith('cache/fetch-cache') && extname(path) === ''
6263
}
6364

6465
/**
6566
* Transform content file paths into cache entries for the blob store
6667
*/
67-
const buildPrerenderedContentEntries = async (cwd: string): Promise<Promise<CacheEntry>[]> => {
68-
const paths = await globby(
69-
[`cache/fetch-cache/*`, `server/+(app|pages)/**/*.+(html|body|json)`],
70-
{
71-
cwd,
72-
extglob: true,
73-
},
74-
)
68+
const buildPrerenderedContentEntries = async (
69+
src: string,
70+
routes: string[],
71+
): Promise<Promise<CacheEntry>[]> => {
72+
const paths = await glob([`cache/fetch-cache/*`, `server/+(app|pages)/**/*.+(html|body)`], {
73+
cwd: resolve(src),
74+
extglob: true,
75+
})
76+
77+
return paths.map(async (path: string): Promise<CacheEntry> => {
78+
const key = joinPosix(dirname(path), basename(path, extname(path)))
79+
let value
80+
81+
if (isPage(key, routes)) {
82+
value = {
83+
kind: 'PAGE',
84+
html: await readFile(resolve(src, `${key}.html`), 'utf-8'),
85+
pageData: JSON.parse(await readFile(resolve(src, `${key}.json`), 'utf-8')),
86+
} satisfies PageCacheValue
87+
}
88+
89+
if (isApp(path)) {
90+
value = {
91+
kind: 'PAGE',
92+
html: await readFile(resolve(src, `${key}.html`), 'utf-8'),
93+
pageData: await readFile(resolve(src, `${key}.rsc`), 'utf-8'),
94+
...JSON.parse(await readFile(resolve(src, `${key}.meta`), 'utf-8')),
95+
} satisfies PageCacheValue
96+
}
7597

76-
return paths
77-
.map(parse)
78-
.filter((path: ParsedPath) => {
79-
return isPage(path, paths) || isApp(path) || isRoute(path) || isFetch(path)
80-
})
81-
.map(async (path: ParsedPath): Promise<CacheEntry> => {
82-
const { dir, name, ext } = path
83-
const key = join(dir, name)
84-
let value
85-
86-
if (isPage(path, paths)) {
87-
value = {
88-
kind: 'PAGE',
89-
html: await readFile(`${cwd}/${key}.html`, 'utf-8'),
90-
pageData: JSON.parse(await readFile(`${cwd}/${key}.json`, 'utf-8')),
91-
} satisfies PageCacheValue
92-
}
93-
94-
if (isApp(path)) {
95-
value = {
96-
kind: 'PAGE',
97-
html: await readFile(`${cwd}/${key}.html`, 'utf-8'),
98-
pageData: await readFile(`${cwd}/${key}.rsc`, 'utf-8'),
99-
...JSON.parse(await readFile(`${cwd}/${key}.meta`, 'utf-8')),
100-
} satisfies PageCacheValue
101-
}
102-
103-
if (isRoute(path)) {
104-
value = {
105-
kind: 'ROUTE',
106-
body: await readFile(`${cwd}/${key}.body`, 'utf-8'),
107-
...JSON.parse(await readFile(`${cwd}/${key}.meta`, 'utf-8')),
108-
} satisfies RouteCacheValue
109-
}
110-
111-
if (isFetch(path)) {
112-
value = {
113-
kind: 'FETCH',
114-
...JSON.parse(await readFile(`${cwd}/${key}`, 'utf-8')),
115-
} satisfies FetchCacheValue
116-
}
117-
118-
return {
119-
key,
120-
value: {
121-
lastModified: Date.now(),
122-
value,
123-
},
124-
}
125-
})
98+
if (isRoute(path)) {
99+
value = {
100+
kind: 'ROUTE',
101+
body: await readFile(resolve(src, `${key}.body`), 'utf-8'),
102+
...JSON.parse(await readFile(resolve(src, `${key}.meta`), 'utf-8')),
103+
} satisfies RouteCacheValue
104+
}
105+
106+
if (isFetch(path)) {
107+
value = {
108+
kind: 'FETCH',
109+
...JSON.parse(await readFile(resolve(src, key), 'utf-8')),
110+
} satisfies FetchCacheValue
111+
}
112+
113+
return {
114+
key,
115+
value: {
116+
lastModified: Date.now(),
117+
value,
118+
},
119+
}
120+
})
126121
}
127122

128123
/**
129124
* Upload prerendered content to the blob store and remove it from the bundle
130125
*/
131126
export const uploadPrerenderedContent = async ({
132-
NETLIFY_API_TOKEN,
133-
NETLIFY_API_HOST,
134-
SITE_ID,
135-
}: NetlifyPluginConstants) => {
136-
// initialize the blob store
137-
const blob = getDeployStore({
138-
deployID: process.env.DEPLOY_ID,
139-
siteID: SITE_ID,
140-
token: NETLIFY_API_TOKEN,
141-
apiURL: `https://${NETLIFY_API_HOST}`,
142-
})
127+
constants: { PUBLISH_DIR, NETLIFY_API_TOKEN, NETLIFY_API_HOST, SITE_ID },
128+
}: Pick<NetlifyPluginOptions, 'constants'>) => {
143129
// limit concurrent uploads to 2x the number of CPUs
144130
const limit = pLimit(Math.max(2, cpus().length))
145131

146132
// read prerendered content and build JSON key/values for the blob store
133+
let manifest: PrerenderManifest
134+
let blob: ReturnType<typeof getBlobStore>
135+
try {
136+
manifest = await getPrerenderManifest({ PUBLISH_DIR })
137+
blob = getBlobStore({ NETLIFY_API_TOKEN, NETLIFY_API_HOST, SITE_ID })
138+
} catch (error: any) {
139+
console.error(`Unable to upload prerendered content: ${error.message}`)
140+
return
141+
}
147142
const entries = await Promise.allSettled(
148-
await buildPrerenderedContentEntries(join(process.cwd(), BUILD_DIR, '.next')),
143+
await buildPrerenderedContentEntries(PUBLISH_DIR, Object.keys(manifest.routes)),
149144
)
150145
entries.forEach((result) => {
151146
if (result.status === 'rejected') {
@@ -156,7 +151,7 @@ export const uploadPrerenderedContent = async ({
156151
// upload JSON content data to the blob store
157152
const uploads = await Promise.allSettled(
158153
entries
159-
.filter((entry) => entry.status === 'fulfilled')
154+
.filter((entry) => entry.status === 'fulfilled' && entry.value.value.value !== undefined)
160155
.map((entry: PromiseSettledResult<CacheEntry>) => {
161156
const result = entry as PromiseFulfilledResult<CacheEntry>
162157
const { key, value } = result.value
@@ -166,7 +161,7 @@ export const uploadPrerenderedContent = async ({
166161
uploads.forEach((upload, index) => {
167162
if (upload.status === 'rejected') {
168163
const result = entries[index] as PromiseFulfilledResult<CacheEntry>
169-
console.error(`Unable to store ${result.value.key}: ${upload.reason.message}`)
164+
console.error(`Unable to store ${result.value?.key}: ${upload.reason.message}`)
170165
}
171166
})
172167
}

‎src/build/content/server.ts‎

Lines changed: 18 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1,28 +1,32 @@
1-
import { globby } from 'globby'
2-
import { existsSync } from 'node:fs'
3-
import { copyFile, mkdir } from 'node:fs/promises'
1+
import glob from 'fast-glob'
2+
import { mkdir, symlink } from 'node:fs/promises'
43
import { dirname, join } from 'node:path'
54

65
/**
76
* Copy App/Pages Router Javascript needed by the server handler
87
*/
9-
export const copyServerContent = async (src: string, dest: string): Promise<void> => {
10-
const paths = await globby([`*`, `server/*`, `server/chunks/*`, `server/+(app|pages)/**/*.js`], {
8+
export const linkServerContent = async (src: string, dest: string): Promise<void> => {
9+
const paths = await glob([`*`, `server/*`, `server/chunks/*`, `server/+(app|pages)/**/*.js`], {
1110
cwd: src,
1211
extglob: true,
1312
})
14-
1513
await Promise.all(
1614
paths.map(async (path: string) => {
17-
const srcPath = join(src, path)
18-
const destPath = join(dest, path)
19-
20-
if (!existsSync(srcPath)) {
21-
throw new Error(`Source file does not exist: ${srcPath}`)
22-
}
15+
await mkdir(join(dest, dirname(path)), { recursive: true })
16+
await symlink(join(src, path), join(dest, path))
17+
}),
18+
)
19+
}
2320

24-
await mkdir(dirname(destPath), { recursive: true })
25-
await copyFile(srcPath, destPath)
21+
export const linkServerDependencies = async (src: string, dest: string): Promise<void> => {
22+
const paths = await glob([`**`], {
23+
cwd: src,
24+
extglob: true,
25+
})
26+
await Promise.all(
27+
paths.map(async (path: string) => {
28+
await mkdir(join(dest, dirname(path)), { recursive: true })
29+
await symlink(join(src, path), join(dest, path))
2630
}),
2731
)
2832
}

0 commit comments

Comments
 (0)