Skip to content

Commit 72d34ce

Browse files
ascorbicascorbic
authored
Merge pull request #334 from netlify/mk/page-function-worker
perf: parallelize page setup
2 parents 6107137 + b27f30d commit 72d34ce

File tree

13 files changed

+11034
-317
lines changed

13 files changed

+11034
-317
lines changed

Diff for: ‎package-lock.json

+10,907-219
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

Diff for: ‎package.json

+1
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,7 @@
6262
"commander": "^7.1.0",
6363
"debounce-fn": "^4.0.0",
6464
"execa": "^5.0.0",
65+
"fastq": "^1.11.0",
6566
"find-cache-dir": "^3.3.1",
6667
"find-up": "^5.0.0",
6768
"fs-extra": "^9.1.0",

Diff for: ‎src/lib/helpers/copyDynamicImportChunks.js

+14-11
Original file line numberDiff line numberDiff line change
@@ -12,27 +12,30 @@ const copyDynamicImportChunks = async (functionPath) => {
1212
const filesWP4 = readdirSync(chunksPathWebpack4)
1313
const chunkRegexWP4 = new RegExp(/^(\.?[-$~\w]+)+\.js$/g)
1414
const excludeFiles = new Set(['init-server.js.js', 'on-error-server.js.js'])
15+
const copyPathWP4 = join(functionPath, 'nextPage')
16+
if (filesWP4.length !== 0) {
17+
logTitle('💼 Copying WP4 dynamic import chunks to', copyPathWP4)
18+
}
1519
filesWP4.forEach((file) => {
1620
if (!excludeFiles.has(file) && chunkRegexWP4.test(file)) {
17-
// WP4 files are looked for one level up (../) in runtime
18-
// This is a hack to make the file one level up i.e. with
19-
// nextPage/nextPage/index.js, the chunk is moved to the inner nextPage
20-
const copyPath = join(functionPath, 'nextPage')
21-
logTitle('💼 Copying WP4 dynamic import chunks to', copyPath)
22-
copySync(join(chunksPathWebpack4, file), join(copyPath, file), {
21+
copySync(join(chunksPathWebpack4, file), join(copyPathWP4, file), {
2322
overwrite: false,
2423
errorOnExist: true,
2524
})
2625
}
2726
})
27+
28+
// Chunks are copied into the nextPage directory, as a sibling to "pages" or "api".
29+
// This matches the Next output, so that imports work correctly
2830
const chunksPathWebpack5 = join(nextDistDir, 'serverless', 'chunks')
2931
const filesWP5 = existsSync(chunksPathWebpack5) ? readdirSync(chunksPathWebpack5) : []
32+
const copyPathWP5 = join(functionPath, 'nextPage', 'chunks')
33+
if (filesWP5.length !== 0) {
34+
logTitle('💼 Copying WB5 dynamic import chunks to', copyPathWP5)
35+
}
36+
3037
filesWP5.forEach((file) => {
31-
// Chunks are copied into the nextPage directory, as a sibling to pages or api.
32-
// This matches the Next output, so that imports work correctly
33-
const copyPath = join(functionPath, 'nextPage', 'chunks')
34-
logTitle('💼 Copying WB5 dynamic import chunks to', copyPath)
35-
copySync(join(chunksPathWebpack5, file), join(copyPath, file), {
38+
copySync(join(chunksPathWebpack5, file), join(copyPathWP5, file), {
3639
overwrite: false,
3740
errorOnExist: true,
3841
})

Diff for: ‎src/lib/helpers/runJobsQueue.js

+18
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
const os = require('os')
2+
3+
const fastq = require('fastq')
4+
5+
const { processPage } = require('../pages/worker')
6+
7+
// TODO: benchmark a large site to find a good value for this
8+
const PAGE_CONCURRENCY = 4
9+
10+
const runJobsQueue = (jobs) => {
11+
console.log(`Building ${jobs.length} pages`)
12+
13+
const queue = fastq.promise(processPage, PAGE_CONCURRENCY)
14+
15+
return Promise.all(jobs.map((job) => queue.push(job)))
16+
}
17+
18+
module.exports = runJobsQueue

Diff for: ‎src/lib/pages/api/setup.js

+3-8
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,4 @@
1-
const asyncForEach = require('../../helpers/asyncForEach')
2-
const { logTitle, logItem } = require('../../helpers/logger')
3-
const setupNetlifyFunctionForPage = require('../../helpers/setupNetlifyFunctionForPage')
1+
const { logTitle } = require('../../helpers/logger')
42

53
const getPages = require('./pages')
64

@@ -10,11 +8,8 @@ const setup = async (functionsPath) => {
108

119
const pages = await getPages()
1210

13-
// Create Netlify Function for every page
14-
await asyncForEach(pages, async ({ filePath }) => {
15-
logItem(filePath)
16-
await setupNetlifyFunctionForPage({ filePath, functionsPath, isApiPage: true })
17-
})
11+
// Create Netlify Function job for every page
12+
return pages.map(({ filePath }) => ({ type: 'function', filePath, functionsPath, isApiPage: true }))
1813
}
1914

2015
module.exports = setup

Diff for: ‎src/lib/pages/getInitialProps/setup.js

+2-7
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,4 @@
1-
const asyncForEach = require('../../helpers/asyncForEach')
2-
const { logTitle, logItem } = require('../../helpers/logger')
3-
const setupNetlifyFunctionForPage = require('../../helpers/setupNetlifyFunctionForPage')
1+
const { logTitle } = require('../../helpers/logger')
42

53
const getPages = require('./pages')
64

@@ -11,10 +9,7 @@ const setup = async (functionsPath) => {
119
const pages = await getPages()
1210

1311
// Create Netlify Function for every page
14-
await asyncForEach(pages, async ({ filePath }) => {
15-
logItem(filePath)
16-
await setupNetlifyFunctionForPage({ filePath, functionsPath })
17-
})
12+
return pages.map(({ filePath }) => ({ type: 'function', filePath, functionsPath }))
1813
}
1914

2015
module.exports = setup

Diff for: ‎src/lib/pages/getServerSideProps/setup.js

+2-7
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,4 @@
1-
const asyncForEach = require('../../helpers/asyncForEach')
2-
const { logTitle, logItem } = require('../../helpers/logger')
3-
const setupNetlifyFunctionForPage = require('../../helpers/setupNetlifyFunctionForPage')
1+
const { logTitle } = require('../../helpers/logger')
42

53
const getPages = require('./pages')
64

@@ -11,10 +9,7 @@ const setup = async (functionsPath) => {
119
const pages = await getPages()
1210

1311
// Create Netlify Function for every page
14-
await asyncForEach(pages, async ({ filePath }) => {
15-
logItem(filePath)
16-
await setupNetlifyFunctionForPage({ filePath, functionsPath })
17-
})
12+
return pages.map(({ filePath }) => ({ type: 'function', filePath, functionsPath }))
1813
}
1914

2015
module.exports = setup

Diff for: ‎src/lib/pages/getStaticProps/setup.js

+39-34
Original file line numberDiff line numberDiff line change
@@ -1,51 +1,56 @@
11
const { join } = require('path')
22

3-
const asyncForEach = require('../../helpers/asyncForEach')
43
const getFilePathForRoute = require('../../helpers/getFilePathForRoute')
54
const isRouteWithFallback = require('../../helpers/isRouteWithFallback')
6-
const { logTitle, logItem } = require('../../helpers/logger')
7-
const setupNetlifyFunctionForPage = require('../../helpers/setupNetlifyFunctionForPage')
8-
const setupStaticFileForPage = require('../../helpers/setupStaticFileForPage')
5+
const { logTitle } = require('../../helpers/logger')
96

107
const getPages = require('./pages')
118

129
// Copy pre-rendered SSG pages
1310
const setup = async ({ functionsPath, publishPath }) => {
1411
logTitle('🔥 Copying pre-rendered pages with getStaticProps and JSON data to', publishPath)
15-
1612
// Keep track of the functions that have been set up, so that we do not set up
1713
// a function for the same file path twice
18-
const filePathsDone = []
19-
14+
const filePathsDone = new Set()
2015
const pages = await getPages()
2116

22-
await asyncForEach(pages, async ({ route, dataRoute, srcRoute }) => {
23-
logItem(route)
24-
25-
// Copy pre-rendered HTML page
26-
const htmlPath = getFilePathForRoute(route, 'html')
27-
await setupStaticFileForPage({ inputPath: htmlPath, publishPath })
28-
29-
// Copy page's JSON data
30-
const jsonPath = getFilePathForRoute(route, 'json')
31-
await setupStaticFileForPage({
32-
inputPath: jsonPath,
33-
outputPath: dataRoute,
34-
publishPath,
35-
})
36-
37-
// Set up the Netlify function (this is ONLY for preview mode)
38-
const relativePath = getFilePathForRoute(srcRoute || route, 'js')
39-
const filePath = join('pages', relativePath)
40-
41-
// Skip if we have already set up a function for this file
42-
// or if the source route has a fallback (handled by getStaticPropsWithFallback)
43-
if (filePathsDone.includes(filePath) || (await isRouteWithFallback(srcRoute))) return
44-
45-
logItem(filePath)
46-
await setupNetlifyFunctionForPage({ filePath, functionsPath })
47-
filePathsDone.push(filePath)
48-
})
17+
const jobs = []
18+
19+
await Promise.all(
20+
pages.map(async ({ route, dataRoute, srcRoute }) => {
21+
// Copy pre-rendered HTML page
22+
const htmlPath = getFilePathForRoute(route, 'html')
23+
24+
jobs.push({ type: 'static', inputPath: htmlPath, publishPath })
25+
26+
// Copy page's JSON data
27+
const jsonPath = getFilePathForRoute(route, 'json')
28+
jobs.push({
29+
type: 'static',
30+
inputPath: jsonPath,
31+
outputPath: dataRoute,
32+
publishPath,
33+
})
34+
35+
// Set up the Netlify function (this is ONLY for preview mode)
36+
const relativePath = getFilePathForRoute(srcRoute || route, 'js')
37+
const filePath = join('pages', relativePath)
38+
39+
// Skip if we have already set up a function for this file
40+
41+
if (filePathsDone.has(filePath)) {
42+
return
43+
}
44+
filePathsDone.add(filePath)
45+
46+
// or if the source route has a fallback (handled by getStaticPropsWithFallback)
47+
if (await isRouteWithFallback(srcRoute)) {
48+
return
49+
}
50+
jobs.push({ type: 'function', filePath, functionsPath })
51+
}),
52+
)
53+
return jobs
4954
}
5055

5156
module.exports = setup

Diff for: ‎src/lib/pages/getStaticPropsWithFallback/setup.js

+3-6
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,7 @@
11
const { join } = require('path')
22

3-
const asyncForEach = require('../../helpers/asyncForEach')
43
const getFilePathForRoute = require('../../helpers/getFilePathForRoute')
5-
const { logTitle, logItem } = require('../../helpers/logger')
6-
const setupNetlifyFunctionForPage = require('../../helpers/setupNetlifyFunctionForPage')
4+
const { logTitle } = require('../../helpers/logger')
75

86
const getPages = require('./pages')
97

@@ -14,11 +12,10 @@ const setup = async (functionsPath) => {
1412
const pages = await getPages()
1513

1614
// Create Netlify Function for every page
17-
await asyncForEach(pages, async ({ route }) => {
15+
return pages.map(({ route }) => {
1816
const relativePath = getFilePathForRoute(route, 'js')
1917
const filePath = join('pages', relativePath)
20-
logItem(filePath)
21-
await setupNetlifyFunctionForPage({ filePath, functionsPath, isISR: true })
18+
return { type: 'function', filePath, functionsPath, isISR: true }
2219
})
2320
}
2421

Diff for: ‎src/lib/pages/getStaticPropsWithRevalidate/setup.js

+12-13
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,7 @@
11
const { join } = require('path')
22

3-
const asyncForEach = require('../../helpers/asyncForEach')
43
const getFilePathForRoute = require('../../helpers/getFilePathForRoute')
5-
const { logTitle, logItem } = require('../../helpers/logger')
6-
const setupNetlifyFunctionForPage = require('../../helpers/setupNetlifyFunctionForPage')
4+
const { logTitle } = require('../../helpers/logger')
75

86
const getPages = require('./pages')
97

@@ -17,23 +15,24 @@ const setup = async (functionsPath) => {
1715

1816
// Keep track of the functions that have been set up, so that we do not set up
1917
// a function for the same file path twice
20-
const filePathsDone = []
18+
const filePathsDone = new Set()
2119

2220
const pages = await getPages()
2321

2422
// Create Netlify Function for every page
25-
await asyncForEach(pages, async ({ route, srcRoute }) => {
26-
const relativePath = getFilePathForRoute(srcRoute || route, 'js')
27-
const filePath = join('pages', relativePath)
2823

29-
// Skip if we have already set up a function for this file
30-
if (filePathsDone.includes(filePath)) return
24+
const jobs = []
3125

32-
// Set up the function
33-
logItem(filePath)
34-
await setupNetlifyFunctionForPage({ filePath, functionsPath })
35-
filePathsDone.push(filePath)
26+
pages.forEach(({ route, srcRoute }) => {
27+
const relativePath = getFilePathForRoute(srcRoute || route, 'js')
28+
const filePath = join('pages', relativePath)
29+
if (filePathsDone.has(filePath)) {
30+
return
31+
}
32+
filePathsDone.add(filePath)
33+
jobs.push({ type: 'function', filePath, functionsPath })
3634
})
35+
return jobs
3736
}
3837

3938
module.exports = setup

Diff for: ‎src/lib/pages/withoutProps/setup.js

+2-5
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,6 @@ const asyncForEach = require('../../helpers/asyncForEach')
66
const getI18n = require('../../helpers/getI18n')
77
const getNextDistDir = require('../../helpers/getNextDistDir')
88
const { logTitle, logItem } = require('../../helpers/logger')
9-
const setupStaticFileForPage = require('../../helpers/setupStaticFileForPage')
109

1110
const getPages = require('./pages')
1211

@@ -20,17 +19,15 @@ const setup = async (publishPath) => {
2019
const pages = await getPages()
2120

2221
// Copy each page to the Netlify publish directory
23-
await asyncForEach(pages, async ({ filePath }) => {
24-
logItem(filePath)
25-
22+
return pages.map(({ filePath }) => {
2623
// HACK: If i18n, 404.html needs to be at the top level of the publish directory
2724
if (i18n.defaultLocale && filePath === `pages/${i18n.defaultLocale}/404.html`) {
2825
copySync(join(nextDistDir, 'serverless', filePath), join(publishPath, '404.html'))
2926
}
3027

3128
// The path to the file, relative to the pages directory
3229
const relativePath = relative('pages', filePath)
33-
await setupStaticFileForPage({ inputPath: relativePath, publishPath })
30+
return { type: 'static', inputPath: relativePath, publishPath }
3431
})
3532
}
3633

Diff for: ‎src/lib/pages/worker.js

+19
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
const setupNetlifyFunctionForPage = require('../helpers/setupNetlifyFunctionForPage')
2+
const setupStaticFileForPage = require('../helpers/setupStaticFileForPage')
3+
4+
exports.processPage = function processPage(job) {
5+
try {
6+
switch (job.type) {
7+
case 'function': {
8+
return setupNetlifyFunctionForPage(job)
9+
}
10+
case 'static': {
11+
return setupStaticFileForPage(job)
12+
}
13+
default:
14+
console.log('Unknown job type', job.type)
15+
}
16+
} catch (error) {
17+
console.error('Error in worker', error, 'Job', job)
18+
}
19+
}

Diff for: ‎src/lib/steps/setupPages.js

+12-7
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
const runJobsQueue = require('../helpers/runJobsQueue')
12
const apiSetup = require('../pages/api/setup')
23
const getInitialPropsSetup = require('../pages/getInitialProps/setup')
34
const getServerSidePropsSetup = require('../pages/getServerSideProps/setup')
@@ -8,13 +9,17 @@ const withoutPropsSetup = require('../pages/withoutProps/setup')
89

910
// Set up all our NextJS pages according to the recipes defined in pages/
1011
const setupPages = async ({ functionsPath, publishPath }) => {
11-
await apiSetup(functionsPath)
12-
await getInitialPropsSetup(functionsPath)
13-
await getServerSidePropsSetup(functionsPath)
14-
await getStaticPropsSetup({ functionsPath, publishPath })
15-
await getSPFallbackSetup(functionsPath)
16-
await getSPRevalidateSetup(functionsPath)
17-
await withoutPropsSetup(publishPath)
12+
const jobs = [
13+
...(await apiSetup(functionsPath)),
14+
...(await getInitialPropsSetup(functionsPath)),
15+
...(await getServerSidePropsSetup(functionsPath)),
16+
...(await getStaticPropsSetup({ functionsPath, publishPath })),
17+
...(await getSPFallbackSetup(functionsPath)),
18+
...(await getSPRevalidateSetup(functionsPath)),
19+
...(await withoutPropsSetup(publishPath)),
20+
]
21+
22+
return runJobsQueue(jobs)
1823
}
1924

2025
module.exports = setupPages

0 commit comments

Comments
 (0)