Skip to content

Commit 7184379

Browse files
ascorbicascorbic
committed
perf: parallelize page processing
1 parent c7860ab commit 7184379

File tree

13 files changed

+11019
-308
lines changed

13 files changed

+11019
-308
lines changed

‎package-lock.json

Lines changed: 10907 additions & 219 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎package.json

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,7 @@
6262
"commander": "^7.1.0",
6363
"debounce-fn": "^4.0.0",
6464
"execa": "^5.0.0",
65+
"fastq": "^1.11.0",
6566
"find-cache-dir": "^3.3.1",
6667
"find-up": "^5.0.0",
6768
"fs-extra": "^9.1.0",

‎src/lib/helpers/copyDynamicImportChunks.js

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ const copyDynamicImportChunks = async (functionPath) => {
1818
// This is a hack to make the file one level up i.e. with
1919
// nextPage/nextPage/index.js, the chunk is moved to the inner nextPage
2020
const copyPath = join(functionPath, 'nextPage')
21-
logTitle('💼 Copying WP4 dynamic import chunks to', copyPath)
21+
// logTitle('💼 Copying WP4 dynamic import chunks to', copyPath)
2222
copySync(join(chunksPathWebpack4, file), join(copyPath, file), {
2323
overwrite: false,
2424
errorOnExist: true,
@@ -31,7 +31,7 @@ const copyDynamicImportChunks = async (functionPath) => {
3131
// Chunks are copied into the nextPage directory, as a sibling to pages or api.
3232
// This matches the Next output, so that imports work correctly
3333
const copyPath = join(functionPath, 'nextPage', 'chunks')
34-
logTitle('💼 Copying WB5 dynamic import chunks to', copyPath)
34+
// logTitle('💼 Copying WB5 dynamic import chunks to', copyPath)
3535
copySync(join(chunksPathWebpack5, file), join(copyPath, file), {
3636
overwrite: false,
3737
errorOnExist: true,

‎src/lib/helpers/runJobsQueue.js

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
const os = require('os')
2+
3+
const fastq = require('fastq')
4+
5+
const { processPage } = require('../pages/worker')
6+
7+
const runJobsQueue = (jobs) => {
8+
console.log(`Building ${jobs.length} pages`)
9+
10+
const queue = fastq.promise(processPage, 4)
11+
12+
return Promise.all(jobs.map((job) => queue.push(job)))
13+
}
14+
15+
module.exports = runJobsQueue

‎src/lib/pages/api/setup.js

Lines changed: 3 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,4 @@
1-
const asyncForEach = require('../../helpers/asyncForEach')
2-
const { logTitle, logItem } = require('../../helpers/logger')
3-
const setupNetlifyFunctionForPage = require('../../helpers/setupNetlifyFunctionForPage')
1+
const { logTitle } = require('../../helpers/logger')
42

53
const getPages = require('./pages')
64

@@ -10,11 +8,8 @@ const setup = async (functionsPath) => {
108

119
const pages = await getPages()
1210

13-
// Create Netlify Function for every page
14-
await asyncForEach(pages, async ({ filePath }) => {
15-
logItem(filePath)
16-
await setupNetlifyFunctionForPage({ filePath, functionsPath, isApiPage: true })
17-
})
11+
// Create Netlify Function job for every page
12+
return pages.map(({ filePath }) => ({ type: 'function', filePath, functionsPath, isApiPage: true }))
1813
}
1914

2015
module.exports = setup

‎src/lib/pages/getInitialProps/setup.js

Lines changed: 2 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,4 @@
1-
const asyncForEach = require('../../helpers/asyncForEach')
2-
const { logTitle, logItem } = require('../../helpers/logger')
3-
const setupNetlifyFunctionForPage = require('../../helpers/setupNetlifyFunctionForPage')
1+
const { logTitle } = require('../../helpers/logger')
42

53
const getPages = require('./pages')
64

@@ -11,10 +9,7 @@ const setup = async (functionsPath) => {
119
const pages = await getPages()
1210

1311
// Create Netlify Function for every page
14-
await asyncForEach(pages, async ({ filePath }) => {
15-
logItem(filePath)
16-
await setupNetlifyFunctionForPage({ filePath, functionsPath })
17-
})
12+
return pages.map(({ filePath }) => ({ type: 'function', filePath, functionsPath }))
1813
}
1914

2015
module.exports = setup

‎src/lib/pages/getServerSideProps/setup.js

Lines changed: 2 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,4 @@
1-
const asyncForEach = require('../../helpers/asyncForEach')
2-
const { logTitle, logItem } = require('../../helpers/logger')
3-
const setupNetlifyFunctionForPage = require('../../helpers/setupNetlifyFunctionForPage')
1+
const { logTitle } = require('../../helpers/logger')
42

53
const getPages = require('./pages')
64

@@ -11,10 +9,7 @@ const setup = async (functionsPath) => {
119
const pages = await getPages()
1210

1311
// Create Netlify Function for every page
14-
await asyncForEach(pages, async ({ filePath }) => {
15-
logItem(filePath)
16-
await setupNetlifyFunctionForPage({ filePath, functionsPath })
17-
})
12+
return pages.map(({ filePath }) => ({ type: 'function', filePath, functionsPath }))
1813
}
1914

2015
module.exports = setup

‎src/lib/pages/getStaticProps/setup.js

Lines changed: 39 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -1,51 +1,56 @@
11
const { join } = require('path')
22

3-
const asyncForEach = require('../../helpers/asyncForEach')
43
const getFilePathForRoute = require('../../helpers/getFilePathForRoute')
54
const isRouteWithFallback = require('../../helpers/isRouteWithFallback')
6-
const { logTitle, logItem } = require('../../helpers/logger')
7-
const setupNetlifyFunctionForPage = require('../../helpers/setupNetlifyFunctionForPage')
8-
const setupStaticFileForPage = require('../../helpers/setupStaticFileForPage')
5+
const { logTitle } = require('../../helpers/logger')
96

107
const getPages = require('./pages')
118

129
// Copy pre-rendered SSG pages
1310
const setup = async ({ functionsPath, publishPath }) => {
1411
logTitle('🔥 Copying pre-rendered pages with getStaticProps and JSON data to', publishPath)
15-
1612
// Keep track of the functions that have been set up, so that we do not set up
1713
// a function for the same file path twice
18-
const filePathsDone = []
19-
14+
const filePathsDone = new Set()
2015
const pages = await getPages()
2116

22-
await asyncForEach(pages, async ({ route, dataRoute, srcRoute }) => {
23-
logItem(route)
24-
25-
// Copy pre-rendered HTML page
26-
const htmlPath = getFilePathForRoute(route, 'html')
27-
await setupStaticFileForPage({ inputPath: htmlPath, publishPath })
28-
29-
// Copy page's JSON data
30-
const jsonPath = getFilePathForRoute(route, 'json')
31-
await setupStaticFileForPage({
32-
inputPath: jsonPath,
33-
outputPath: dataRoute,
34-
publishPath,
35-
})
36-
37-
// Set up the Netlify function (this is ONLY for preview mode)
38-
const relativePath = getFilePathForRoute(srcRoute || route, 'js')
39-
const filePath = join('pages', relativePath)
40-
41-
// Skip if we have already set up a function for this file
42-
// or if the source route has a fallback (handled by getStaticPropsWithFallback)
43-
if (filePathsDone.includes(filePath) || (await isRouteWithFallback(srcRoute))) return
44-
45-
logItem(filePath)
46-
await setupNetlifyFunctionForPage({ filePath, functionsPath })
47-
filePathsDone.push(filePath)
48-
})
17+
const jobs = []
18+
19+
await Promise.all(
20+
pages.map(async ({ route, dataRoute, srcRoute }) => {
21+
// Copy pre-rendered HTML page
22+
const htmlPath = getFilePathForRoute(route, 'html')
23+
24+
jobs.push({ type: 'static', inputPath: htmlPath, publishPath })
25+
26+
// Copy page's JSON data
27+
const jsonPath = getFilePathForRoute(route, 'json')
28+
jobs.push({
29+
type: 'static',
30+
inputPath: jsonPath,
31+
outputPath: dataRoute,
32+
publishPath,
33+
})
34+
35+
// Set up the Netlify function (this is ONLY for preview mode)
36+
const relativePath = getFilePathForRoute(srcRoute || route, 'js')
37+
const filePath = join('pages', relativePath)
38+
39+
// Skip if we have already set up a function for this file
40+
41+
if (filePathsDone.has(filePath)) {
42+
return
43+
}
44+
filePathsDone.add(filePath)
45+
46+
// or if the source route has a fallback (handled by getStaticPropsWithFallback)
47+
if (await isRouteWithFallback(srcRoute)) {
48+
return
49+
}
50+
jobs.push({ type: 'function', filePath, functionsPath })
51+
}),
52+
)
53+
return jobs
4954
}
5055

5156
module.exports = setup

‎src/lib/pages/getStaticPropsWithFallback/setup.js

Lines changed: 3 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,7 @@
11
const { join } = require('path')
22

3-
const asyncForEach = require('../../helpers/asyncForEach')
43
const getFilePathForRoute = require('../../helpers/getFilePathForRoute')
5-
const { logTitle, logItem } = require('../../helpers/logger')
6-
const setupNetlifyFunctionForPage = require('../../helpers/setupNetlifyFunctionForPage')
4+
const { logTitle } = require('../../helpers/logger')
75

86
const getPages = require('./pages')
97

@@ -14,11 +12,10 @@ const setup = async (functionsPath) => {
1412
const pages = await getPages()
1513

1614
// Create Netlify Function for every page
17-
await asyncForEach(pages, async ({ route }) => {
15+
return pages.map(({ route }) => {
1816
const relativePath = getFilePathForRoute(route, 'js')
1917
const filePath = join('pages', relativePath)
20-
logItem(filePath)
21-
await setupNetlifyFunctionForPage({ filePath, functionsPath, isISR: true })
18+
return { type: 'function', filePath, functionsPath, isISR: true }
2219
})
2320
}
2421

‎src/lib/pages/getStaticPropsWithRevalidate/setup.js

Lines changed: 12 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,7 @@
11
const { join } = require('path')
22

3-
const asyncForEach = require('../../helpers/asyncForEach')
43
const getFilePathForRoute = require('../../helpers/getFilePathForRoute')
5-
const { logTitle, logItem } = require('../../helpers/logger')
6-
const setupNetlifyFunctionForPage = require('../../helpers/setupNetlifyFunctionForPage')
4+
const { logTitle } = require('../../helpers/logger')
75

86
const getPages = require('./pages')
97

@@ -17,23 +15,24 @@ const setup = async (functionsPath) => {
1715

1816
// Keep track of the functions that have been set up, so that we do not set up
1917
// a function for the same file path twice
20-
const filePathsDone = []
18+
const filePathsDone = new Set()
2119

2220
const pages = await getPages()
2321

2422
// Create Netlify Function for every page
25-
await asyncForEach(pages, async ({ route, srcRoute }) => {
26-
const relativePath = getFilePathForRoute(srcRoute || route, 'js')
27-
const filePath = join('pages', relativePath)
2823

29-
// Skip if we have already set up a function for this file
30-
if (filePathsDone.includes(filePath)) return
24+
const jobs = []
3125

32-
// Set up the function
33-
logItem(filePath)
34-
await setupNetlifyFunctionForPage({ filePath, functionsPath })
35-
filePathsDone.push(filePath)
26+
pages.forEach(({ route, srcRoute }) => {
27+
const relativePath = getFilePathForRoute(srcRoute || route, 'js')
28+
const filePath = join('pages', relativePath)
29+
if (filePathsDone.has(filePath)) {
30+
return
31+
}
32+
filePathsDone.add(filePath)
33+
jobs.push({ type: 'function', filePath, functionsPath })
3634
})
35+
return jobs
3736
}
3837

3938
module.exports = setup

0 commit comments

Comments
 (0)