Skip to content

Commit 06a41be

Browse files
implement webpack chunks file updating using ast manipulation (#12)
* implement webpack chunks file updating using ast manipulation this allows us not to have to require the use of the `serverMinification: false` option (as we're no longer relying on known variable names) * remove no longer required `serverMinification: false` options * update TODO and builder/README files --------- Co-authored-by: Pete Bacon Darwin <[email protected]>
1 parent 87e4032 commit 06a41be

23 files changed

+1667
-245
lines changed

.prettierignore

+2
Original file line numberDiff line numberDiff line change
@@ -2,3 +2,5 @@
22
.wrangler
33
pnpm-lock.yaml
44
.vscode/setting.json
5+
test-fixtures
6+
test-snapshots

TODO.md

+1-15
Original file line numberDiff line numberDiff line change
@@ -15,27 +15,13 @@ DONE:
1515

1616
- `npx create-next-app@latest <app-name> --use-npm` (use npm to avoid symlinks)
1717

18-
- update next.config.mjs as follows
19-
20-
```typescript
21-
/** @type {import('next').NextConfig} */
22-
const nextConfig = {
23-
output: "standalone",
24-
experimental: {
25-
serverMinification: false,
26-
},
27-
};
28-
29-
export default nextConfig;
30-
```
31-
3218
- add the following devDependency to the package.json:
3319

3420
```json
3521
"wrangler": "^3.78.6"
3622
```
3723

38-
- add a wrangler.toml int the generated app
24+
- add a wrangler.toml into the generated app
3925

4026
```toml
4127
#:schema node_modules/wrangler/config-schema.json

builder/README.md

-14
Original file line numberDiff line numberDiff line change
@@ -2,20 +2,6 @@
22

33
## Build your app
44

5-
- update the `next.config.mjs` as follows
6-
7-
```typescript
8-
/** @type {import('next').NextConfig} */
9-
const nextConfig = {
10-
output: "standalone",
11-
experimental: {
12-
serverMinification: false,
13-
},
14-
};
15-
16-
export default nextConfig;
17-
```
18-
195
- add the following `devDependency` to the `package.json`:
206

217
```json

builder/package.json

+12-8
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,9 @@
44
"version": "0.0.1",
55
"scripts": {
66
"build": "tsup",
7-
"build:watch": "tsup --watch src"
7+
"build:watch": "tsup --watch src",
8+
"test": "vitest --run",
9+
"test:watch": "vitest"
810
},
911
"bin": "dist/index.mjs",
1012
"files": [
@@ -27,12 +29,14 @@
2729
},
2830
"homepage": "https://github.com/flarelabs-net/poc-next",
2931
"devDependencies": {
30-
"@cloudflare/workers-types": "^4.20240909.0",
31-
"@types/node": "^22.2.0",
32-
"esbuild": "^0.23.0",
33-
"glob": "^11.0.0",
34-
"next": "14.2.5",
35-
"tsup": "^8.2.4",
36-
"typescript": "^5.5.4"
32+
"@types/node": "catalog:",
33+
"esbuild": "catalog:",
34+
"glob": "catalog:",
35+
"tsup": "catalog:",
36+
"typescript": "catalog:",
37+
"vitest": "catalog:"
38+
},
39+
"dependencies": {
40+
"ts-morph": "catalog:"
3741
}
3842
}

builder/src/build/build-worker.ts

+2-44
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
import { NextjsAppPaths } from "../nextjs-paths";
22
import { build, Plugin } from "esbuild";
3-
import { readdirSync, readFileSync, writeFileSync } from "node:fs";
3+
import { readFileSync } from "node:fs";
44
import { cp, readFile, writeFile } from "node:fs/promises";
55

66
import { patchRequire } from "./patches/investigated/patch-require";
@@ -11,6 +11,7 @@ import { patchFindDir } from "./patches/to-investigate/patch-find-dir";
1111
import { inlineNextRequire } from "./patches/to-investigate/inline-next-require";
1212
import { inlineEvalManifest } from "./patches/to-investigate/inline-eval-manifest";
1313
import { patchWranglerDeps } from "./patches/to-investigate/wrangler-deps";
14+
import { updateWebpackChunksFile } from "./patches/investigated/update-webpack-chunks-file";
1415

1516
/**
1617
* Using the Next.js build output in the `.next` directory builds a workerd compatible output
@@ -155,49 +156,6 @@ async function updateWorkerBundledCode(
155156
await writeFile(workerOutputFile, patchedCode);
156157
}
157158

158-
/**
159-
* Fixes the webpack-runtime.js file by removing its webpack dynamic requires.
160-
*
161-
* This hack is especially bad for two reasons:
162-
* - it requires setting `experimental.serverMinification` to `false` in the app's config file
163-
* - indicates that files inside the output directory still get a hold of files from the outside: `${nextjsAppPaths.standaloneAppServerDir}/webpack-runtime.js`
164-
* so this shows that not everything that's needed to deploy the application is in the output directory...
165-
*/
166-
async function updateWebpackChunksFile(nextjsAppPaths: NextjsAppPaths) {
167-
console.log("# updateWebpackChunksFile");
168-
const webpackRuntimeFile = `${nextjsAppPaths.standaloneAppServerDir}/webpack-runtime.js`;
169-
170-
console.log({ webpackRuntimeFile });
171-
172-
const fileContent = readFileSync(webpackRuntimeFile, "utf-8");
173-
174-
const chunks = readdirSync(`${nextjsAppPaths.standaloneAppServerDir}/chunks`)
175-
.filter((chunk) => /^\d+\.js$/.test(chunk))
176-
.map((chunk) => {
177-
console.log(` - chunk ${chunk}`);
178-
return chunk.replace(/\.js$/, "");
179-
});
180-
181-
const updatedFileContent = fileContent.replace(
182-
"__webpack_require__.f.require = (chunkId, promises) => {",
183-
`__webpack_require__.f.require = (chunkId, promises) => {
184-
if (installedChunks[chunkId]) return;
185-
${chunks
186-
.map(
187-
(chunk) => `
188-
if (chunkId === ${chunk}) {
189-
installChunk(require("./chunks/${chunk}.js"));
190-
return;
191-
}
192-
`
193-
)
194-
.join("\n")}
195-
`
196-
);
197-
198-
writeFileSync(webpackRuntimeFile, updatedFileContent);
199-
}
200-
201159
function createFixRequiresESBuildPlugin(templateDir: string): Plugin {
202160
return {
203161
name: "replaceRelative",
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
import { readFile } from "node:fs/promises";
2+
3+
import { expect, test, describe } from "vitest";
4+
5+
import { getChunkInstallationIdentifiers } from "./get-chunk-installation-identifiers";
6+
import { tsParseFile } from "../../../utils";
7+
8+
describe("getChunkInstallationIdentifiers", () => {
9+
test("gets chunk identifiers from unminified code", async () => {
10+
const fileContent = await readFile(
11+
`${import.meta.dirname}/test-fixtures/unminified-webpacks-file.js`,
12+
"utf8"
13+
);
14+
const tsSourceFile = tsParseFile(fileContent);
15+
const { installChunk, installedChunks } = await getChunkInstallationIdentifiers(tsSourceFile);
16+
expect(installChunk).toEqual("installChunk");
17+
expect(installedChunks).toEqual("installedChunks");
18+
});
19+
20+
test("gets chunk identifiers from minified code", async () => {
21+
const fileContent = await readFile(
22+
`${import.meta.dirname}/test-fixtures/minified-webpacks-file.js`,
23+
"utf8"
24+
);
25+
const tsSourceFile = tsParseFile(fileContent);
26+
const { installChunk, installedChunks } = await getChunkInstallationIdentifiers(tsSourceFile);
27+
expect(installChunk).toEqual("r");
28+
expect(installedChunks).toEqual("e");
29+
});
30+
});
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,102 @@
1+
import * as ts from "ts-morph";
2+
3+
/**
4+
* Gets the names of the variables that in the unminified webpack runtime file are called `installedChunks` and `installChunk`.
5+
*
6+
* Variables example: https://github.com/webpack/webpack/blob/dae16ad11e/examples/module-worker/README.md?plain=1#L256-L282
7+
*
8+
* @param sourceFile the webpack runtime file parsed with ts-morph
9+
* @returns an object containing the two variable names
10+
*/
11+
export async function getChunkInstallationIdentifiers(sourceFile: ts.SourceFile): Promise<{
12+
installedChunks: string;
13+
installChunk: string;
14+
}> {
15+
const installChunkDeclaration = getInstallChunkDeclaration(sourceFile);
16+
const installedChunksDeclaration = getInstalledChunksDeclaration(sourceFile, installChunkDeclaration);
17+
18+
return {
19+
installChunk: installChunkDeclaration.getName(),
20+
installedChunks: installedChunksDeclaration.getName(),
21+
};
22+
}
23+
24+
/**
25+
* Gets the declaration for what in the unminified webpack runtime file is called `installChunk`(which is a function that registers the various chunks.
26+
*
27+
* `installChunk` example: https://github.com/webpack/webpack/blob/dae16ad11e/examples/module-worker/README.md?plain=1#L263-L282
28+
*
29+
* @param sourceFile the webpack runtime file parsed with ts-morph
30+
* @returns the `installChunk` declaration
31+
*/
32+
function getInstallChunkDeclaration(sourceFile: ts.SourceFile): ts.VariableDeclaration {
33+
const installChunkDeclaration = sourceFile
34+
.getDescendantsOfKind(ts.SyntaxKind.VariableDeclaration)
35+
.find((declaration) => {
36+
const arrowFunction = declaration.getInitializerIfKind(ts.SyntaxKind.ArrowFunction);
37+
// we're looking for an arrow function
38+
if (!arrowFunction) return false;
39+
40+
const functionParameters = arrowFunction.getParameters();
41+
// the arrow function we're looking for has a single parameter (the chunkId)
42+
if (functionParameters.length !== 1) return false;
43+
44+
const arrowFunctionBodyBlock = arrowFunction.getFirstChildByKind(ts.SyntaxKind.Block);
45+
46+
// the arrow function we're looking for has a block body
47+
if (!arrowFunctionBodyBlock) return false;
48+
49+
const statementKinds = arrowFunctionBodyBlock.getStatements().map((statement) => statement.getKind());
50+
51+
// the function we're looking for has 2 for loops (a standard one and a for-in one)
52+
const forInStatements = statementKinds.filter((s) => s === ts.SyntaxKind.ForInStatement);
53+
const forStatements = statementKinds.filter((s) => s === ts.SyntaxKind.ForStatement);
54+
if (forInStatements.length !== 1 || forStatements.length !== 1) return false;
55+
56+
// the function we're looking for accesses its parameter three times, and it
57+
// accesses its `modules`, `ids` and `runtime` properties (in this order)
58+
const parameterName = functionParameters[0].getText();
59+
const functionParameterAccessedProperties = arrowFunctionBodyBlock
60+
.getDescendantsOfKind(ts.SyntaxKind.PropertyAccessExpression)
61+
.filter(
62+
(propertyAccessExpression) => propertyAccessExpression.getExpression().getText() === parameterName
63+
)
64+
.map((propertyAccessExpression) => propertyAccessExpression.getName());
65+
if (functionParameterAccessedProperties.join(", ") !== "modules, ids, runtime") return false;
66+
67+
return true;
68+
});
69+
70+
if (!installChunkDeclaration) {
71+
throw new Error("ERROR: unable to find the installChunk function declaration");
72+
}
73+
74+
return installChunkDeclaration;
75+
}
76+
77+
/**
78+
* Gets the declaration for what in the unminified webpack runtime file is called `installedChunks` which is an object that holds the various registered chunks.
79+
*
80+
* `installedChunks` example: https://github.com/webpack/webpack/blob/dae16ad11e/examples/module-worker/README.md?plain=1#L256-L261
81+
*
82+
* @param sourceFile the webpack runtime file parsed with ts-morph
83+
* @param installChunkDeclaration the declaration for the `installChunk` variable
84+
* @returns the `installedChunks` declaration
85+
*/
86+
function getInstalledChunksDeclaration(
87+
sourceFile: ts.SourceFile,
88+
installChunkDeclaration: ts.VariableDeclaration
89+
): ts.VariableDeclaration {
90+
const allVariableDeclarations = sourceFile.getDescendantsOfKind(ts.SyntaxKind.VariableDeclaration);
91+
const installChunkDeclarationIdx = allVariableDeclarations.findIndex(
92+
(declaration) => declaration === installChunkDeclaration
93+
);
94+
95+
// the installedChunks declaration comes right before the installChunk one
96+
const installedChunksDeclaration = allVariableDeclarations[installChunkDeclarationIdx - 1];
97+
98+
if (!installedChunksDeclaration?.getInitializer()?.isKind(ts.SyntaxKind.ObjectLiteralExpression)) {
99+
throw new Error("ERROR: unable to find the installedChunks declaration");
100+
}
101+
return installedChunksDeclaration;
102+
}
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
import { readFile } from "node:fs/promises";
2+
3+
import { expect, test, describe } from "vitest";
4+
5+
import { getFileContentWithUpdatedWebpackFRequireCode } from "./get-file-content-with-updated-webpack-f-require-code";
6+
import { tsParseFile } from "../../../utils";
7+
8+
describe("getFileContentWithUpdatedWebpackFRequireCode", () => {
9+
test("returns the updated content of the f.require function from unminified webpack runtime code", async () => {
10+
const fileContent = await readFile(
11+
`${import.meta.dirname}/test-fixtures/unminified-webpacks-file.js`,
12+
"utf8"
13+
);
14+
const tsSourceFile = tsParseFile(fileContent);
15+
const updatedFCode = await getFileContentWithUpdatedWebpackFRequireCode(
16+
tsSourceFile,
17+
{ installChunk: "installChunk", installedChunks: "installedChunks" },
18+
["658"]
19+
);
20+
expect(unstyleCode(updatedFCode)).toContain(`if (installedChunks[chunkId]) return;`);
21+
expect(unstyleCode(updatedFCode)).toContain(
22+
`if (chunkId === 658) return installChunk(require("./chunks/658.js"));`
23+
);
24+
});
25+
26+
test("returns the updated content of the f.require function from minified webpack runtime code", async () => {
27+
const fileContent = await readFile(
28+
`${import.meta.dirname}/test-fixtures/minified-webpacks-file.js`,
29+
"utf8"
30+
);
31+
const tsSourceFile = tsParseFile(fileContent);
32+
const updatedFCode = await getFileContentWithUpdatedWebpackFRequireCode(
33+
tsSourceFile,
34+
{ installChunk: "r", installedChunks: "e" },
35+
["658"]
36+
);
37+
expect(unstyleCode(updatedFCode)).toContain("if (e[o]) return;");
38+
expect(unstyleCode(updatedFCode)).toContain(`if (o === 658) return r(require("./chunks/658.js"));`);
39+
});
40+
});
41+
42+
function unstyleCode(text: string): string {
43+
return text.replace(/\n\s+/g, "\n").replace(/\n/g, " ");
44+
}

0 commit comments

Comments
 (0)