feat: use new blob store api (#100)

* feat: use new blob store api

* chore: fix tests

* chore: install deno for e2e

* chore: change blobs dir

Author: lukasholzer

.github/workflows/run-tests.yml

@@ -17,6 +17,11 @@ jobs:
       - uses: oven-sh/setup-bun@v1
       - name: Setup PNPM
         uses: ./.github/actions/pnpm
+      - name: Install Deno
+        uses: denoland/setup-deno@v1
+        with:
+          # Should match the `DENO_VERSION_RANGE` from https://github.com/netlify/edge-bundler/blob/e55f825bd985d3c92e21d1b765d71e70d5628fba/node/bridge.ts#L17
+          deno-version: v1.37.0
       - name: 'Install dependencies'
         run: npm ci
       - name: 'Netlify Login'

package-lock.json

@@ -1,6 +1,6 @@
 {
   "name": "@netlify/next-runtime",
-  "version": "5.0.0-alpha.26",
+  "version": "5.0.0-alpha.27",
   "description": "Run Next.js seamlessly on Netlify",
   "main": "./dist/index.js",
   "type": "module",
@@ -42,7 +42,7 @@
   "devDependencies": {
     "@fastly/http-compute-js": "1.1.1",
     "@netlify/blobs": "^6.3.1",
-    "@netlify/build": "^29.29.4",
+    "@netlify/build": "^29.31.0",
     "@netlify/edge-functions": "^2.2.0",
     "@netlify/eslint-config-node": "^7.0.1",
     "@netlify/functions": "^2.4.0",

package.json

@@ -6,13 +6,13 @@ import { defineConfig, devices } from '@playwright/test'
 export default defineConfig({
   testDir: './tests/e2e',
   /* Run tests in files in parallel */
-  fullyParallel: true,
+  fullyParallel: false,
   /* Fail the build on CI if you accidentally left test.only in the source code. */
   forbidOnly: !!process.env.CI,
   /* Retry on CI only */
   retries: process.env.CI ? 2 : 0,
-  /* Opt out of parallel tests on CI. */
-  workers: process.env.CI ? 1 : undefined,
+  /* Limit the number of workers on CI, use default locally */
+  workers: process.env.CI ? 3 : undefined,
   /* Reporter to use. See https://playwright.dev/docs/test-reporters */
   reporter: 'html',
   /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */

playwright.config.ts

@@ -10,8 +10,11 @@ const pkg = JSON.parse(readFileSync(join(PLUGIN_DIR, 'package.json'), 'utf-8'))
 export const PLUGIN_NAME = pkg.name
 export const PLUGIN_VERSION = pkg.version
 
+/** The directory that is published to Netlify */
 export const STATIC_DIR = '.netlify/static'
 export const TEMP_DIR = '.netlify/temp'
+/** The directory inside the publish directory that will be uploaded by build to the blob store */
+export const BLOB_DIR = '.netlify/blobs/deploy'
 
 export const SERVER_FUNCTIONS_DIR = '.netlify/functions-internal'
 export const SERVER_HANDLER_NAME = '___netlify-server-handler'

src/build/blob.ts

@@ -1,13 +1,10 @@
 import type { NetlifyPluginOptions } from '@netlify/build'
 import glob from 'fast-glob'
-import type { PrerenderManifest } from 'next/dist/build/index.js'
-import { readFile } from 'node:fs/promises'
+import { mkdir, readFile, writeFile } from 'node:fs/promises'
 import { basename, dirname, extname, resolve } from 'node:path'
 import { join as joinPosix } from 'node:path/posix'
-import { cpus } from 'os'
-import pLimit from 'p-limit'
-import { getBlobStore } from '../blob.js'
 import { getPrerenderManifest } from '../config.js'
+import { BLOB_DIR } from '../constants.js'
 
 export type CacheEntry = {
   key: string
@@ -124,44 +121,31 @@ const buildPrerenderedContentEntries = async (
  * Upload prerendered content to the blob store and remove it from the bundle
  */
 export const uploadPrerenderedContent = async ({
-  constants: { PUBLISH_DIR, NETLIFY_API_TOKEN, NETLIFY_API_HOST, SITE_ID },
-}: Pick<NetlifyPluginOptions, 'constants'>) => {
-  // limit concurrent uploads to 2x the number of CPUs
-  const limit = pLimit(Math.max(2, cpus().length))
-
-  // read prerendered content and build JSON key/values for the blob store
-  let manifest: PrerenderManifest
-  let blob: ReturnType<typeof getBlobStore>
+  constants: { PUBLISH_DIR },
+  utils,
+}: Pick<NetlifyPluginOptions, 'constants' | 'utils'>) => {
   try {
-    manifest = await getPrerenderManifest({ PUBLISH_DIR })
-    blob = getBlobStore({ NETLIFY_API_TOKEN, NETLIFY_API_HOST, SITE_ID })
-  } catch (error: any) {
-    console.error(`Unable to upload prerendered content: ${error.message}`)
-    return
+    // read prerendered content and build JSON key/values for the blob store
+    const manifest = await getPrerenderManifest({ PUBLISH_DIR })
+    const entries = await Promise.all(
+      await buildPrerenderedContentEntries(PUBLISH_DIR, Object.keys(manifest.routes)),
+    )
+
+    // movce JSON content to the blob store directory for upload
+    await Promise.all(
+      entries
+        .filter((entry) => entry.value.value !== undefined)
+        .map(async (entry) => {
+          const dest = resolve(BLOB_DIR, entry.key)
+          await mkdir(dirname(dest), { recursive: true })
+          await writeFile(resolve(BLOB_DIR, entry.key), JSON.stringify(entry.value), 'utf-8')
+        }),
+    )
+  } catch (error) {
+    utils.build.failBuild(
+      'Failed assembling prerendered content for upload',
+      error instanceof Error ? { error } : {},
+    )
+    throw error
   }
-  const entries = await Promise.allSettled(
-    await buildPrerenderedContentEntries(PUBLISH_DIR, Object.keys(manifest.routes)),
-  )
-  entries.forEach((result) => {
-    if (result.status === 'rejected') {
-      console.error(`Unable to read prerendered content: ${result.reason.message}`)
-    }
-  })
-
-  // upload JSON content data to the blob store
-  const uploads = await Promise.allSettled(
-    entries
-      .filter((entry) => entry.status === 'fulfilled' && entry.value.value.value !== undefined)
-      .map((entry: PromiseSettledResult<CacheEntry>) => {
-        const result = entry as PromiseFulfilledResult<CacheEntry>
-        const { key, value } = result.value
-        return limit(() => blob.setJSON(key, value))
-      }),
-  )
-  uploads.forEach((upload, index) => {
-    if (upload.status === 'rejected') {
-      const result = entries[index] as PromiseFulfilledResult<CacheEntry>
-      console.error(`Unable to store ${result.value?.key}: ${upload.reason.message}`)
-    }
-  })
 }

src/build/constants.ts

@@ -1,26 +1,20 @@
-import type { NetlifyPluginOptions } from '@netlify/build'
+import type {
+  NetlifyPluginConstants,
+  NetlifyPluginOptions,
+  NetlifyPluginUtils,
+} from '@netlify/build'
 import glob from 'fast-glob'
 import { Mock, afterEach, beforeEach, expect, test, vi } from 'vitest'
 import { mockFileSystem } from '../../../tests/index.js'
 import { FixtureTestContext, createFsFixture } from '../../../tests/utils/fixture.js'
-import { getBlobStore } from '../blob.js'
-import { STATIC_DIR } from '../constants.js'
+import { BLOB_DIR, STATIC_DIR } from '../constants.js'
 import { copyStaticAssets, uploadStaticContent } from './static.js'
+import * as fs from 'fs'
+import { join } from 'path'
 
-afterEach(() => {
-  vi.restoreAllMocks()
-})
-
-vi.mock('../blob.js', () => ({
-  getBlobStore: vi.fn(),
-}))
-
-let mockBlobSet = vi.fn()
-beforeEach(() => {
-  ;(getBlobStore as Mock).mockReturnValue({
-    set: mockBlobSet,
-  })
-})
+const utils = {
+  build: { failBuild: vi.fn() },
+} as unknown as NetlifyPluginUtils
 
 test('should clear the static directory contents', async () => {
   const PUBLISH_DIR = '.next'
@@ -30,8 +24,8 @@ test('should clear the static directory contents', async () => {
   })
 
   await copyStaticAssets({
-    constants: { PUBLISH_DIR },
-  } as Pick<NetlifyPluginOptions, 'constants'>)
+    constants: { PUBLISH_DIR } as NetlifyPluginConstants,
+  })
 
   expect(Object.keys(vol.toJSON())).toEqual(
     expect.not.arrayContaining([`${STATIC_DIR}/remove-me.js`]),
@@ -50,8 +44,8 @@ test<FixtureTestContext>('should link static content from the publish directory
   )
 
   await copyStaticAssets({
-    constants: { PUBLISH_DIR },
-  } as Pick<NetlifyPluginOptions, 'constants'>)
+    constants: { PUBLISH_DIR } as NetlifyPluginConstants,
+  })
 
   const files = await glob('**/*', { cwd, dot: true })
 
@@ -77,11 +71,10 @@ test<FixtureTestContext>('should link static content from the public directory t
   )
 
   await copyStaticAssets({
-    constants: { PUBLISH_DIR },
-  } as Pick<NetlifyPluginOptions, 'constants'>)
+    constants: { PUBLISH_DIR } as NetlifyPluginConstants,
+  })
 
   const files = await glob('**/*', { cwd, dot: true })
-
   expect(files).toEqual(
     expect.arrayContaining([
       'public/another-asset.json',
@@ -94,7 +87,6 @@ test<FixtureTestContext>('should link static content from the public directory t
 
 test<FixtureTestContext>('should copy the static pages to the publish directory if the routes do not exist in the prerender-manifest', async (ctx) => {
   const PUBLISH_DIR = '.next'
-
   const { cwd } = await createFsFixture(
     {
       [`${PUBLISH_DIR}/prerender-manifest.json`]: JSON.stringify({
@@ -108,12 +100,14 @@ test<FixtureTestContext>('should copy the static pages to the publish directory
   )
 
   await uploadStaticContent({
-    constants: { PUBLISH_DIR },
-  } as Pick<NetlifyPluginOptions, 'constants'>)
+    constants: { PUBLISH_DIR } as NetlifyPluginConstants,
+    utils,
+  })
 
-  expect(mockBlobSet).toHaveBeenCalledTimes(2)
-  expect(mockBlobSet).toHaveBeenCalledWith('server/pages/test.html', 'test-1')
-  expect(mockBlobSet).toHaveBeenCalledWith('server/pages/test2.html', 'test-2')
+  expect((await glob('**/*', { cwd: join(cwd, BLOB_DIR), dot: true })).sort()).toEqual([
+    'server/pages/test.html',
+    'server/pages/test2.html',
+  ])
 })
 
 test<FixtureTestContext>('should not copy the static pages to the publish directory if the routes exist in the prerender-manifest', async (ctx) => {
@@ -135,8 +129,9 @@ test<FixtureTestContext>('should not copy the static pages to the publish direct
   )
 
   await uploadStaticContent({
-    constants: { PUBLISH_DIR },
-  } as Pick<NetlifyPluginOptions, 'constants'>)
+    constants: { PUBLISH_DIR } as NetlifyPluginConstants,
+    utils,
+  })
 
-  expect(mockBlobSet).not.toHaveBeenCalled()
+  expect(await glob('**/*', { cwd: join(cwd, BLOB_DIR), dot: true })).toHaveLength(0)
 })