Skip to content

Commit 60885d2

Browse files
orinokaiorinokai
authored
feat: cache 404s indefinitely for bot probes (#2668)
* feat: cache 404s indefinitely for bot probes * test: update unit tests * chore: refactor to clean up function signature with object parameterisation
1 parent 0ab5748 commit 60885d2

File tree

4 files changed

+121
-85
lines changed

4 files changed

+121
-85
lines changed

‎src/run/handlers/server.ts

+1-1
Original file line numberDiff line numberDiff line change
@@ -112,7 +112,7 @@ export default async (request: Request, context: FutureContext) => {
112112

113113
await adjustDateHeader({ headers: response.headers, request, span, tracer, requestContext })
114114

115-
setCacheControlHeaders(response.headers, request, requestContext)
115+
setCacheControlHeaders(response, request, requestContext)
116116
setCacheTagsHeaders(response.headers, requestContext)
117117
setVaryHeaders(response.headers, request, nextConfig)
118118
setCacheStatusHeader(response.headers)

‎src/run/headers.test.ts

+105-83
Original file line numberDiff line numberDiff line change
@@ -199,43 +199,43 @@ describe('headers', () => {
199199
const givenHeaders = {
200200
'cdn-cache-control': 'public, max-age=0, must-revalidate',
201201
}
202-
const headers = new Headers(givenHeaders)
203202
const request = new Request(defaultUrl)
204-
vi.spyOn(headers, 'set')
203+
const response = new Response(null, { headers: givenHeaders })
204+
vi.spyOn(response.headers, 'set')
205205

206206
const ctx: RequestContext = { ...createRequestContext(), routeHandlerRevalidate: false }
207-
setCacheControlHeaders(headers, request, ctx)
207+
setCacheControlHeaders(response, request, ctx)
208208

209-
expect(headers.set).toHaveBeenCalledTimes(0)
209+
expect(response.headers.set).toHaveBeenCalledTimes(0)
210210
})
211211

212212
test('should not set any headers if "netlify-cdn-cache-control" is present', () => {
213213
const givenHeaders = {
214214
'netlify-cdn-cache-control': 'public, max-age=0, must-revalidate',
215215
}
216-
const headers = new Headers(givenHeaders)
217216
const request = new Request(defaultUrl)
218-
vi.spyOn(headers, 'set')
217+
const response = new Response(null, { headers: givenHeaders })
218+
vi.spyOn(response.headers, 'set')
219219

220220
const ctx: RequestContext = { ...createRequestContext(), routeHandlerRevalidate: false }
221-
setCacheControlHeaders(headers, request, ctx)
221+
setCacheControlHeaders(response, request, ctx)
222222

223-
expect(headers.set).toHaveBeenCalledTimes(0)
223+
expect(response.headers.set).toHaveBeenCalledTimes(0)
224224
})
225225

226226
test('should mark content as stale if "{netlify-,}cdn-cache-control" is not present and "x-nextjs-cache" is "STALE" (GET)', () => {
227227
const givenHeaders = {
228228
'x-nextjs-cache': 'STALE',
229229
}
230-
const headers = new Headers(givenHeaders)
231230
const request = new Request(defaultUrl)
232-
vi.spyOn(headers, 'set')
231+
const response = new Response(null, { headers: givenHeaders })
232+
vi.spyOn(response.headers, 'set')
233233

234234
const ctx: RequestContext = { ...createRequestContext(), routeHandlerRevalidate: false }
235-
setCacheControlHeaders(headers, request, ctx)
235+
setCacheControlHeaders(response, request, ctx)
236236

237-
expect(headers.set).toHaveBeenCalledTimes(1)
238-
expect(headers.set).toHaveBeenNthCalledWith(
237+
expect(response.headers.set).toHaveBeenCalledTimes(1)
238+
expect(response.headers.set).toHaveBeenNthCalledWith(
239239
1,
240240
'netlify-cdn-cache-control',
241241
'public, max-age=0, must-revalidate, durable',
@@ -246,107 +246,129 @@ describe('headers', () => {
246246
const givenHeaders = {
247247
'x-nextjs-cache': 'STALE',
248248
}
249-
const headers = new Headers(givenHeaders)
250249
const request = new Request(defaultUrl)
251-
vi.spyOn(headers, 'set')
250+
const response = new Response(null, { headers: givenHeaders })
251+
vi.spyOn(response.headers, 'set')
252252

253253
const ctx: RequestContext = { ...createRequestContext(), routeHandlerRevalidate: false }
254-
setCacheControlHeaders(headers, request, ctx)
254+
setCacheControlHeaders(response, request, ctx)
255255

256-
expect(headers.set).toHaveBeenCalledTimes(1)
257-
expect(headers.set).toHaveBeenNthCalledWith(
256+
expect(response.headers.set).toHaveBeenCalledTimes(1)
257+
expect(response.headers.set).toHaveBeenNthCalledWith(
258258
1,
259259
'netlify-cdn-cache-control',
260260
'public, max-age=0, must-revalidate, durable',
261261
)
262262
})
263263

264264
test('should set durable SWC=1yr with 1yr TTL if "{netlify-,}cdn-cache-control" is not present and `revalidate` is `false` (HEAD)', () => {
265-
const headers = new Headers()
266265
const request = new Request(defaultUrl, { method: 'HEAD' })
267-
vi.spyOn(headers, 'set')
266+
const response = new Response()
267+
vi.spyOn(response.headers, 'set')
268268

269269
const ctx: RequestContext = { ...createRequestContext(), routeHandlerRevalidate: false }
270-
setCacheControlHeaders(headers, request, ctx)
270+
setCacheControlHeaders(response, request, ctx)
271271

272-
expect(headers.set).toHaveBeenCalledTimes(1)
273-
expect(headers.set).toHaveBeenNthCalledWith(
272+
expect(response.headers.set).toHaveBeenCalledTimes(1)
273+
expect(response.headers.set).toHaveBeenNthCalledWith(
274274
1,
275275
'netlify-cdn-cache-control',
276276
's-maxage=31536000, stale-while-revalidate=31536000, durable',
277277
)
278278
})
279279

280280
test('should set durable SWC=1yr with given TTL if "{netlify-,}cdn-cache-control" is not present and `revalidate` is a number (GET)', () => {
281-
const headers = new Headers()
282281
const request = new Request(defaultUrl)
283-
vi.spyOn(headers, 'set')
282+
const response = new Response()
283+
vi.spyOn(response.headers, 'set')
284284

285285
const ctx: RequestContext = { ...createRequestContext(), routeHandlerRevalidate: 7200 }
286-
setCacheControlHeaders(headers, request, ctx)
286+
setCacheControlHeaders(response, request, ctx)
287287

288-
expect(headers.set).toHaveBeenCalledTimes(1)
289-
expect(headers.set).toHaveBeenNthCalledWith(
288+
expect(response.headers.set).toHaveBeenCalledTimes(1)
289+
expect(response.headers.set).toHaveBeenNthCalledWith(
290290
1,
291291
'netlify-cdn-cache-control',
292292
's-maxage=7200, stale-while-revalidate=31536000, durable',
293293
)
294294
})
295295

296296
test('should set durable SWC=1yr with 1yr TTL if "{netlify-,}cdn-cache-control" is not present and `revalidate` is a number (HEAD)', () => {
297-
const headers = new Headers()
298297
const request = new Request(defaultUrl, { method: 'HEAD' })
299-
vi.spyOn(headers, 'set')
298+
const response = new Response()
299+
vi.spyOn(response.headers, 'set')
300300

301301
const ctx: RequestContext = { ...createRequestContext(), routeHandlerRevalidate: 7200 }
302-
setCacheControlHeaders(headers, request, ctx)
302+
setCacheControlHeaders(response, request, ctx)
303303

304-
expect(headers.set).toHaveBeenCalledTimes(1)
305-
expect(headers.set).toHaveBeenNthCalledWith(
304+
expect(response.headers.set).toHaveBeenCalledTimes(1)
305+
expect(response.headers.set).toHaveBeenNthCalledWith(
306306
1,
307307
'netlify-cdn-cache-control',
308308
's-maxage=7200, stale-while-revalidate=31536000, durable',
309309
)
310310
})
311311

312312
test('should not set any headers on POST request', () => {
313-
const headers = new Headers()
314313
const request = new Request(defaultUrl, { method: 'POST' })
315-
vi.spyOn(headers, 'set')
314+
const response = new Response()
315+
vi.spyOn(response.headers, 'set')
316316

317317
const ctx: RequestContext = { ...createRequestContext(), routeHandlerRevalidate: false }
318-
setCacheControlHeaders(headers, request, ctx)
318+
setCacheControlHeaders(response, request, ctx)
319319

320-
expect(headers.set).toHaveBeenCalledTimes(0)
320+
expect(response.headers.set).toHaveBeenCalledTimes(0)
321321
})
322322
})
323323

324324
test('should not set any headers if "cache-control" is not set and "requestContext.usedFsRead" is not truthy', () => {
325-
const headers = new Headers()
326325
const request = new Request(defaultUrl)
327-
vi.spyOn(headers, 'set')
326+
const response = new Response()
327+
vi.spyOn(response.headers, 'set')
328328

329-
setCacheControlHeaders(headers, request, createRequestContext())
329+
setCacheControlHeaders(response, request, createRequestContext())
330330

331-
expect(headers.set).toHaveBeenCalledTimes(0)
331+
expect(response.headers.set).toHaveBeenCalledTimes(0)
332332
})
333333

334334
test('should set permanent, durable "netlify-cdn-cache-control" if "cache-control" is not set and "requestContext.usedFsRead" is truthy', () => {
335-
const headers = new Headers()
336335
const request = new Request(defaultUrl)
337-
vi.spyOn(headers, 'set')
336+
const response = new Response()
337+
vi.spyOn(response.headers, 'set')
338+
339+
const requestContext = createRequestContext()
340+
requestContext.usedFsRead = true
341+
342+
setCacheControlHeaders(response, request, requestContext)
343+
344+
expect(response.headers.set).toHaveBeenNthCalledWith(
345+
1,
346+
'cache-control',
347+
'public, max-age=0, must-revalidate',
348+
)
349+
expect(response.headers.set).toHaveBeenNthCalledWith(
350+
2,
351+
'netlify-cdn-cache-control',
352+
'max-age=31536000, durable',
353+
)
354+
})
355+
356+
test('should set permanent, durable "netlify-cdn-cache-control" if 404 response for URl ending in .php', () => {
357+
const request = new Request(`${defaultUrl}/admin.php`)
358+
const response = new Response(null, { status: 404 })
359+
vi.spyOn(response.headers, 'set')
338360

339361
const requestContext = createRequestContext()
340362
requestContext.usedFsRead = true
341363

342-
setCacheControlHeaders(headers, request, requestContext, true)
364+
setCacheControlHeaders(response, request, requestContext)
343365

344-
expect(headers.set).toHaveBeenNthCalledWith(
366+
expect(response.headers.set).toHaveBeenNthCalledWith(
345367
1,
346368
'cache-control',
347369
'public, max-age=0, must-revalidate',
348370
)
349-
expect(headers.set).toHaveBeenNthCalledWith(
371+
expect(response.headers.set).toHaveBeenNthCalledWith(
350372
2,
351373
'netlify-cdn-cache-control',
352374
'max-age=31536000, durable',
@@ -358,45 +380,45 @@ describe('headers', () => {
358380
'cache-control': 'public, max-age=0, must-revalidate',
359381
'cdn-cache-control': 'public, max-age=0, must-revalidate',
360382
}
361-
const headers = new Headers(givenHeaders)
362383
const request = new Request(defaultUrl)
363-
vi.spyOn(headers, 'set')
384+
const response = new Response(null, { headers: givenHeaders })
385+
vi.spyOn(response.headers, 'set')
364386

365-
setCacheControlHeaders(headers, request, createRequestContext())
387+
setCacheControlHeaders(response, request, createRequestContext())
366388

367-
expect(headers.set).toHaveBeenCalledTimes(0)
389+
expect(response.headers.set).toHaveBeenCalledTimes(0)
368390
})
369391

370392
test('should not set any headers if "cache-control" is set and "netlify-cdn-cache-control" is present', () => {
371393
const givenHeaders = {
372394
'cache-control': 'public, max-age=0, must-revalidate',
373395
'netlify-cdn-cache-control': 'public, max-age=0, must-revalidate',
374396
}
375-
const headers = new Headers(givenHeaders)
376397
const request = new Request(defaultUrl)
377-
vi.spyOn(headers, 'set')
398+
const response = new Response(null, { headers: givenHeaders })
399+
vi.spyOn(response.headers, 'set')
378400

379-
setCacheControlHeaders(headers, request, createRequestContext())
401+
setCacheControlHeaders(response, request, createRequestContext())
380402

381-
expect(headers.set).toHaveBeenCalledTimes(0)
403+
expect(response.headers.set).toHaveBeenCalledTimes(0)
382404
})
383405

384406
test('should set expected headers if "cache-control" is set and "cdn-cache-control" and "netlify-cdn-cache-control" are not present (GET request)', () => {
385407
const givenHeaders = {
386408
'cache-control': 'public, max-age=0, must-revalidate',
387409
}
388-
const headers = new Headers(givenHeaders)
389410
const request = new Request(defaultUrl)
390-
vi.spyOn(headers, 'set')
411+
const response = new Response(null, { headers: givenHeaders })
412+
vi.spyOn(response.headers, 'set')
391413

392-
setCacheControlHeaders(headers, request, createRequestContext())
414+
setCacheControlHeaders(response, request, createRequestContext())
393415

394-
expect(headers.set).toHaveBeenNthCalledWith(
416+
expect(response.headers.set).toHaveBeenNthCalledWith(
395417
1,
396418
'cache-control',
397419
'public, max-age=0, must-revalidate',
398420
)
399-
expect(headers.set).toHaveBeenNthCalledWith(
421+
expect(response.headers.set).toHaveBeenNthCalledWith(
400422
2,
401423
'netlify-cdn-cache-control',
402424
'public, max-age=0, must-revalidate, durable',
@@ -407,18 +429,18 @@ describe('headers', () => {
407429
const givenHeaders = {
408430
'cache-control': 'public, max-age=0, must-revalidate',
409431
}
410-
const headers = new Headers(givenHeaders)
411432
const request = new Request(defaultUrl, { method: 'HEAD' })
412-
vi.spyOn(headers, 'set')
433+
const response = new Response(null, { headers: givenHeaders })
434+
vi.spyOn(response.headers, 'set')
413435

414-
setCacheControlHeaders(headers, request, createRequestContext())
436+
setCacheControlHeaders(response, request, createRequestContext())
415437

416-
expect(headers.set).toHaveBeenNthCalledWith(
438+
expect(response.headers.set).toHaveBeenNthCalledWith(
417439
1,
418440
'cache-control',
419441
'public, max-age=0, must-revalidate',
420442
)
421-
expect(headers.set).toHaveBeenNthCalledWith(
443+
expect(response.headers.set).toHaveBeenNthCalledWith(
422444
2,
423445
'netlify-cdn-cache-control',
424446
'public, max-age=0, must-revalidate, durable',
@@ -429,27 +451,27 @@ describe('headers', () => {
429451
const givenHeaders = {
430452
'cache-control': 'public, max-age=0, must-revalidate',
431453
}
432-
const headers = new Headers(givenHeaders)
433454
const request = new Request(defaultUrl, { method: 'POST' })
434-
vi.spyOn(headers, 'set')
455+
const response = new Response(null, { headers: givenHeaders })
456+
vi.spyOn(response.headers, 'set')
435457

436-
setCacheControlHeaders(headers, request, createRequestContext())
458+
setCacheControlHeaders(response, request, createRequestContext())
437459

438-
expect(headers.set).toHaveBeenCalledTimes(0)
460+
expect(response.headers.set).toHaveBeenCalledTimes(0)
439461
})
440462

441463
test('should remove "s-maxage" from "cache-control" header', () => {
442464
const givenHeaders = {
443465
'cache-control': 'public, s-maxage=604800',
444466
}
445-
const headers = new Headers(givenHeaders)
446467
const request = new Request(defaultUrl)
447-
vi.spyOn(headers, 'set')
468+
const response = new Response(null, { headers: givenHeaders })
469+
vi.spyOn(response.headers, 'set')
448470

449-
setCacheControlHeaders(headers, request, createRequestContext())
471+
setCacheControlHeaders(response, request, createRequestContext())
450472

451-
expect(headers.set).toHaveBeenNthCalledWith(1, 'cache-control', 'public')
452-
expect(headers.set).toHaveBeenNthCalledWith(
473+
expect(response.headers.set).toHaveBeenNthCalledWith(1, 'cache-control', 'public')
474+
expect(response.headers.set).toHaveBeenNthCalledWith(
453475
2,
454476
'netlify-cdn-cache-control',
455477
'public, s-maxage=604800, durable',
@@ -460,14 +482,14 @@ describe('headers', () => {
460482
const givenHeaders = {
461483
'cache-control': 'max-age=604800, stale-while-revalidate=86400',
462484
}
463-
const headers = new Headers(givenHeaders)
464485
const request = new Request(defaultUrl)
465-
vi.spyOn(headers, 'set')
486+
const response = new Response(null, { headers: givenHeaders })
487+
vi.spyOn(response.headers, 'set')
466488

467-
setCacheControlHeaders(headers, request, createRequestContext())
489+
setCacheControlHeaders(response, request, createRequestContext())
468490

469-
expect(headers.set).toHaveBeenNthCalledWith(1, 'cache-control', 'max-age=604800')
470-
expect(headers.set).toHaveBeenNthCalledWith(
491+
expect(response.headers.set).toHaveBeenNthCalledWith(1, 'cache-control', 'max-age=604800')
492+
expect(response.headers.set).toHaveBeenNthCalledWith(
471493
2,
472494
'netlify-cdn-cache-control',
473495
'max-age=604800, stale-while-revalidate=86400, durable',
@@ -478,18 +500,18 @@ describe('headers', () => {
478500
const givenHeaders = {
479501
'cache-control': 's-maxage=604800, stale-while-revalidate=86400',
480502
}
481-
const headers = new Headers(givenHeaders)
482503
const request = new Request(defaultUrl)
483-
vi.spyOn(headers, 'set')
504+
const response = new Response(null, { headers: givenHeaders })
505+
vi.spyOn(response.headers, 'set')
484506

485-
setCacheControlHeaders(headers, request, createRequestContext())
507+
setCacheControlHeaders(response, request, createRequestContext())
486508

487-
expect(headers.set).toHaveBeenNthCalledWith(
509+
expect(response.headers.set).toHaveBeenNthCalledWith(
488510
1,
489511
'cache-control',
490512
'public, max-age=0, must-revalidate',
491513
)
492-
expect(headers.set).toHaveBeenNthCalledWith(
514+
expect(response.headers.set).toHaveBeenNthCalledWith(
493515
2,
494516
'netlify-cdn-cache-control',
495517
's-maxage=604800, stale-while-revalidate=86400, durable',

0 commit comments

Comments
 (0)