Adds support for Workers Sites

bin/cloudworker.js

@@ -20,9 +20,12 @@ program
   .option('-d, --debug', 'Debug', false)
   .option('-s, --kv-set [variable.key=value]', 'Binds variable to a local implementation of Workers KV and sets key to value', collect, [])
   .option('-f, --kv-file [variable=path]', 'Set the filepath for value peristence for the local implementation of Workers KV', collect, [])
+  .option('-b, --bind [variable=value]', 'Binds variable to the value provided', collect, [])
+  .option('-a, --bind-file [variable=path]', 'Binds variable to the contents of the given file', collect, [])
   .option('-w, --wasm [variable=path]', 'Binds variable to wasm located at path', collect, [])
   .option('-c, --enable-cache', 'Enables cache <BETA>', false)
   .option('-r, --watch', 'Watch the worker script and restart the worker when changes are detected', false)
+  .option('-t, --site-bucket [path]', 'Simulate Workers Sites deploy by generating KV assets and bindings for the bucket at path', false)
   .option('-s, --set [variable.key=value]', '(Deprecated) Binds variable to a local implementation of Workers KV and sets key to value', collect, [])
   .action(f => { file = f })
   .parse(process.argv)
@@ -40,12 +43,14 @@ wasmLoader.loadBindings(wasmBindings).then(res => {
   process.exit(1)
 })
 
-function run (file, wasmBindings) {
+async function run (file, wasmBindings) {
   console.log('Starting up...')
   const fullpath = path.resolve(process.cwd(), file)
   const script = utils.read(fullpath)
-  const bindings = utils.extractKVBindings(program.kvSet.concat(program.set), program.kvFile)
-  Object.assign(bindings, wasmBindings)
+  const { siteKvFile, siteManifest } = await utils.generateSite(program.siteBucket)
+  const kvBindings = utils.extractKVBindings(program.kvSet.concat(program.set), program.kvFile.concat(siteKvFile))
+  const bindings = utils.extractBindings(program.bind, program.bindFile.concat(siteManifest))
+  Object.assign(bindings, kvBindings, wasmBindings)
 
   // Add a warning log for deprecation
   if (program.set.length > 0) console.warn('Warning: Flag --set is now deprecated, please use --kv-set instead')

lib/__tests__/utils.test.js

@@ -35,4 +35,128 @@ describe('utils', () => {
 
     cb()
   })
+
+  test('extractKVBindings throws on invalid filepath format', async () => {
+    expect(() => { utils.extractKVBindings([], ['invalid format']) }).toThrow()
+  })
+
+  test('extractKVBindings handles files and sets', async (cb) => {
+    const kv = new KeyValueStore(path.resolve('test-kv.json'))
+    kv.put('hello', 'world')
+
+    const bindings = utils.extractKVBindings(['test.this=great'], ['test=test-kv.json'])
+    const {test} = bindings
+
+    fs.unlinkSync(kv.path)
+    expect(await test.get('hello')).toEqual('world')
+    expect(await test.get('this')).toEqual('great')
+
+    cb()
+  })
+
+  test('extractBindings throws on invalid format', async () => {
+    expect(() => { utils.extractBindings(['invalid format'], []) }).toThrow()
+  })
+
+  test('extractBindings parses properly', async () => {
+    const bindings = utils.extractBindings(['foo=bar', 'baz=qux'], [])
+    expect(bindings.foo).toEqual('bar')
+    expect(bindings.baz).toEqual('qux')
+  })
+
+  test('extractBindings allows = in value', async () => {
+    const bindings = utils.extractBindings(['foo="const bar=\'abc\';"'], [])
+    expect(bindings.foo).toEqual('"const bar=\'abc\';"')
+  })
+
+  test('extractBindings handles file as binding value', async () => {
+    const content = JSON.stringify({
+      foo: 'abc',
+      bar: 12345,
+      baz: {
+        qux: ['a', 'b', 'c', 'd'],
+        plugh: [ { id: 987 }, { id: 876 } ],
+      },
+    })
+    const path = '/tmp/__cloudworker_test.json'
+    await fs.writeFileSync(path, content)
+
+    const bindings = utils.extractBindings([], [`__DATA=${path}`])
+    expect(bindings.__DATA).toEqual(content)
+
+    await fs.unlinkSync(path)
+  })
+
+  test('extractBindings throws on invalid file format', async () => {
+    expect(() => { utils.extractBindings([], ['invalid file format']) }).toThrow()
+  })
+
+  test('extractBindings throws on nonexistent path', async () => {
+    expect(() => { utils.extractBindings([], ['foo=/tmp/__cloudworker_fake_file.json']) }).toThrow()
+  })
+
+  test('parseWasmFlags throws on invalid format', async () => {
+    expect(() => { utils.parseWasmFlags(['invalid format']) }).toThrow()
+  })
+
+  test('parseWasmFlags parses properly', async () => {
+    const bindings = utils.parseWasmFlags(['foo=bar'])
+    expect(bindings.foo).toEqual('bar')
+  })
+
+  test('generateSite throws on nonexistent path', async () => {
+    expect.assertions(1)
+    try {
+      await utils.generateSite('/not/a/real/bucket/path')
+    } catch (e) {
+      expect(e).toBeDefined()
+    }
+  })
+
+  describe('generateSite', () => {
+    const path = '/tmp/__cloudworker_test'
+    const subdir = '/subdir'
+
+    beforeAll(async () => {
+      await fs.mkdirSync(path)
+      await fs.mkdirSync(`${path}${subdir}`)
+      await fs.writeFileSync(`${path}/a`, 'a123')
+      await fs.writeFileSync(`${path}${subdir}/b`, 'b456')
+    })
+
+    test('generateSite recurses down a directory tree', async () => {
+      const { siteKvFile, siteManifest } = await utils.generateSite(path)
+
+      const siteKVFileContent = fs.readFileSync(siteKvFile.split('=')[1], 'utf-8')
+      const siteManifestContent = fs.readFileSync(siteManifest.split('=')[1], 'utf-8')
+
+      expect(siteKVFileContent).toEqual("{\n  \"a\": \"a123\",\n  \"subdir/b\": \"b456\"\n}\n") // eslint-disable-line
+      expect(siteManifestContent).toEqual(JSON.stringify({
+        'a': 'a',
+        'subdir/b': 'subdir/b',
+      }))
+    })
+
+    test('handles a path with a trailing slash the same', async () => {
+      const { siteKvFile, siteManifest } = await utils.generateSite(path)
+
+      const siteKVFileContent = fs.readFileSync(siteKvFile.split('=')[1], 'utf-8')
+      const siteManifestContent = fs.readFileSync(siteManifest.split('=')[1], 'utf-8')
+
+      const { siteKvFile: siteKvFile2, siteManifest: siteManifest2 } = await utils.generateSite(`${path}/`)
+
+      const siteKVFileContent2 = fs.readFileSync(siteKvFile2.split('=')[1], 'utf-8')
+      const siteManifestContent2 = fs.readFileSync(siteManifest2.split('=')[1], 'utf-8')
+
+      expect(siteKVFileContent2).toEqual(siteKVFileContent)
+      expect(siteManifestContent2).toEqual(siteManifestContent)
+    })
+
+    afterAll(async () => {
+      await fs.unlinkSync(`${path}/a`)
+      await fs.unlinkSync(`${path}${subdir}/b`)
+      await fs.rmdirSync(`${path}${subdir}`)
+      await fs.rmdirSync(path)
+    })
+  })
 })

lib/utils.js

@@ -1,64 +1,176 @@
 const fs = require('fs')
+const {promisify} = require('util')
+const pathModule = require('path')
+const tmp = require('tmp')
 const {KeyValueStore} = require('./kv')
 module.exports.read = f => fs.readFileSync(f).toString('utf-8')
 module.exports.extractKVBindings = (kvSetFlags, kvFileFlags) => {
   const bindings = {}
 
-  const filepaths = extractKVPaths(kvFileFlags)
+  const filepaths = parseFlags('kv-file', kvFileFlags)
 
   for (const [variable, path] of Object.entries(filepaths)) {
     bindings[variable] = new KeyValueStore(path)
   }
 
-  for (const flag of kvSetFlags) {
-    const comps = flag.split('.')
-    if (comps.length < 2 || comps[1].split('=').length < 2) {
-      throw new Error('Invalid kv-set flag format. Expected format of [variable].[key]=[value]')
-    }
-
-    const variable = comps[0]
-    const kvFragment = comps.slice(1).join('.')
-    const kvComponents = kvFragment.split('=')
-    const key = kvComponents[0]
-    const value = kvComponents.slice(1).join('=')
+  const kvStores = parseFlags('kv-set', kvSetFlags, true)
 
+  for (const [variable, obj] of Object.entries(kvStores)) {
     if (!bindings[variable]) {
       bindings[variable] = new KeyValueStore(filepaths[variable])
     }
 
-    bindings[variable].put(key, value)
+    for (const [key, value] of Object.entries(obj)) {
+      bindings[variable].put(key, value)
+    }
   }
 
   return bindings
 }
 
-const extractKVPaths = (kvFileFlags) => {
-  const paths = {}
+module.exports.extractBindings = (bindingFlags, bindingFileFlags) => {
+  return Object.assign(
+    {},
+    parseFlags('bind', bindingFlags),
+    parseFlags('bind-file', bindingFileFlags, false, (variable, filepath) => {
+      if (!fs.existsSync(filepath)) {
+        throw new Error(`Invalid bind-file path "${filepath}"`)
+      }
 
-  if (!kvFileFlags) return paths
+      return fs.readFileSync(filepath).toString('utf-8')
+    })
+  )
+}
 
-  for (const flag of kvFileFlags) {
-    const components = flag.split('=')
+module.exports.parseWasmFlags = (wasmFlags) => parseFlags('wasm', wasmFlags)
 
-    if (flag.length < 2) {
-      throw new Error('Invalid kv-file flag format. Expected format of [variable]=[value]')
-    }
+/**
+ * Parse flags into bindings.
+ *
+ * @param {string} type Type of binding being parsed.
+ * @param {Array<string>} flags Command line flags to parse.
+ * @param {boolean} objectVariable Whether the variable represents an object name and key
+ * @param {Function} handleVariable Function to call for custom variable/value handling.
+ *
+ * @returns {Object} bindings The variable bindings parsed from the flags.
+ */
+function parseFlags (type, flags = [], objectVariable = false, handleVariable = null) {
+  const bindings = {}
+
+  for (const flag of flags) {
+    if (objectVariable) {
+      const comps = flag.split('.')
+      if (comps.length < 2 || comps[1].split('=').length < 2) {
+        throw new Error(`Invalid ${type} flag format. Expected format of [variable].[key]=[value]`)
+      }
 
-    paths[components[0]] = components[1]
+      const variable = comps[0]
+      const kvFragment = comps.slice(1).join('.')
+      const kvComponents = kvFragment.split('=')
+      const key = kvComponents[0]
+      const value = kvComponents.slice(1).join('=')
+
+      bindings[variable] = Object.assign({}, bindings[variable], { [key]: value })
+    } else {
+      const comps = flag.split('=')
+      if (comps.length < 2) {
+        throw new Error(`Invalid ${type} flag format. Expected format of [variable]=[value]`)
+      }
+
+      const variable = comps[0]
+      const value = comps.slice(1).join('=')
+      if (handleVariable) bindings[variable] = handleVariable(variable, value)
+      else bindings[variable] = value
+    }
   }
 
-  return paths
+  return bindings
 }
 
-module.exports.parseWasmFlags = (wasmFlags) => {
-  const bindings = {}
-  for (const flag of wasmFlags) {
-    const comps = flag.split('=')
-    if (comps.length !== 2) {
-      throw new Error('Invalid wasm flag format. Expected format of [variable=path]')
+/**
+ * Recursively generate a list of files in a given directory and its subdirectories.
+ *
+ * @param {string} path The path to the directory. Should end with a trailing separator.
+ *
+ * @returns {Array<string>} List of files in the directory and its subdirectories, relative to path.
+ */
+async function getDirEntries (path) {
+  const readdir = promisify(fs.readdir)
+
+  let entries = []
+
+  const dir = await readdir(path, { withFileTypes: true })
+  for (const entry of dir) {
+    if (entry.isDirectory()) {
+      entries = entries.concat(await getDirEntries(`${path}${entry.name}${pathModule.sep}`))
+    } else {
+      entries.push(`${path}${entry.name}`)
     }
-    const [variable, path] = comps
-    bindings[variable] = path
   }
-  return bindings
+
+  return entries
+}
+
+/*
+ * Create a JSON file suitable for populating an in-memory KV store, as well as a manifest
+ * file mapping paths to entries in the KV store.
+ *
+ * @param {string} path The path to the directory to generate the KV file JSON from.
+ * @param {string} file The path to the JSON file.
+ *
+ * @returns {Object}
+ *   @property {string} file The path to the KV store JSON file.
+ *   @property {string} manifestFile The path to the manifest binding JSON file.
+ */
+async function populateKVFile (path, file) {
+  const appendFile = promisify(fs.appendFile)
+  const readFile = promisify(fs.readFile)
+  if (path.slice(-1) !== pathModule.sep) path += pathModule.sep
+  const entries = await getDirEntries(path)
+
+  await appendFile(file, '{\n')
+  let entry, contents
+  for (let index = 0; index < entries.length; index++) {
+    entry = entries[index]
+    contents = await readFile(`${entry}`, { encoding: 'utf-8' })
+    entry = entry.replace(path, '')
+
+    await appendFile(
+      file,
+      `  ${JSON.stringify(entry)}: ${JSON.stringify(contents)}` +
+      (index < entries.length - 1 ? ',' : '') +
+      '\n'
+    )
+  }
+  await appendFile(file, '}\n')
+
+  const manifestFile = file.replace('.json', '-manifest.json')
+  const manifestKeys = {}
+  for (const key of entries) {
+    const shortKey = key.replace(path, '')
+    manifestKeys[shortKey] = shortKey
+  }
+
+  await appendFile(manifestFile, JSON.stringify(manifestKeys))
+
+  return { file, manifestFile }
+}
+
+/**
+ * Generate the necessary KV store and binding for a static site using the
+ * assets in a given directory.
+ *
+ * @param {string} bucketPath The path to the static site's asset bucket.
+ *
+ * @returns {Object}
+ *   @property {string} siteKvFile The --kv-file param for the static content KV file.
+ *   @property {string} siteManifest The --bind-file param for the static content manifest file.
+ */
+module.exports.generateSite = async (bucketPath) => {
+  const tmpFile = tmp.fileSync({ prefix: 'cloudworker-', postfix: '.json' })
+  const { file, manifestFile } = await populateKVFile(bucketPath, tmpFile.name)
+  return {
+    siteKvFile: `__STATIC_CONTENT=${file}`,
+    siteManifest: `__STATIC_CONTENT_MANIFEST=${manifestFile}`,
+  }
 }