Skip to content

Commit

Permalink
Optimize deploy (#193)
Browse files Browse the repository at this point in the history 
* add folder-hash
* optimize build + deploy, skip if not changed
* remove old hash return values, debug log for unchanged actions
* added useForce param which is set to true when --force-deploy flag is set. +tests:100%
  • Loading branch information
@purplecabbage
purplecabbage authored Sep 25, 2024
1 parent 759cb6d commit 7128745
Show file tree
Hide file tree
Showing 7 changed files with 139 additions and 91 deletions.
1 change: 1 addition & 0 deletions package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
"@adobe/aio-lib-env": "^3.0.0",
"archiver": "^6.0.1",
"execa": "^4.0.3",
"folder-hash": "^4.0.4",
"fs-extra": "^11.1.1",
"globby": "^11.0.1",
"js-yaml": "^4.1.0",
Expand Down
93 changes: 56 additions & 37 deletions src/build-actions.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,13 +11,14 @@ governing permissions and limitations under the License.
*/

const fs = require('fs-extra')
const path = require('path')
const path = require('node:path')
const webpack = require('webpack')
const globby = require('globby')
const utils = require('./utils')
const aioLogger = require('@adobe/aio-lib-core-logging')('@adobe/aio-lib-runtime:action-builder', { provider: 'debug' })
const cloneDeep = require('lodash.clonedeep')
const { getCliEnv } = require('@adobe/aio-lib-env')
const { hashElement } = require('folder-hash')

const uniqueArr = (items) => {
return [...new Set(items)]
Expand Down Expand Up @@ -164,9 +165,9 @@ const loadWebpackConfig = async (configPath, actionPath, tempBuildDir, outBuildF
* @returns {Promise<ActionBuild>} Relevant data for the zip process..
*/
const prepareToBuildAction = async (action, root, dist) => {
// dist is something like ext-id/actions/ typically
const { name: actionName, defaultPackage, packageName } = action
const zipFileName = utils.getActionZipFileName(packageName, actionName, false)
let statsInfo // this is the object returned by bundler run, it has the hash
// path.resolve supports both relative and absolute action.function
const actionPath = path.resolve(root, action.function)
const outPath = path.join(dist, `${zipFileName}.zip`)
Expand All @@ -187,17 +188,23 @@ const prepareToBuildAction = async (action, root, dist) => {
}
})

// quick helper
const filePathExists = (dir, file) => {
return fs.existsSync(path.join(dir, file))
}

const actionDir = path.dirname(actionPath)
const srcHash = await hashElement(actionDir, { folders: { exclude: ['node_modules'] } })
if (isDirectory) {
// make sure package.json exists OR index.js
const packageJsonPath = path.join(actionPath, 'package.json')
if (!fs.existsSync(packageJsonPath)) {
if (!fs.existsSync(path.join(actionPath, 'index.js'))) {
throw new Error(`missing required ${utils._relApp(root, packageJsonPath)} or index.js for folder actions`)
// make sure package.json exists OR index.js exists
if (!filePathExists(actionPath, 'package.json')) {
if (!filePathExists(actionPath, 'index.js')) {
throw new Error('missing required package.json or index.js for folder actions')
}
aioLogger.debug('action directory has an index.js, allowing zip')
} else {
// make sure package.json exposes main or there is an index.js
const expectedActionName = utils.getActionEntryFile(packageJsonPath)
const expectedActionName = utils.getActionEntryFile(path.join(actionPath, 'package.json'))
if (!fs.existsSync(path.join(actionPath, expectedActionName))) {
throw new Error(`the directory ${action.function} must contain either a package.json with a 'main' flag or an index.js file at its root`)
}
Expand All @@ -210,15 +217,17 @@ const prepareToBuildAction = async (action, root, dist) => {
const webpackConfig = await loadWebpackConfig(webpackConfigPath, actionPath, tempBuildDir, 'index.js')
const compiler = webpack(webpackConfig)

// run the compiler and wait for a result
statsInfo = await new Promise((resolve, reject) => {
// run the compiler and wait
await new Promise((resolve, reject) => {
compiler.run((err, stats) => {
if (err) {
reject(err)
}
// stats must be defined at this point
const info = stats.toJson()
if (stats.hasWarnings()) {
// this might need to be evaluated, in most cases the user would not see this but
// probably should by default
aioLogger.warn(`webpack compilation warnings:\n${info.warnings}`)
}
if (stats.hasErrors()) {
Expand All @@ -229,21 +238,11 @@ const prepareToBuildAction = async (action, root, dist) => {
})
}

let buildHash
let contentHash
if (isDirectory) {
contentHash = actionFileStats.mtime.valueOf()
buildHash = { [zipFileName]: contentHash }
} else {
contentHash = statsInfo.hash
buildHash = { [zipFileName]: contentHash }
}

return {
actionName,
buildHash,
legacy: defaultPackage,
outPath,
srcHash,
tempBuildDir,
tempActionName: 'index.js'
}
Expand All @@ -257,10 +256,9 @@ const prepareToBuildAction = async (action, root, dist) => {
* @param {Array<ActionBuild>} buildsList Array of data about actions available to be zipped.
* @param {string} lastBuildsPath Path to the last built actions data.
* @param {string} distFolder Path to the output root.
* @param {boolean} skipCheck If true, zip all the actions from the buildsList
* @returns {string[]} Array of zipped actions.
*/
const zipActions = async (buildsList, lastBuildsPath, distFolder, skipCheck) => {
const zipActions = async (buildsList, lastBuildsPath, distFolder) => {
let dumpData = {}
const builtList = []
let lastBuiltData = ''
Expand All @@ -270,22 +268,17 @@ const zipActions = async (buildsList, lastBuildsPath, distFolder, skipCheck) =>
for (const build of buildsList) {
const { outPath, buildHash, tempBuildDir } = build
aioLogger.debug(`action buildHash ${JSON.stringify(buildHash)}`)
const previouslyBuilt = utils.actionBuiltBefore(lastBuiltData, buildHash)
if (!previouslyBuilt || skipCheck) {
aioLogger.debug(`action ${build.actionName} has changed since last build, zipping`)
dumpData = { ...dumpData, ...buildHash }
await utils.zip(tempBuildDir, outPath)
builtList.push(outPath)
} else {
aioLogger.debug(`action ${build.actionName} was not modified since last build, skipping`)
}
aioLogger.debug(`action ${build.actionName} has changed since last build, zipping`)
dumpData = { ...dumpData, ...buildHash }
await utils.zip(tempBuildDir, outPath)
builtList.push(outPath)
}
const parsedLastBuiltData = utils.safeParse(lastBuiltData)
await utils.dumpActionsBuiltInfo(lastBuildsPath, dumpData, parsedLastBuiltData)
return builtList
}

const buildActions = async (config, filterActions, skipCheck = false, emptyDist = true) => {
const buildActions = async (config, filterActions, skipCheck = false, emptyDist = false) => {
if (!config.app.hasBackend) {
throw new Error('cannot build actions, app has no backend')
}
Expand All @@ -296,30 +289,56 @@ const buildActions = async (config, filterActions, skipCheck = false, emptyDist
// If using old format of <actionname>, convert it to <package>/<actionname> using default/first package in the manifest
sanitizedFilterActions = sanitizedFilterActions.map(actionName => actionName.indexOf('/') === -1 ? modifiedConfig.ow.package + '/' + actionName : actionName)
}
// action specific, ext-id/actions/
const distFolder = config.actions.dist

// clear out dist dir
if (emptyDist) {
fs.emptyDirSync(distFolder)
}
const toBuildList = []
const lastBuiltActionsPath = path.join(config.root, 'dist', 'last-built-actions.json')
let lastBuiltData = {}
if (fs.existsSync(lastBuiltActionsPath)) {
lastBuiltData = await fs.readJson(lastBuiltActionsPath)
}

for (const [pkgName, pkg] of Object.entries(modifiedConfig.manifest.full.packages)) {
const actionsToBuild = Object.entries(pkg.actions || {})
// build all sequentially (todo make bundler execution parallel)
for (const [actionName, action] of actionsToBuild) {
const actionFullName = pkgName + '/' + actionName
// here we check if this action should be skipped
if (Array.isArray(sanitizedFilterActions) && !sanitizedFilterActions.includes(actionFullName)) {
continue
}
action.name = actionName
action.packageName = pkgName
action.defaultPackage = modifiedConfig.ow.package === pkgName
toBuildList.push(await prepareToBuildAction(action, config.root, distFolder))

// here we should check if there are changes since the last build
const actionPath = path.resolve(config.root, action.function)
const actionDir = path.dirname(actionPath)

// get a hash of the current action folder
const srcHash = await hashElement(actionDir, { folders: { exclude: ['node_modules'] } })
// lastBuiltData[actionName] === contentHash
// if the flag to skip is set, then we ALWAYS build
// if the hash is different, we build
// if the user has specified a filter, we build even if hash is the same, they are explicitly asking for it
// but we don't need to add a case, before we are called, skipCheck is set to true if there is a filter
if (skipCheck || lastBuiltData[actionFullName] !== srcHash.hash) {
// todo: inform the user that the action has changed and we are rebuilding
// console.log('action has changed since last build, zipping', actionFullName)
const buildResult = await prepareToBuildAction(action, config.root, distFolder)
buildResult.buildHash = { [actionFullName]: srcHash.hash }
toBuildList.push(buildResult)
} else {
// inform the user that the action has not changed ???
aioLogger.debug(`action ${actionFullName} has not changed`)
}
}
}

return zipActions(toBuildList, lastBuiltActionsPath, distFolder, skipCheck)
return zipActions(toBuildList, lastBuiltActionsPath, distFolder)
}

module.exports = buildActions
54 changes: 47 additions & 7 deletions src/deploy-actions.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,13 +18,13 @@ const aioLogger = require('@adobe/aio-lib-core-logging')('@adobe/aio-lib-runtime
const IOruntime = require('./RuntimeAPI')
const PACKAGE_ITEMS = ['actions', 'sequences']
const FILTERABLE_ITEMS = ['apis', 'triggers', 'rules', 'dependencies', ...PACKAGE_ITEMS]

const { createHash } = require('node:crypto')
/**
* runs the command
*
* @param {object} config app config
* @param {object} [deployConfig={}] deployment config
* @param {boolean} [deployConfig.isLocalDev] local dev flag
* @param {boolean} [deployConfig.isLocalDev] local dev flag // todo: remove
* @param {object} [deployConfig.filterEntities] add filters to deploy only specified OpenWhisk entities
* @param {Array} [deployConfig.filterEntities.actions] filter list of actions to deploy by provided array, e.g. ['name1', ..]
* @param {boolean} [deployConfig.filterEntities.byBuiltActions] if true, trim actions from the manifest based on the already built actions
Expand All @@ -33,13 +33,17 @@ const FILTERABLE_ITEMS = ['apis', 'triggers', 'rules', 'dependencies', ...PACKAG
* @param {Array} [deployConfig.filterEntities.rules] filter list of rules to deploy, e.g. ['name1', ..]
* @param {Array} [deployConfig.filterEntities.apis] filter list of apis to deploy, e.g. ['name1', ..]
* @param {Array} [deployConfig.filterEntities.dependencies] filter list of package dependencies to deploy, e.g. ['name1', ..]
* @param {boolean} [deployConfig.useForce] force deploy of actions
* @param {object} [logFunc] custom logger function
* @returns {Promise<object>} deployedEntities
*/
async function deployActions (config, deployConfig = {}, logFunc) {
if (!config.app.hasBackend) throw new Error('cannot deploy actions, app has no backend')
if (!config.app.hasBackend) {
throw new Error('cannot deploy actions, app has no backend')
}

const isLocalDev = deployConfig.isLocalDev
const isLocalDev = deployConfig.isLocalDev // todo: remove
const useForce = deployConfig.useForce
const log = logFunc || console.log
let filterEntities = deployConfig.filterEntities

Expand All @@ -64,6 +68,8 @@ async function deployActions (config, deployConfig = {}, logFunc) {
aioLogger.debug('Trimming out the manifest\'s actions...')
filterEntities = undefined
const builtActions = []
// this is a little weird, we are getting the list of built actions from the dist folder
// instead of it being passed, or simply reading manifest/config
const distFiles = fs.readdirSync(path.resolve(__dirname, dist))
distFiles.forEach(distFile => {
const packageFolder = path.resolve(__dirname, dist, distFile)
Expand Down Expand Up @@ -121,12 +127,14 @@ async function deployActions (config, deployConfig = {}, logFunc) {
}
})
}

// 2. deploy manifest
const deployedEntities = await deployWsk(
modifiedConfig,
manifest,
log,
filterEntities
filterEntities,
useForce
)
// enrich actions array with urls
if (Array.isArray(deployedEntities.actions)) {
Expand All @@ -148,9 +156,11 @@ async function deployActions (config, deployConfig = {}, logFunc) {
* @param {object} manifestContent manifest
* @param {object} logFunc custom logger function
* @param {object} filterEntities entities (actions, sequences, triggers, rules etc) to be filtered
* @param {boolean} useForce force deploy of actions
* @returns {Promise<object>} deployedEntities
*/
async function deployWsk (scriptConfig, manifestContent, logFunc, filterEntities) {
async function deployWsk (scriptConfig, manifestContent, logFunc, filterEntities, useForce) {
// note, logFunc is always defined here because we can only ever be called by deployActions
const packageName = scriptConfig.ow.package
const manifestPath = scriptConfig.manifest.src
const owOptions = {
Expand All @@ -160,6 +170,17 @@ async function deployWsk (scriptConfig, manifestContent, logFunc, filterEntities
namespace: scriptConfig.ow.namespace
}

const lastDeployedActionsPath = path.join(scriptConfig.root, 'dist', 'last-deployed-actions.json')
let lastDeployData = {}
if (useForce) {
logFunc('Force deploy enabled, skipping last deployed actions')
} else if (fs.existsSync(lastDeployedActionsPath)) {
lastDeployData = await fs.readJson(lastDeployedActionsPath)
} else {
// we will create it later
logFunc('lastDeployedActions not found, it will be created after first deployment')
}

const ow = await new IOruntime().init(owOptions)

/**
Expand Down Expand Up @@ -204,6 +225,25 @@ async function deployWsk (scriptConfig, manifestContent, logFunc, filterEntities
// note we must filter before processPackage, as it expect all built actions to be there
const entities = utils.processPackage(packages, {}, {}, {}, false, owOptions)

// entities.actions is an array of actions
entities.actions = entities.actions?.filter(action => {
// action name here includes the package name, ie manyactions/__secured_generic1
const hash = createHash('sha256')
hash.update(JSON.stringify(action))
const actionHash = hash.digest('hex')
if (lastDeployData[action.name] !== actionHash) {
lastDeployData[action.name] = actionHash
return true
}
lastDeployData[action.name] = actionHash
return false
})

fs.ensureFileSync(lastDeployedActionsPath)
fs.writeJSONSync(lastDeployedActionsPath,
lastDeployData,
{ spaces: 2 })

// Note1: utils.processPackage sets the headless-v2 validator for all
// require-adobe-auth annotated actions. Here, we have the context on whether
// an app has a frontend or not.
Expand All @@ -224,7 +264,6 @@ async function deployWsk (scriptConfig, manifestContent, logFunc, filterEntities
}
const DEFAULT_VALIDATOR = DEFAULT_VALIDATORS[env]
const APP_REGISTRY_VALIDATOR = APP_REGISTRY_VALIDATORS[env]

const replaceValidator = { [DEFAULT_VALIDATOR]: APP_REGISTRY_VALIDATOR }
entities.actions.forEach(a => {
const needsReplacement = a.exec && a.exec.kind === 'sequence' && a.exec.components && a.exec.components.includes(DEFAULT_VALIDATOR)
Expand All @@ -236,6 +275,7 @@ async function deployWsk (scriptConfig, manifestContent, logFunc, filterEntities
}

// do the deployment, manifestPath and manifestContent needed for creating a project hash
//
await utils.syncProject(packageName, manifestPath, manifestContent, entities, ow, logFunc, scriptConfig.imsOrgId, deleteOldEntities)
return entities
}
Expand Down
20 changes: 0 additions & 20 deletions src/utils.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2039,25 +2039,6 @@ function activationLogBanner (logFunc, activation, activationLogs) {
}
}

/**
* Will tell if the action was built before based on it's contentHash.
*
* @param {string} lastBuildsData Data with the last builds
* @param {object} buildData Object where key is the name of the action and value is its contentHash
* @returns {boolean} true if the action was built before
*/
function actionBuiltBefore (lastBuildsData, buildData) {
if (buildData && Object.keys(buildData).length > 0) {
const [actionName, contentHash] = Object.entries(buildData)[0]
const storedData = safeParse(lastBuildsData)
if (contentHash) {
return storedData[actionName] === contentHash
}
}
aioLogger.debug('actionBuiltBefore > Invalid actionBuiltData')
return false
}

/**
* Will dump the previously actions built data information.
*
Expand Down Expand Up @@ -2148,7 +2129,6 @@ module.exports = {
getActionZipFileName,
getActionNameFromZipFile,
dumpActionsBuiltInfo,
actionBuiltBefore,
safeParse,
isSupportedActionKind,
DEFAULT_PACKAGE_RESERVED_NAME
Expand Down
Loading

0 comments on commit 7128745

Please sign in to comment.