Skip to content

Commit

Permalink
SITES-28176 - Support product URLs with only urlKeys (#18)
Browse files Browse the repository at this point in the history 
* Introduce path format and fetching products by urlKey

* Fixes and testing

* SITES-28179 - Add support for locale URLs (#19)

* Add support for locale URLs

* Allow config overrides from context variable

* Adjust poller to use same configuration parameters

* Fixes

* Use HLX_LOCALES for allowedLocales

* Update mapLocale to throw instead of using a default value
  • Loading branch information
@herzog31
herzog31 authored Jan 24, 2025
1 parent e442dfb commit 5b08368
Show file tree
Hide file tree
Showing 16 changed files with 703 additions and 265 deletions.
22 changes: 11 additions & 11 deletions actions/check-product-changes/lib/aem.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -123,17 +123,17 @@ class AdminAPI {

doPreview(item) {
this.trackInFlight(`preview ${item.record.path}`, async (complete) => {
const { log } = this.context;
const { logger } = this.context;
const { record } = item;
const start = new Date();

try {
record.previewedAt = new Date();
await this.execAdminRequest('POST', 'preview', record.path);
log.info(`Previewed ${record.path}`);
logger.info(`Previewed ${record.path}`);
this.publishQueue.push(item);
} catch (e) {
log.error(e);
logger.error(e);
// only resolve the item promise in case of an error
item.resolve(record);
} finally {
Expand All @@ -145,18 +145,18 @@ class AdminAPI {

doPublish(items) {
this.trackInFlight(`publish ${items.length}`, async (complete) => {
const { log } = this.context;
const { logger } = this.context;

try {
const paths = items.map(({ record }) => record.path);
const body = { forceUpdate: false, paths };
await this.execAdminRequest('POST', 'live', '/*', body);
log.info(`Published ${items.length} items`);
logger.info(`Published ${items.length} items`);

// set published date after publishing done
items.forEach(({ record }) => record.publishedAt = new Date());
} catch (e) {
log.error(e);
logger.error(e);
} finally {
complete();
// resolve the original promises
Expand All @@ -167,16 +167,16 @@ class AdminAPI {

doUnpublishAndDelete(item) {
this.trackInFlight(`unpublish ${item.record.path}`, async (complete) => {
const { log } = this.context;
const { logger } = this.context;
const { record } = item;

try {
await this.execAdminRequest('DELETE', 'live', record.path);
await this.execAdminRequest('DELETE', 'preview', record.path);
log.info(`Unpublished ${record.path}`);
logger.info(`Unpublished ${record.path}`);
record.deletedAt = new Date();
} catch (e) {
log.error(e);
logger.error(e);
} finally {
complete();
item.resolve(record);
Expand All @@ -186,8 +186,8 @@ class AdminAPI {

processQueues() {
if (this.lastStatusLog < new Date() - 60000) {
const { log } = this.context;
log.info(`Queues: preview=${this.previewQueue.length},`
const { logger } = this.context;
logger.info(`Queues: preview=${this.previewQueue.length},`
+ ` publish=${this.publishQueue.length},`
+ ` unpublish=${this.unpublishQueue.length},`
+ ` inflight=${this.inflight.length}`);
Expand Down
72 changes: 38 additions & 34 deletions actions/check-product-changes/poller.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,18 +12,18 @@ governing permissions and limitations under the License.

const { Timings, aggregate } = require('./lib/benchmark');
const { AdminAPI } = require('./lib/aem');
const { requestSaaS, requestSpreadsheet, isValidUrl } = require('../utils');
const { requestSaaS, requestSpreadsheet, isValidUrl, getProductUrl, mapLocale } = require('../utils');
const { GetAllSkusQuery, GetLastModifiedQuery } = require('../queries');
const { Core } = require('@adobe/aio-sdk');

const BATCH_SIZE = 50;

async function loadState(storeCode, stateLib) {
const stateKey = storeCode ? `${storeCode}` : 'default';
async function loadState(locale, stateLib) {
const stateKey = locale ? `${locale}` : 'default';
const stateData = await stateLib.get(stateKey);
if (!stateData?.value) {
return {
storeCode,
locale,
skusLastQueriedAt: new Date(0),
skus: {},
};
Expand All @@ -34,7 +34,7 @@ async function loadState(storeCode, stateLib) {
// folloed by a pair of SKUs and timestamps which are the last preview times per SKU
const [catalogQueryTimestamp, ...skus] = stateData && stateData.value ? stateData.value.split(',') : [0];
return {
storeCode,
locale,
skusLastQueriedAt: new Date(parseInt(catalogQueryTimestamp)),
skus: Object.fromEntries(skus
.map((sku, i, arr) => (i % 2 === 0 ? [sku, new Date(parseInt(arr[i + 1]))] : null))
Expand All @@ -43,11 +43,11 @@ async function loadState(storeCode, stateLib) {
}

async function saveState(state, stateLib) {
let { storeCode } = state;
if (!storeCode) {
storeCode = 'default';
let { locale } = state;
if (!locale) {
locale = 'default';
}
const stateKey = `${storeCode}`;
const stateKey = `${locale}`;
const stateData = [
state.skusLastQueriedAt.getTime(),
...Object.entries(state.skus).flatMap(([sku, lastPreviewedAt]) => [sku, lastPreviewedAt.getTime()]),
Expand All @@ -61,28 +61,28 @@ async function saveState(state, stateLib) {
* state accordingly.
*
* @param {Object} params - The parameters object.
* @param {string} params.siteName - The name of the site (repo or repoless).
* @param {string} params.PDPURIPrefix - The URI prefix for Product Detail Pages.
* @param {string} params.HLX_SITE_NAME - The name of the site (repo or repoless).
* @param {string} params.HLX_PATH_FORMAT - The URL format for product detail pages.
* @param {string} params.PLPURIPrefix - The URI prefix for Product List Pages.
* @param {string} params.orgName - The name of the organization.
* @param {string} params.configName - The name of the configuration json/xlsx.
* @param {string} params.HLX_ORG_NAME - The name of the organization.
* @param {string} params.HLX_CONFIG_NAME - The name of the configuration json/xlsx.
* @param {number} [params.requestPerSecond=5] - The number of requests per second allowed by the throttling logic.
* @param {string} params.authToken - The authentication token.
* @param {number} [params.skusRefreshInterval=600000] - The interval for refreshing SKUs in milliseconds.
* @param {string} [params.storeUrl] - The store's base URL.
* @param {string} [params.storeCodes] - Comma-separated list of store codes.
* @param {string} [params.HLX_STORE_URL] - The store's base URL.
* @param {string} [params.HLX_LOCALES] - Comma-separated list of allowed locales.
* @param {string} [params.LOG_LEVEL] - The log level.
* @param {Object} stateLib - The state provider object.
* @returns {Promise<Object>} The result of the polling action.
*/
function checkParams(params) {
const requiredParams = ['siteName', 'PDPURIPrefix', 'PLPURIPrefix', 'orgName', 'configName', 'authToken'];
const requiredParams = ['HLX_SITE_NAME', 'HLX_PATH_FORMAT', 'PLPURIPrefix', 'HLX_ORG_NAME', 'HLX_CONFIG_NAME', 'authToken'];
const missingParams = requiredParams.filter(param => !params[param]);
if (missingParams.length > 0) {
throw new Error(`Missing required parameters: ${missingParams.join(', ')}`);
}

if (params.storeUrl && !isValidUrl(params.storeUrl)) {
if (params.HLX_STORE_URL && !isValidUrl(params.HLX_STORE_URL)) {
throw new Error('Invalid storeUrl');
}
}
Expand Down Expand Up @@ -110,43 +110,47 @@ function shouldProcessProduct(product) {
async function poll(params, stateLib) {
checkParams(params);

const log = Core.Logger('main', { level: params.LOG_LEVEL || 'info' });
const logger = Core.Logger('main', { level: params.LOG_LEVEL || 'info' });
const {
siteName,
PDPURIPrefix,
orgName,
configName,
HLX_SITE_NAME: siteName,
HLX_PATH_FORMAT: pathFormat,
HLX_ORG_NAME: orgName,
HLX_CONFIG_NAME: configName,
requestPerSecond = 5,
authToken,
skusRefreshInterval = 600000,
} = params;
const storeUrl = params.storeUrl ? params.storeUrl : `https://main--${siteName}--${orgName}.aem.live`;
const storeCodes = params.storeCodes ? params.storeCodes.split(',') : [null];
const storeUrl = params.HLX_STORE_URL ? params.HLX_STORE_URL : `https://main--${siteName}--${orgName}.aem.live`;
const locales = params.HLX_LOCALES ? params.HLX_LOCALES.split(',') : [null];

const counts = {
published: 0, unpublished: 0, ignored: 0, failed: 0,
};
const sharedContext = {
storeUrl, configName, log, counts,
storeUrl, configName, logger, counts, pathFormat,
};
const timings = new Timings();
const adminApi = new AdminAPI({
org: orgName,
site: siteName,
}, sharedContext, { requestPerSecond, authToken });

log.info(`Starting poll from ${storeUrl} for store codes ${storeCodes}`);
logger.info(`Starting poll from ${storeUrl} for locales ${locales}`);

try {
// start processing preview and publish queues
await adminApi.startProcessing();

const results = await Promise.all(storeCodes.map(async (storeCode) => {
const results = await Promise.all(locales.map(async (locale) => {
const timings = new Timings();
// load state
const state = await loadState(storeCode, stateLib);
const state = await loadState(locale, stateLib);
timings.sample('loadedState');
const context = { ...sharedContext, storeCode };

let context = { ...sharedContext };
if (locale) {
context = { ...context, ...mapLocale(locale, context) };
}

// setup preview / publish queues

Expand All @@ -173,7 +177,7 @@ async function poll(params, stateLib) {
const skus = Object.keys(state.skus);
const lastModifiedResp = await requestSaaS(GetLastModifiedQuery, 'getLastModified', { skus }, context);
timings.sample('fetchedLastModifiedDates');
log.info(`Fetched last modified date for ${lastModifiedResp.data.products.length} skus, total ${skus.length}`);
logger.info(`Fetched last modified date for ${lastModifiedResp.data.products.length} skus, total ${skus.length}`);

// group preview in batches of 50
let products = lastModifiedResp.data.products
Expand All @@ -198,7 +202,7 @@ async function poll(params, stateLib) {
const batches = products.filter(shouldProcessProduct)
.reduce((acc, product, i, arr) => {
const { sku, urlKey } = product;
const path = (storeCode ? `/${storeCode}${PDPURIPrefix}/${urlKey}/${sku}` : `${PDPURIPrefix}/${urlKey}/${sku}`).toLowerCase();
const path = getProductUrl({ urlKey, sku }, context, false).toLowerCase();
const req = adminApi.previewAndPublish({ path, sku });
acc.push(req);
if (acc.length === BATCH_SIZE || i === arr.length - 1) {
Expand Down Expand Up @@ -251,7 +255,7 @@ async function poll(params, stateLib) {
}
} catch (e) {
// in case the index doesn't yet exist or any other error
log.error(e);
logger.error(e);
}

timings.sample('unpublishedPaths');
Expand All @@ -277,14 +281,14 @@ async function poll(params, stateLib) {
}
timings.measures.previewDuration = aggregate(adminApi.previewDurations);
} catch (e) {
log.error(e);
logger.error(e);
// wait for queues to finish, even in error case
await adminApi.stopProcessing();
}

const elapsed = new Date() - timings.now;

log.info(`Finished polling, elapsed: ${elapsed}ms`);
logger.info(`Finished polling, elapsed: ${elapsed}ms`);

return {
state: 'completed',
Expand Down
79 changes: 58 additions & 21 deletions actions/pdp-renderer/index.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,9 +15,9 @@ const path = require('path');

const { Core } = require('@adobe/aio-sdk')
const Handlebars = require('handlebars');
const { errorResponse, stringParameters, requestSaaS } = require('../utils');
const { errorResponse, stringParameters, requestSaaS, mapLocale } = require('../utils');
const { extractPathDetails, findDescription, prepareBaseTemplate, getPrimaryImage, generatePriceString, getImageList } = require('./lib');
const { ProductQuery } = require('../queries');
const { ProductQuery, ProductByUrlKeyQuery } = require('../queries');
const { generateLdJson } = require('./ldJson');

function toTemplateProductData(baseProduct) {
Expand All @@ -39,41 +39,78 @@ function toTemplateProductData(baseProduct) {
* Parameters
* @param {Object} params The parameters object
* @param {string} params.__ow_path The path of the request
* @param {string} params.__ow_query The query parameters of the request
* @param {string} params.__ow_query.configName Overwrite for HLX_CONFIG_NAME
* @param {string} params.__ow_query.contentUrl Overwrite for HLX_CONTENT_URL
* @param {string} params.__ow_query.storeUrl Overwrite for HLX_STORE_URL
* @param {string} params.__ow_query.productsTemplate Overwrite for HLX_PRODUCTS_TEMPLATE
* @param {string} params.configName Overwrite for HLX_CONFIG_NAME using query parameter
* @param {string} params.contentUrl Overwrite for HLX_CONTENT_URL using query parameter
* @param {string} params.storeUrl Overwrite for HLX_STORE_URL using query parameter
* @param {string} params.productsTemplate Overwrite for HLX_PRODUCTS_TEMPLATE using query parameter
* @param {string} params.pathFormat Overwrite for HLX_PATH_FORMAT using query parameter
* @param {string} params.HLX_CONFIG_NAME The config sheet to use (e.g. configs for prod, configs-dev for dev)
* @param {string} params.HLX_CONTENT_URL Edge Delivery URL of the store (e.g. aem.live)
* @param {string} params.HLX_STORE_URL Public facing URL of the store
* @param {string} params.HLX_PRODUCTS_TEMPLATE URL to the products template page
* @param {string} params.HLX_PATH_FORMAT The path format to use for parsing
*/
async function main (params) {
const logger = Core.Logger('main', { level: params.LOG_LEVEL || 'info' })

try {
logger.debug(stringParameters(params))
const { __ow_path, __ow_query, HLX_STORE_URL, HLX_CONTENT_URL, HLX_CONFIG_NAME, HLX_PRODUCTS_TEMPLATE } = params;
const { sku } = extractPathDetails(__ow_path);

const configName = __ow_query?.configName || HLX_CONFIG_NAME;
const contentUrl = __ow_query?.contentUrl || HLX_CONTENT_URL;
const storeUrl = __ow_query?.storeUrl || HLX_STORE_URL || contentUrl;
const productsTemplate = __ow_query?.productsTemplate || HLX_PRODUCTS_TEMPLATE;
const context = { contentUrl, storeUrl, configName, logger };

if (!sku || !contentUrl) {
const {
__ow_path,
pathFormat : pathFormatQuery,
configName : configNameQuery,
contentUrl : contentUrlQuery,
storeUrl : storeUrlQuery,
productsTemplate : productsTemplateQuery,
HLX_STORE_URL,
HLX_CONTENT_URL,
HLX_CONFIG_NAME,
HLX_PRODUCTS_TEMPLATE,
HLX_PATH_FORMAT,
HLX_LOCALES,
} = params;

const pathFormat = pathFormatQuery || HLX_PATH_FORMAT || '/products/{urlKey}/{sku}';
const configName = configNameQuery || HLX_CONFIG_NAME;
const contentUrl = contentUrlQuery || HLX_CONTENT_URL;
const storeUrl = storeUrlQuery || HLX_STORE_URL || contentUrl;
const productsTemplate = productsTemplateQuery || HLX_PRODUCTS_TEMPLATE;
const allowedLocales = HLX_LOCALES ? HLX_LOCALES.split(',').map(a => a.trim()) : [];
let context = { contentUrl, storeUrl, configName, logger, pathFormat, allowedLocales };

const result = extractPathDetails(__ow_path, pathFormat);
logger.debug('Path parse results', JSON.stringify(result, null, 4));
const { sku, urlKey, locale } = result;

if ((!sku && !urlKey) || !contentUrl) {
return errorResponse(400, 'Invalid path', logger);
}

// Map locale to context
if (locale) {
try {
context = { ...context, ...mapLocale(locale, context) };
// eslint-disable-next-line no-unused-vars
} catch(e) {
return errorResponse(400, 'Invalid locale', logger);
}
}

// Retrieve base product
const baseProductData = await requestSaaS(ProductQuery, 'ProductQuery', { sku }, context);
if (!baseProductData.data.products || baseProductData.data.products.length === 0) {
let baseProduct;
if (sku) {
const baseProductData = await requestSaaS(ProductQuery, 'ProductQuery', { sku: sku.toUpperCase() }, context);
if (!baseProductData?.data?.products || baseProductData?.data?.products?.length === 0) {
return errorResponse(404, 'Product not found', logger);
}
baseProduct = baseProductData.data.products[0];
} else if (urlKey) {
const baseProductData = await requestSaaS(ProductByUrlKeyQuery, 'ProductByUrlKey', { urlKey }, context);
if (!baseProductData?.data?.productSearch || baseProductData?.data?.productSearch?.items?.length === 0) {
return errorResponse(404, 'Product not found', logger);
}
baseProduct = baseProductData.data.productSearch.items[0].productView;
}
const baseProduct = baseProductData.data.products[0];

logger.debug('Retrieved base product', JSON.stringify(baseProduct, null, 4));

// Assign meta tag data for template
Expand Down
8 changes: 4 additions & 4 deletions actions/pdp-renderer/ldJson.js
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
const { requestSaaS } = require('../utils');
const { getProductUrl, findDescription, getPrimaryImage } = require('./lib');
const { requestSaaS, getProductUrl } = require('../utils');
const { findDescription, getPrimaryImage } = require('./lib');
const { VariantsQuery } = require('../queries');

function getOffer(product, url) {
Expand Down Expand Up @@ -64,9 +64,9 @@ async function getVariants(baseProduct, url, axes, context) {
}

async function generateLdJson(product, context) {
const { name, sku, urlKey, __typename } = product;
const { name, sku, __typename } = product;
const image = getPrimaryImage(product);
const url = getProductUrl(urlKey, sku, context);
const url = getProductUrl(product, context);
const gtin = ''; // TODO: Add based on your data model (https://schema.org/gtin)

let ldJson;
Expand Down
Loading

0 comments on commit 5b08368

Please sign in to comment.