Skip to content
This repository has been archived by the owner on Feb 13, 2025. It is now read-only.

Commit

Permalink
MWPW-141219 FaaS Report
Browse files Browse the repository at this point in the history
  • Loading branch information
@Brandon32
Brandon32 committed Jan 31, 2024
1 parent d329c92 commit c171b72
Show file tree
Hide file tree
Showing 22 changed files with 1,565 additions and 47 deletions.
49 changes: 34 additions & 15 deletions bulk-update/bulk-update.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,28 +2,42 @@ import fs from 'fs';
import { fetch } from '@adobe/fetch';
import { loadDocument } from './document-manager/document-manager.js';

const delay = (milliseconds) => new Promise((resolve) => { setTimeout(resolve, milliseconds); });

function getJsonData(json, sheetName) {
if (json[':type'] === 'multi-sheet') {
return json[sheetName];
}
return json;
}

/**
* Loads a list of entries from a Query Index.
*
* @param {*} url - The URL to fetch the data from.
* @returns {Promise<string[]>} - List of entries.
*/
export async function loadQueryIndex(url, fetchFunction = fetch) {
export async function loadQueryIndex(url, fetchFunction = fetch, fetchWaitMs = 500) {
console.log(`Loading Query Index from ${url}`);
const entries = [];
await delay(fetchWaitMs);
const response = await fetchFunction(url);

if (!response.ok) {
throw new Error(`Failed to fetch data from ${url}`);
}

const json = await response.json();
const { total, offset, limit, data } = json;
const { total, offset, limit, data } = getJsonData(json, 'sitemap');

entries.push(...data.map((entry) => entry.path));
if (!Array.isArray(data)) throw new Error(`Invalid data format: ${url}`);
entries.push(...data.map((entry) => entry.path || entry.entry || entry.url));
const remaining = total - offset - limit;
if (remaining > 0) {
const nextUrl = `${url}?limit=${limit}&offset=${offset + limit}`;
entries.push(...await loadQueryIndex(nextUrl, fetchFunction));
const nextUrl = new URL(url);
nextUrl.searchParams.set('limit', limit);
nextUrl.searchParams.set('offset', offset + limit);
entries.push(...await loadQueryIndex(nextUrl.toString(), fetchFunction));
}

return entries;
Expand All @@ -39,10 +53,17 @@ export async function loadQueryIndex(url, fetchFunction = fetch) {
* @throws {Error} - If the list format or entry is unsupported.
*/
export async function loadListData(source, fetchFunction = fetch) {
if (!source) return [];
if (Array.isArray(source) || source.includes(',')) {
const entries = Array.isArray(source) ? source : source.split(',');
return (await Promise.all(entries.map((entry) => loadListData(entry.trim())))).flat();
const loadedEntries = [];
for (const entry of entries) {
const loadedData = await loadListData(entry.trim(), fetchFunction);
if (loadedData) loadedEntries.push(...loadedData);
}
return loadedEntries;
}

const extension = source.includes('.') ? source.split('.').pop() : null;

if (!extension) {
Expand All @@ -54,9 +75,9 @@ export async function loadListData(source, fetchFunction = fetch) {
if (source.startsWith('http')) {
return loadQueryIndex(source, fetchFunction);
}
return JSON.parse(fs.readFileSync(source, 'utf8').trim());
return loadListData(JSON.parse(fs.readFileSync(source, 'utf8').trim()), fetchFunction);
case 'txt':
return fs.readFileSync(source, 'utf8').trim().split('\n');
return loadListData(fs.readFileSync(source, 'utf8').trim().split('\n'), fetchFunction);
default:
throw new Error(`Unsupported list format or entry: ${source}`);
}
Expand All @@ -72,18 +93,17 @@ export async function loadListData(source, fetchFunction = fetch) {
* @returns {object} - The totals generated by the reporter.
*/
export default async function main(config, migrate, reporter = null) {
config.reporter ??= reporter;
config.reporter = reporter || config.reporter;

try {
const entryList = await loadListData(config.list);

for (const [i, entry] of entryList.entries()) {
console.log(`Processing entry ${i + 1} of ${entryList.length} ${entry}`);
for (const [i, entry] of config.list.entries()) {
console.log(`Processing entry ${i + 1} of ${config.list.length} ${entry}`);
const document = await loadDocument(entry, config);
await migrate(document);
}
} catch (e) {
console.error('Bulk Update Error:', e);
config.reporter.log('Bulk Update Error', 'error', e.message, e.stack);
}

return config.reporter.generateTotals();
Expand All @@ -101,8 +121,7 @@ if (import.meta.url === `file://${process.argv[1]}`) {
const migrationFile = `${process.cwd()}/${migrationFolder}/migration.js`;
// eslint-disable-next-line import/no-dynamic-require, global-require
const migration = await import(migrationFile);
const config = migration.init();
config.list = list || config.list;
const config = await migration.init(list);

await main(config, migration.migrate);
process.exit(0);
Expand Down
17 changes: 10 additions & 7 deletions bulk-update/document-manager/document-manager.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,12 +29,12 @@ export function entryToPath(entry) {
*
* @param {string} url - The URL to fetch the markdown file from.
* @param {function} reporter - A logging function.
* @param {number} waitMs - The number of milliseconds to wait before fetching the markdown.
* @param {number} fetchWaitMs - The number of milliseconds to wait before fetching the markdown.
* @returns {Promise<string>} A promise that resolves to the fetched markdown.
*/
async function getMarkdown(url, reporter, waitMs = 500, fetchFunction = fetch) {
async function getMarkdown(url, reporter, fetchWaitMs = 500, fetchFunction = fetch) {
try {
await delay(waitMs); // Wait 500ms to avoid rate limiting, not needed for live.
await delay(fetchWaitMs); // Wait 500ms to avoid rate limiting, not needed for live.
const response = await fetchFunction(url);

if (!response.ok) {
Expand Down Expand Up @@ -65,25 +65,28 @@ function getMdast(mdTxt, reporter) {
}

/**
* Load markdown from a file or URL.
* Load entry markdown from a file or URL.
*
* If a save directory is provided in the config and a file exists at that path,
* this function will return the contents of that file if it was modified
* within the cache time. Otherwise, it will fetch the markdown from the
* specified path or URL, save it to the save directory if one is provided, and
* return the fetched markdown.
*
* @param {string} entry - The path or URL to fetch the markdown from.
* @param {string} entry - The entry path of the document.
* @param {Object} config - The configuration options.
* @param {string} config.mdDir - The directory to save the fetched markdown to.
* @param {string} config.siteUrl - The base URL for relative markdown paths.
* @param {function} config.reporter - A logging function.
* @param {number} config.mdCacheMs - The cache time in milliseconds. If -1, the cache never expires.
* @param {Function} [fetchFunction=fetch] - The fetch function to use for fetching markdown.
* @returns {Promise<Object>} An object containing the markdown content, the markdown abstract syntax tree (mdast), the entry, the markdown path, and the markdown URL.
* @throws {Error} - If config is missing or entry is invalid.
*/
export async function loadDocument(entry, config, fetchFunction = fetch) {
if (!config) throw new Error('Missing config');
const { mdDir, siteUrl, reporter, waitMs, mdCacheMs = 0 } = config;
if (!entry || !entry.startsWith('/')) throw new Error(`Invalid path: ${entry}`);
const { mdDir, siteUrl, reporter, fetchWaitMs, mdCacheMs = 0 } = config;
const document = { entry, path: entryToPath(entry) };
document.url = new URL(document.path, siteUrl).href;
document.markdownFile = `${mdDir}${document.path}.md`;
Expand All @@ -100,7 +103,7 @@ export async function loadDocument(entry, config, fetchFunction = fetch) {
}

if (!document.markdown) {
document.markdown = await getMarkdown(`${document.url}.md`, reporter, waitMs, fetchFunction);
document.markdown = await getMarkdown(`${document.url}.md`, reporter, fetchWaitMs, fetchFunction);
reporter.log('load', 'success', 'Fetched markdown', `${document.url}.md`);

if (document.markdown && mdDir) {
Expand Down
12 changes: 12 additions & 0 deletions bulk-update/index.js
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
import { saveDocument } from './document-manager/document-manager.js';

Check warning on line 1 in bulk-update/index.js

View workflow job for this annotation

GitHub Actions / Running eslint 

[eslint] reported by reviewdog 🐶
File ignored because of a matching ignore pattern. Use "--no-ignore" to override.

Raw Output:
{"ruleId":null,"fatal":false,"severity":1,"message":"File ignored because of a matching ignore pattern. Use \"--no-ignore\" to override.","nodeType":null}
import ConsoleReporter from './reporter/console-reporter.js';
import ExcelReporter from './reporter/excel-reporter.js';
import BulkUpdate, { loadListData } from './bulk-update.js';

export {
BulkUpdate,
saveDocument,
ConsoleReporter,
ExcelReporter,
loadListData,
};
66 changes: 52 additions & 14 deletions bulk-update/reporter/excel-reporter.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,17 +3,42 @@ import * as fs from 'fs';
import BaseReporter from './reporter.js';

/**
* ExcelReporter class extending BaseReporter and logging to an XLSX file.
* ExcelReporter class extending BaseReporter and logging to an Excel report file.
*
* @extends BaseReporter
*/
class ExcelReporter extends BaseReporter {
constructor(filepath) {
/**
* Creates a new instance of the ExcelReporter class.
*
* @param {string} filepath - The file path where the Excel file will be saved.
* @param {boolean} [autoSave=true] - Excel file should be automatically saved when logging.
*/
constructor(filepath, autoSave = true) {
super();
this.filepath = filepath;
this.autoSave = autoSave;
this.workbook = xlsx.utils.book_new();
const totalsSheet = xlsx.utils.aoa_to_sheet([['Topic', 'Status', 'Count']]);
xlsx.utils.book_append_sheet(this.workbook, totalsSheet, 'Totals');
}

this.saveReport();
/**
* Get date string in the format of YYYY-MM-DD_HH-MM for file naming.
*
* @returns {string} - date string
*/
static getDateString(date = new Date()) {
return date.toLocaleString('en-US', {
hour12: false,
year: 'numeric',
month: '2-digit',
day: '2-digit',
hour: '2-digit',
minute: '2-digit',
})
.replace(/\/|,|:| /g, '-')
.replace('--', '_');
}

/**
Expand All @@ -25,16 +50,29 @@ class ExcelReporter extends BaseReporter {
* @param {...any} args - Additional arguments to be included in the log.
*/
log(topic, status, message, ...args) {
const header = ['Status', 'Message'];
const log = [status, message];
args.forEach((arg) => {
if (typeof arg === 'object' && !Array.isArray(arg)) {
Object.entries(arg).forEach(([key, value]) => {
header.push(key);
log.push(value);
});
} else if (Array.isArray(arg)) {
log.push(...arg);
} else {
log.push(arg);
}
});
super.log(topic, status, message, ...args);

const sheetName = topic || 'Uncategorized';
let sheet = this.workbook.Sheets[sheetName];
if (!sheet) {
sheet = xlsx.utils.aoa_to_sheet([['Status', 'Message']]);
sheet = xlsx.utils.aoa_to_sheet([header]);
xlsx.utils.book_append_sheet(this.workbook, sheet, sheetName);
}

const log = [status, message, ...args];
const range = xlsx.utils.decode_range(sheet['!ref']);
const newRow = range.e.r + 1;

Expand All @@ -45,7 +83,7 @@ class ExcelReporter extends BaseReporter {

sheet['!ref'] = xlsx.utils.encode_range({ s: range.s, e: { r: newRow, c: log.length - 1 } });

this.saveReport();
if (this.autoSave) this.saveReport();
}

/**
Expand All @@ -62,6 +100,7 @@ class ExcelReporter extends BaseReporter {
});
});
xlsx.utils.sheet_add_aoa(totalsSheet, data, { origin: 'A2' });
if (!this.filepath) return totals;
try {
this.saveReport();
console.log(`Report saved to ${this.filepath}`);
Expand All @@ -73,15 +112,14 @@ class ExcelReporter extends BaseReporter {
}

/**
* Saves the generated report to the specified filepath.
*/
* Saves the generated report to the specified filepath.
*/
saveReport() {
if (this.filepath) {
const directoryPath = this.filepath.split('/').slice(0, -1).join('/');
fs.mkdirSync(directoryPath, { recursive: true });
xlsx.set_fs(fs);
xlsx.writeFile(this.workbook, this.filepath);
}
if (!this.filepath) return;
const directoryPath = this.filepath.split('/').slice(0, -1).join('/');
fs.mkdirSync(directoryPath, { recursive: true });
xlsx.set_fs(fs);
xlsx.writeFile(this.workbook, this.filepath);
}
}

Expand Down
80 changes: 80 additions & 0 deletions faas-variations-report/query-indexes.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,80 @@
[
"https://main--bacom--adobecom.hlx.live/query-index.json",
"https://main--bacom--adobecom.hlx.live/ae_ar/query-index.json",
"https://main--bacom--adobecom.hlx.live/ae_en/query-index.json",
"https://main--bacom--adobecom.hlx.live/africa/query-index.json",
"https://main--bacom--adobecom.hlx.live/ar/query-index.json",
"https://main--bacom--adobecom.hlx.live/at/query-index.json",
"https://main--bacom--adobecom.hlx.live/au/query-index.json",
"https://main--bacom--adobecom.hlx.live/be_en/query-index.json",
"https://main--bacom--adobecom.hlx.live/be_fr/query-index.json",
"https://main--bacom--adobecom.hlx.live/be_nl/query-index.json",
"https://main--bacom--adobecom.hlx.live/bg/query-index.json",
"https://main--bacom--adobecom.hlx.live/br/query-index.json",
"https://main--bacom--adobecom.hlx.live/ca/query-index.json",
"https://main--bacom--adobecom.hlx.live/ca_fr/query-index.json",
"https://main--bacom--adobecom.hlx.live/ch_de/query-index.json",
"https://main--bacom--adobecom.hlx.live/ch_fr/query-index.json",
"https://main--bacom--adobecom.hlx.live/ch_it/query-index.json",
"https://main--bacom--adobecom.hlx.live/cl/query-index.json",
"https://main--bacom--adobecom.hlx.live/cn/query-index.json",
"https://main--bacom--adobecom.hlx.live/co/query-index.json",
"https://main--bacom--adobecom.hlx.live/cy_en/query-index.json",
"https://main--bacom--adobecom.hlx.live/cz/query-index.json",
"https://main--bacom--adobecom.hlx.live/de/query-index.json",
"https://main--bacom--adobecom.hlx.live/dk/query-index.json",
"https://main--bacom--adobecom.hlx.live/ee/query-index.json",
"https://main--bacom--adobecom.hlx.live/es/query-index.json",
"https://main--bacom--adobecom.hlx.live/fi/query-index.json",
"https://main--bacom--adobecom.hlx.live/fr/query-index.json",
"https://main--bacom--adobecom.hlx.live/gr_en/query-index.json",
"https://main--bacom--adobecom.hlx.live/hk_en/query-index.json",
"https://main--bacom--adobecom.hlx.live/hk_zh/query-index.json",
"https://main--bacom--adobecom.hlx.live/hu/query-index.json",
"https://main--bacom--adobecom.hlx.live/id_en/query-index.json",
"https://main--bacom--adobecom.hlx.live/id_id/query-index.json",
"https://main--bacom--adobecom.hlx.live/ie/query-index.json",
"https://main--bacom--adobecom.hlx.live/il_en/query-index.json",
"https://main--bacom--adobecom.hlx.live/il_he/query-index.json",
"https://main--bacom--adobecom.hlx.live/in/query-index.json",
"https://main--bacom--adobecom.hlx.live/in_hi/query-index.json",
"https://main--bacom--adobecom.hlx.live/it/query-index.json",
"https://main--bacom--adobecom.hlx.live/jp/query-index.json",
"https://main--bacom--adobecom.hlx.live/kr/query-index.json",
"https://main--bacom--adobecom.hlx.live/la/query-index.json",
"https://main--bacom--adobecom.hlx.live/lt/query-index.json",
"https://main--bacom--adobecom.hlx.live/lu_de/query-index.json",
"https://main--bacom--adobecom.hlx.live/lu_en/query-index.json",
"https://main--bacom--adobecom.hlx.live/lu_fr/query-index.json",
"https://main--bacom--adobecom.hlx.live/lv/query-index.json",
"https://main--bacom--adobecom.hlx.live/mena_ar/query-index.json",
"https://main--bacom--adobecom.hlx.live/mena_en/query-index.json",
"https://main--bacom--adobecom.hlx.live/mt/query-index.json",
"https://main--bacom--adobecom.hlx.live/mx/query-index.json",
"https://main--bacom--adobecom.hlx.live/my_en/query-index.json",
"https://main--bacom--adobecom.hlx.live/my_ms/query-index.json",
"https://main--bacom--adobecom.hlx.live/nl/query-index.json",
"https://main--bacom--adobecom.hlx.live/no/query-index.json",
"https://main--bacom--adobecom.hlx.live/nz/query-index.json",
"https://main--bacom--adobecom.hlx.live/pe/query-index.json",
"https://main--bacom--adobecom.hlx.live/ph_en/query-index.json",
"https://main--bacom--adobecom.hlx.live/ph_fil/query-index.json",
"https://main--bacom--adobecom.hlx.live/pl/query-index.json",
"https://main--bacom--adobecom.hlx.live/pt/query-index.json",
"https://main--bacom--adobecom.hlx.live/ro/query-index.json",
"https://main--bacom--adobecom.hlx.live/ru/query-index.json",
"https://main--bacom--adobecom.hlx.live/sa_ar/query-index.json",
"https://main--bacom--adobecom.hlx.live/sa_en/query-index.json",
"https://main--bacom--adobecom.hlx.live/se/query-index.json",
"https://main--bacom--adobecom.hlx.live/sg/query-index.json",
"https://main--bacom--adobecom.hlx.live/si/query-index.json",
"https://main--bacom--adobecom.hlx.live/sk/query-index.json",
"https://main--bacom--adobecom.hlx.live/th_en/query-index.json",
"https://main--bacom--adobecom.hlx.live/th_th/query-index.json",
"https://main--bacom--adobecom.hlx.live/tr/query-index.json",
"https://main--bacom--adobecom.hlx.live/tw/query-index.json",
"https://main--bacom--adobecom.hlx.live/ua/query-index.json",
"https://main--bacom--adobecom.hlx.live/uk/query-index.json",
"https://main--bacom--adobecom.hlx.live/vn_en/query-index.json",
"https://main--bacom--adobecom.hlx.live/vn_vi/query-index.json"
]
Loading

0 comments on commit c171b72

Please sign in to comment.