feat: rerendering of templates

packages/email/docsoc-mail-merge/src/cli.ts

@@ -11,8 +11,7 @@ import { TemplatePreviews } from "./engines/types";
 import { getFileNameSchemeInteractively } from "./interactivity/getFileNameSchemeInteractively";
 import getRunNameInteractively from "./interactivity/getRunNameInteractively";
 import mapCSVFieldsInteractive from "./interactivity/mapCSVFieldsInteractive";
-import { getRecordPreviewPrefixForIndividual, getRecordPreviewPrefixForMetadata, writeMetadata } from "./sideCarData";
-import { SidecardData } from "./sideCarData/types";
+import { getRecordPreviewPrefixForIndividual, writeMetadata } from "./previews/sidecarData";
 import { stopIfCriticalFsError } from "./util/files";
 import createLogger from "./util/logger";
 import { CliOptions, CSVRecord } from "./util/types";
@@ -83,14 +82,15 @@ async function main(opts: CliOptions) {
 
     // 8: Render intermediate results
     logger.info("Rendering template previews/intermediates...");
+    // NOTE: CSVRecord here is the record with the CSV headers mapped to the template fields, rather than with the raw template fields
     const previews: [TemplatePreviews, CSVRecord][] = await Promise.all(
         records.map(async (csvRecord) => {
             const preparedRecord = Object.fromEntries(
                 Object.entries(csvRecord).map(([key, value]) => {
                     return [fieldsMapCSVtoTemplate.get(key) ?? key, value];
                 }),
             );
-            return [await engine.renderPreview(preparedRecord), csvRecord];
+            return [await engine.renderPreview(preparedRecord), preparedRecord];
         }),
     );
 
@@ -106,12 +106,7 @@ async function main(opts: CliOptions) {
             const operations = previews.map(async (preview) => {
                 const fileName = getRecordPreviewPrefixForIndividual(record, fileNamer, opts.templateEngine, preview);
                 logger.debug(`Writing ${fileName}__${opts.templateEngine}__${preview.name}`);
-                await stopIfCriticalFsError(
-                    fs.writeFile(
-                        join(previewsRoot, `${fileName}__${opts.templateEngine}__${preview.name}`),
-                        preview.content,
-                    ),
-                );
+                await stopIfCriticalFsError(fs.writeFile(join(previewsRoot, fileName), preview.content));
             });
 
             // Add metadata write operation

packages/email/docsoc-mail-merge/src/engines/nunjucks-md/index.ts

@@ -3,10 +3,10 @@ import nunjucks from "nunjucks";
 
 import { renderMarkdownToHtml } from "../../markdown/toHtml";
 import { CSVRecord } from "../../util/types";
-import { TemplateEngineOptions } from "../types";
+import { TemplateEngineOptions, TemplatePreviews } from "../types";
 import { TemplateEngine } from "../types";
 import getTemplateFields from "./getFields";
-import { assertIsNunjucksTemplateOptions, NunjucksMarkdownTemplateOptions } from "./types";
+import { assertIsNunjucksTemplateOptions, NunjucksMarkdownTemplateOptions, NunjucksSidecarMetadata } from "./types";
 
 /**
  * A Nunjucks Markdown template engine
@@ -35,19 +35,31 @@ export default class NunjucksMarkdownEngine extends TemplateEngine {
         this.templateOptions = templateOptions;
     }
 
-    async loadTemplate() {
+    private async renderMarkdownToHtmlInsideWrapper(markdown: string) {
+        // Render the MD to HTML
+        const htmlWrapper = await fs.readFile(this.templateOptions.rootHtmlTemplate, "utf-8");
+        const htmlWrapperCompiled = nunjucks.compile(htmlWrapper, nunjucks.configure({ autoescape: false }));
+        const html = renderMarkdownToHtml(markdown);
+
+        // Wrap the rendered markdown html in the wrapper
+        const wrapped = htmlWrapperCompiled.render({ content: html });
+
+        return wrapped;
+    }
+
+    override async loadTemplate() {
         this.loadedTemplate = await fs.readFile(this.templateOptions.templatePath, "utf-8");
     }
 
-    extractFields() {
+    override extractFields() {
         if (!this.loadedTemplate) {
             throw new Error("Template not loaded");
         }
 
         return getTemplateFields(this.loadedTemplate);
     }
 
-    async renderPreview(record: CSVRecord) {
+    override async renderPreview(record: CSVRecord) {
         if (!this.loadedTemplate) {
             throw new Error("Template not loaded");
         }
@@ -58,18 +70,13 @@ export default class NunjucksMarkdownEngine extends TemplateEngine {
                 throwOnUndefined: true,
             }),
         );
-        const htmlWrapper = await fs.readFile(this.templateOptions.rootHtmlTemplate, "utf-8");
-        const htmlWrapperCompiled = nunjucks.compile(htmlWrapper, nunjucks.configure({ autoescape: false }));
 
         // Render the Markdown template with the record, so that we have something to preview
         const expanded = templateCompiled.render({
             name: record["name"],
         });
         // Render the MD to HTML
-        const html = renderMarkdownToHtml(expanded);
-
-        // Wrap the rendered markdown html in the wrapper
-        const wrapped = htmlWrapperCompiled.render({ content: html });
+        const wrappedHtml = await this.renderMarkdownToHtmlInsideWrapper(expanded);
 
         // Return both
         return [
@@ -82,11 +89,44 @@ export default class NunjucksMarkdownEngine extends TemplateEngine {
             },
             {
                 name: "Preview-HTML.html",
-                content: wrapped, // this is what will be sent - do not edit it, re-rendering will overwrite it
+                content: wrappedHtml, // this is what will be sent - do not edit it, re-rendering will overwrite it
                 metadata: {
                     type: "html",
                 },
             },
         ];
     }
+
+    /**
+     * The rerenderer is simple - we just re-render the HTML preview!
+     */
+    override async rerenderPreviews(loadedPreviews: TemplatePreviews): Promise<TemplatePreviews> {
+        const markdownPreview = loadedPreviews.find(
+            (preview) => (preview.metadata as NunjucksSidecarMetadata).type === "markdown",
+        );
+        if (!markdownPreview) {
+            throw new Error("No markdown preview found in sidecar data");
+        }
+        const htmlPreview = loadedPreviews.find(
+            (preview) => (preview.metadata as NunjucksSidecarMetadata).type === "html",
+        );
+        if (!htmlPreview) {
+            throw new Error("No HTML preview found in sidecar data");
+        }
+
+        if (!this.loadedTemplate) {
+            throw new Error("Template not loaded");
+        }
+
+        // Re-render the markdown preview
+        const html = await this.renderMarkdownToHtmlInsideWrapper(markdownPreview.content);
+
+        return [
+            markdownPreview,
+            {
+                ...htmlPreview,
+                content: html,
+            },
+        ];
+    }
 }

packages/email/docsoc-mail-merge/src/engines/nunjucks-md/types.ts

@@ -11,6 +11,11 @@ export interface NunjucksMarkdownTemplateOptions {
     [key: string]: string;
 }
 
+export interface NunjucksSidecarMetadata {
+    type: "markdown" | "html";
+    [key: string]: unknown;
+}
+
 /**
  * Asserts that the given options are valid Nunjucks template options & throws an error if they are not
  */

packages/email/docsoc-mail-merge/src/engines/types.ts

@@ -54,6 +54,23 @@ export abstract class TemplateEngine {
      * @returns A promise that resolves to an array of {@link TemplatePreviews} objects - check the type for more information
      */
     abstract renderPreview(record: CSVRecord): Promise<TemplatePreviews>;
+
+    /**
+     * Given previews generated by {@link TemplateEngine.renderPreview} or this method, re-render a preview.
+     *
+     * The idea of this method is that you would generate several previews for a record,
+     * and then allow one or multiple of them to be edited.
+     *
+     * The user then triggers a rerender, and you should update the other previews accordingly.
+     *
+     * Example: One preview is a markdown preview, another is a HTML preview.
+     * The user edits the markdown preview and you then re-render the HTML preview.
+     *
+     * **IMPORTANT NOTE**: You _must_ rerender _all_ previews, even if only one was edited. This is a replace-all operations, not a patch.
+     * @param loadedPreview
+     * @param associatedRecord
+     */
+    abstract rerenderPreviews(loadedPreviews: TemplatePreviews, associatedRecord: CSVRecord): Promise<TemplatePreviews>;
 }
 
 /**

packages/email/docsoc-mail-merge/src/interactivity/mapCSVFieldsInteractive.ts

@@ -12,7 +12,13 @@ import createLogger from "../util/logger";
 const logger = createLogger("docsoc.util.mapInteractive");
 
 /**
- * Interactive mapping of CSV fields to template fields
+ * Interactive mapping of CSV fields to template fields.
+ *
+ * Prints a prompt for each CSV header, asking the user to select the corresponding template field.
+ * Finishes with a warning if not all template fields were mapped.
+ *
+ * NOTE: It is recommend to use this function in a CLI environment, as it uses inquirer for interactive prompts.
+ * NOTE: Provide this function with any special fields you need e.g. email, name, etc.
  * @param templateFields Set of fields the template wants, extracted using {@link TemplateEngine.extractFields}
  * @param csvHeaders Headers from the CSV
  * @returns Map of csv headers to template fields
@@ -35,6 +41,8 @@ const mapCSVFieldsInteractive = async (
             name: header,
             message: `Map CSV field \`${header}\` to template field:`,
             choices: Array.from(templateFields),
+            // Set default to index in templateField that matches the csvHeader
+            default: Array.from(templateFields).indexOf(header) > 0 ? header : undefined,
         })),
     );
     for (const [csvHeader, templateFieldChosen] of Object.entries(await prompter)) {

packages/email/docsoc-mail-merge/src/sideCarData/index.ts → packages/email/docsoc-mail-merge/src/previews/sidecarData.ts

@@ -12,9 +12,10 @@ import { TemplatePreview, TemplatePreviews } from "../engines/types";
 import { stopIfCriticalFsError } from "../util/files";
 import createLogger from "../util/logger";
 import { CliOptions, CSVRecord } from "../util/types";
-import { SidecardData } from "./types";
+import { SidecarData } from "./types";
 
 const PARTS_SEPARATOR = "__";
+const METADATA_FILE_SUFFIX = "-metadata.json";
 const logger = createLogger("docsoc.sidecar");
 
 /**
@@ -54,11 +55,11 @@ export const getRecordPreviewPrefixForIndividual = (
  * // => "file_1-metadata.json"
  */
 export const getRecordPreviewPrefixForMetadata = (record: CSVRecord, fileNamer: (record: CSVRecord) => string) =>
-    `${getRecordPreviewPrefix(record, fileNamer)}-metadata.json`;
+    `${getRecordPreviewPrefix(record, fileNamer)}${METADATA_FILE_SUFFIX}`;
 
 /**
  * Write the metadata for a record & its associated previews to a JSON file.
- * @param record The record to write metadata for
+ * @param record The record to write metadata for, mapped to the fields required by the template
  * @param templateEngine The engine used to render the previews
  * @param templateOptions The options given to the engine
  * @param previews The previews rendered for the record
@@ -74,7 +75,8 @@ export async function writeMetadata(
     fileNamer: (record: CSVRecord) => string,
     previewsRoot: string,
 ): Promise<void> {
-    const sidecar: SidecardData = {
+    const sidecar: SidecarData = {
+        name: fileNamer(record),
         record: record,
         engine: templateEngine,
         engineOptions: templateOptions,
@@ -89,6 +91,32 @@ export async function writeMetadata(
 
     const metadataFile = getRecordPreviewPrefixForMetadata(record, fileNamer);
     logger.debug(`Writing metadata for ${fileNamer(record)} to ${metadataFile}`);
-    await stopIfCriticalFsError(fs.writeFile(join(previewsRoot, metadataFile), JSON.stringify(sidecar, null, 4)));
+    await writeSidecarFile(previewsRoot, metadataFile, sidecar);
     return Promise.resolve();
 }
+
+/**
+ * Write the sidecar metadata file for a record
+ */
+export async function writeSidecarFile(previewsRoot: string, metadataFile: string, sidecar: SidecarData) {
+    await stopIfCriticalFsError(fs.writeFile(join(previewsRoot, metadataFile), JSON.stringify(sidecar, null, 4)));
+}
+
+/**
+ * Load sidecar metadata files from a directory
+ * @param previewsRoot The root directory to load sidecar metadata from
+ * @returns An async iterator of sidecar metadata objects
+ */
+export async function* loadSidecars(
+    previewsRoot: string,
+): AsyncIterableIterator<SidecarData & { $originalfilename: string }> {
+    logger.info(`Loading sidecar metadata from ${previewsRoot}`);
+    const files = await fs.readdir(previewsRoot);
+    const metadataFiles = files.filter((file) => file.endsWith(METADATA_FILE_SUFFIX));
+
+    for (const metadataFile of metadataFiles) {
+        logger.debug(`Loading metadata from ${metadataFile}`);
+        const metadata = await fs.readFile(join(previewsRoot, metadataFile), "utf-8");
+        yield { ...(JSON.parse(metadata) as SidecarData), $originalfilename: metadataFile };
+    }
+}

packages/email/docsoc-mail-merge/src/sideCarData/types.ts → packages/email/docsoc-mail-merge/src/previews/types.ts

@@ -5,7 +5,9 @@ import { CSVRecord } from "../util/types";
 /**
  * Outputted to JSON files next to rendered template previews, containing metadata about the preview.
  */
-export interface SidecardData {
+export interface SidecarData {
+    /** Name of the template rendered (used for logging) */
+    name: string;
     /** Record associated with the template rendered  */
     record: CSVRecord;
     /** Engine used */

packages/email/docsoc-mail-merge/src/rerender.ts

@@ -0,0 +1,73 @@
+import fs from "fs/promises";
+import { join } from "path";
+
+import packageJSON from "../package.json";
+import { ENGINES_MAP } from "./engines";
+import { TemplatePreviews } from "./engines/types";
+import { loadSidecars, writeSidecarFile } from "./previews/sidecarData";
+import { stopIfCriticalFsError } from "./util/files";
+import createLogger from "./util/logger";
+
+const logger = createLogger("docsoc");
+
+async function main(directory: string) {
+    logger.info("DoCSoc Mail Merge - rerender");
+    logger.info(`v${packageJSON.version}`);
+
+    logger.info(`Rerendering previews at ${directory}...`);
+
+    // 1: Load all sidecars
+    const sidecars = loadSidecars(directory);
+    // 2: Map sidecar files & rerender
+    for await (const sidecar of sidecars) {
+        const { name, engine: engineName, engineOptions, files, record } = sidecar;
+
+        const EngineClass = ENGINES_MAP[engineName as keyof typeof ENGINES_MAP];
+        if (!EngineClass) {
+            logger.error(`Invalid template engine: ${engineName}`);
+            logger.warn(`Skipping record ${name} as the engine is invalid!`);
+            continue;
+        }
+
+        // Load in the engine
+        const engine = new EngineClass(engineOptions);
+        logger.debug(`Loading engine ${engineName} for ${name}...`);
+        await engine.loadTemplate();
+
+        logger.debug("Remapping sidecar files metadata back to TemplatePreviews...");
+        logger.debug(JSON.stringify(files));
+        const previews: TemplatePreviews = await Promise.all(
+            files.map(async (file) => ({
+                ...file.engineData,
+                content: await stopIfCriticalFsError(fs.readFile(join(directory, file.filename), "utf-8")),
+            })),
+        );
+
+        // Rerender previews
+        logger.info(`Rerendering ${name} using engine ${engineName}...`);
+        const renderedPreviews = await engine.rerenderPreviews(previews, record);
+
+        logger.info(`Writing rerendered previews for ${name}...`);
+        await Promise.all(
+            renderedPreviews.map(async (preview, idx) => {
+                const file = files[idx];
+                logger.debug(`Writing rerendered preview ${file.filename}...`);
+                await stopIfCriticalFsError(fs.writeFile(join(directory, file.filename), preview.content));
+                logger.debug("Overwriting sidecar metadata with new metadata...");
+                sidecar.files[idx].engineData = { ...preview, content: undefined };
+            }),
+        );
+
+        logger.info(`Updating sidecar metadata for ${name} at ${sidecar.$originalfilename}...`);
+        await writeSidecarFile(directory, sidecar.$originalfilename, sidecar);
+
+        logger.info(`Finished rerendering ${name}`);
+    }
+}
+
+if (process.argv.length < 3) {
+    logger.error("Please provide the directory to rerender previews in");
+    process.exit(1);
+}
+
+main(process.argv[2]);

packages/email/docsoc-mail-merge/src/util/files.ts

@@ -17,7 +17,7 @@ export async function stopIfCriticalFsError<T>(promise: Promise<T>) {
     } catch (e) {
         if (e instanceof Error && isErrnoException(e)) {
             if (e?.code === "ENOENT") {
-                logger.error("File not found: ", e.message);
+                logger.error("File not found: ", e);
             } else if (e?.code === "EACCES") {
                 logger.error("Permission denied: ", e.message);
             } else {
@@ -28,4 +28,4 @@ export async function stopIfCriticalFsError<T>(promise: Promise<T>) {
         }
         process.exit(1);
     }
-}
+}