Set up custom module for metapages docusaurus

.github/workflows/release.yml

@@ -1,7 +1,7 @@
 name: Release
 on:
   push:
-    branches: [main, release]
+    branches: [main, release, metapage-custom-plugins]
 jobs:
   release:
     name: Release

custom-plugins/README.md

@@ -0,0 +1,45 @@
+# Custom Plugins Directory
+
+This directory contains custom plugins for docu-notion. These plugins will be loaded in addition to the default plugins.
+
+## Plugin Structure
+
+Each plugin should be a TypeScript file that exports a plugin object conforming to the `IPlugin` interface. Here's an example:
+
+```typescript
+import { IPlugin } from '../src/plugins/pluginTypes';
+
+const myPlugin: IPlugin = {
+  name: 'My Custom Plugin',
+  // Add your plugin functionality here
+  regexMarkdownModifications: [
+    {
+      regex: /pattern/g,
+      replacementPattern: 'replacement',
+      includeCodeBlocks: false,
+    },
+  ],
+};
+
+export default myPlugin;
+```
+
+## Available Plugin Features
+
+You can implement any of the following features in your plugin:
+
+1. `notionBlockModifications`: Modify Notion blocks before they are converted to markdown
+2. `notionToMarkdownTransforms`: Override default Notion-to-markdown conversions
+3. `linkModifier`: Modify links after they are converted to markdown
+4. `regexMarkdownModifications`: Perform regex replacements on the markdown output
+5. `init`: Perform async initialization when the plugin is loaded
+
+See the `IPlugin` interface in `src/plugins/pluginTypes.ts` for more details.
+
+## Loading Custom Plugins
+
+Custom plugins in this directory will be automatically loaded when running docu-notion. You can also specify additional custom plugin directories in your configuration.
+
+## Example
+
+See `example.ts` in this directory for a simple example plugin. 

custom-plugins/correctNotionUrlsInMermaid.ts

@@ -0,0 +1,137 @@
+import { CodeBlockObjectResponse } from "@notionhq/client/build/src/api-endpoints";
+import {
+  IDocuNotionContext,
+  IPlugin,
+  Log,
+  NotionBlock,
+} from "../src/plugins/pluginTypes";
+
+// Helper function to join URL paths properly
+function joinPaths(...parts: string[]): string {
+  return parts
+    .map(part => part.replace(/^\/+|\/+$/g, "")) // Remove leading/trailing slashes
+    .filter(part => part.length > 0) // Remove empty parts
+    .join("/"); // Join with a single slash
+}
+
+// The mermaid interactive click syntax:
+// https://mermaid.js.org/syntax/flowchart.html#interaction
+// NB this processing is just for internal link navigation
+export function correctNotionUrlsInMermaid(args?: {
+  slugPrefix?: string;
+}): IPlugin {
+  const { slugPrefix: slugPrefixFromArgs } = args || {};
+  return {
+    name: "correctNotionUrlsInMermaid",
+
+    notionToMarkdownTransforms: [
+      {
+        type: "code",
+        getStringFromBlock: (
+          context: IDocuNotionContext,
+          block: NotionBlock
+        ) => {
+          const slugPrefix =
+            slugPrefixFromArgs !== undefined
+              ? slugPrefixFromArgs
+              : context.options.markdownOutputPath.split("/").pop();
+          const codeBlock = block as CodeBlockObjectResponse;
+          let text: string = codeBlock.code.rich_text[0].plain_text;
+          let language: string = codeBlock.code.language;
+
+          if (language === "plain text") {
+            language = "text";
+          }
+
+          if (language === "mermaid") {
+            text = transformMermaidLinks(text, url =>
+              convertHref({ url, context, slugPrefix })
+            );
+          }
+
+          // HACK JUST FOR ME
+          // HACK use notion code type "coffeescript" to render jsx live
+          if (language === "coffeescript") {
+            language = "jsx live";
+          }
+
+          return `\`\`\`${language}\n${text}\n\`\`\``;
+        },
+      },
+    ],
+  };
+}
+
+const convertHref = (args: {
+  url: string;
+  context: IDocuNotionContext;
+  slugPrefix?: string;
+}) => {
+  const { url, context, slugPrefix } = args;
+
+  // Do not convert non-notion links
+  if (!url.startsWith("https://www.notion.so/")) {
+    return url;
+  }
+  const notionId = new URL(url).pathname.split("-").pop() || "";
+
+  const page = context.pages.find(p => {
+    return p.matchesLinkId(notionId);
+  });
+  if (page) {
+    let convertedLink = context.layoutStrategy.getLinkPathForPage(page);
+    // console.log("convertedLink", convertedLink);
+
+    if (slugPrefix) {
+      // Ensure the path starts with a slash if needed
+      convertedLink = "/" + joinPaths(slugPrefix, convertedLink);
+      // Remove duplicate leading slash if slugPrefix already had one
+      convertedLink = convertedLink.replace(/^\/+/, "/");
+    }
+
+    Log.verbose(`Converting Link ${url} --> ${convertedLink}`);
+    return convertedLink;
+  }
+
+  // About this situation. See https://github.com/sillsdev/docu-notion/issues/9
+  Log.warning(
+    `Could not find the target of this link. Note that links to outline sections are not supported. ${url}. https://github.com/sillsdev/docu-notion/issues/9`
+  );
+
+  return url;
+};
+
+const transformMermaidLinks = (
+  pageMarkdown: string,
+  convertHref: (url: string) => string
+) => {
+  // The mermaid interactive click syntax:
+  // https://mermaid.js.org/syntax/flowchart.html#interaction
+  // NB this processing is just for internal link navigation
+  const linkRegExp =
+    /\s*click\s+([A-za-z][A-za-z0-9_-]*)\s+"?(https:\/\/www\.notion\.so\/\S*)"/g;
+  let output = pageMarkdown;
+  let match: RegExpExecArray | null;
+
+  // The key to understanding this while is that linkRegExp actually has state, and
+  // it gives you a new one each time. https://stackoverflow.com/a/1520853/723299
+
+  while ((match = linkRegExp.exec(pageMarkdown)) !== null) {
+    const originalLink = match[0];
+
+    const hrefFromNotion = match[2];
+    const hrefForDocusaurus = convertHref(hrefFromNotion);
+
+    if (hrefForDocusaurus) {
+      output = output.replace(
+        match[0],
+        `\n  click ${match[1]} "${hrefForDocusaurus}"`
+      );
+      Log.verbose(`transformed link: ${originalLink}-->${hrefForDocusaurus}`);
+    } else {
+      Log.verbose(`Maybe problem with link ${JSON.stringify(match)}`);
+    }
+  }
+
+  return output;
+};

custom-plugins/embed/index.ts

@@ -0,0 +1,96 @@
+import { getHashParamValueJsonFromUrl } from "@metapages/hash-query";
+import { MetapageDefinitionV3 } from "@metapages/metapage";
+import { IDocuNotionContext, IPlugin } from "../../src/plugins/pluginTypes";
+
+import fetch from "node-fetch";
+
+export const embedToIframe: IPlugin = {
+  name: "embed-to-iframe",
+
+  regexMarkdownModifications: [
+    // replace embeds with iframes
+    {
+      regex: /\[embed\]\((\S+)\)/,
+      getReplacement: async (
+        context: IDocuNotionContext,
+        match: RegExpExecArray
+      ): Promise<string> => {
+        const urlString = match[1];
+        if (!urlString) {
+          return match[0];
+        }
+
+        const url = new URL(urlString);
+
+        let possibleMetapageDefinition: MetapageDefinitionV3 | undefined =
+          undefined;
+        let iframeHeight = 300;
+        if (url.hostname.includes("metapage")) {
+          // We can get more info
+          possibleMetapageDefinition = getHashParamValueJsonFromUrl(
+            urlString,
+            "definition"
+          );
+
+          if (!possibleMetapageDefinition) {
+            // We can get more info
+            const urlBLobToFetchJson = new URL(urlString);
+            if (!urlBLobToFetchJson.pathname.endsWith("metapage.json")) {
+              urlBLobToFetchJson.pathname =
+                urlBLobToFetchJson.pathname +
+                (urlBLobToFetchJson.pathname.endsWith("/") ? "" : "/") +
+                "metapage.json";
+            }
+            try {
+              const r = await fetch(urlBLobToFetchJson.href);
+              possibleMetapageDefinition =
+                (await r.json()) as MetapageDefinitionV3;
+            } catch (err) {
+              //ignored
+            }
+          }
+          if (possibleMetapageDefinition) {
+            // We can get better layout info
+            const gridlayout =
+              possibleMetapageDefinition?.meta?.layouts?.["react-grid-layout"];
+            if (gridlayout) {
+              const rowHeight: number = gridlayout?.props?.rowHeight || 100;
+              const margin: number = gridlayout?.props?.margin?.[1] || 20;
+              const maxHeightBlocks = gridlayout?.layout.reduce(
+                (acc: number, cur: { h: number; y: number }) => {
+                  return Math.max(acc, cur.h + cur.y);
+                },
+                1
+              );
+              const headerHeight = 40; // Copied from metapage.io code, how to get this in here?
+              const height =
+                maxHeightBlocks * rowHeight +
+                (margin * maxHeightBlocks - 1) +
+                headerHeight;
+              iframeHeight = height;
+            }
+          }
+          // fudge
+          iframeHeight += 70; // for the header plus padding, currently not part of the compute
+        }
+
+        // Check for redirects
+        try {
+          const maybeRedirectResponse = await fetch(urlString, {
+            redirect: "manual",
+          });
+          if (maybeRedirectResponse.status === 302) {
+            const location = maybeRedirectResponse.headers.get("location");
+            if (location) {
+              url.href = location;
+            }
+          }
+        } catch (err) {
+          // ignored
+        }
+
+        return `\n<iframe scrolling="yes" allow="fullscreen *; camera *; speaker *;" style={{width:"100%",height:"${iframeHeight}px",overflow:"hidden"}} src="${url.href}"></iframe>\n`;
+      },
+    },
+  ],
+};

custom-plugins/example.ts

@@ -0,0 +1,14 @@
+import { IPlugin } from '../src/plugins/pluginTypes';
+
+const examplePlugin: IPlugin = {
+  name: 'Example Custom Plugin',
+  regexMarkdownModifications: [
+    {
+      regex: /\[\[(.*?)\]\]/g,
+      replacementPattern: '[[$1]]',
+      includeCodeBlocks: false,
+    },
+  ],
+};
+
+export default examplePlugin; 

custom-plugins/index.ts

@@ -0,0 +1,4 @@
+export * from "./correctNotionUrlsInMermaid";
+export * from "./modifiedInternalLinks";
+export * from "./notionColumnsUpgraded";
+export * from "./embed";

custom-plugins/modifiedInternalLinks.ts

@@ -0,0 +1,24 @@
+// current docu-notion does not prefix the slug with
+// e.g. /docs which breaks links
+import { IDocuNotionContext, IPlugin } from "../src/plugins/pluginTypes";
+
+export const modifiedStandardInternalLinkConversion: IPlugin = {
+  name: "modified standard internal link conversion",
+  regexMarkdownModifications: [
+    {
+      regex: /\[([^\]]+)?\]\((?!mailto:)(\/?[^),^\/]+)\)/,
+      getReplacement: async (
+        context: IDocuNotionContext,
+        match: RegExpExecArray
+      ): Promise<string> => {
+        const slugPrefix = context.options.markdownOutputPath.split("/").pop();
+        const label = match[1];
+        let url = match[2];
+        if (!url.startsWith(`/${slugPrefix}`)) {
+          url = `/${slugPrefix}${url}`;
+        }
+        return `[${label}](${url})`;
+      },
+    },
+  ],
+};