Support for multiple API docs repos (#590)

Misc changes to support analyzing multiple repos/versions when generating API docs.

Author: aomarks

.dockerignore

@@ -3,5 +3,5 @@ Dockerfile
 **/node_modules
 npm-debug.log
 **/tsconfig.tsbuildinfo
-/packages/lit-dev-api/lit/
+/packages/lit-dev-api/api-data/*/repo/
 /packages/lit-dev-tests/

.gitignore

@@ -11,7 +11,7 @@ packages/lit-dev-content/site/fonts
 packages/lit-dev-content/temp
 packages/lit-dev-content/samples/js
 
-packages/lit-dev-api/lit/
+packages/lit-dev-api/api-data/*/repo/
 
 # We only want to keep '-linux' screenshots which are tested by Github Actions.
 packages/lit-dev-tests/src/playwright/**/*.png

README.md

@@ -79,13 +79,9 @@ code packages/lit-dev-api/lit/
 
 The `lit` directory is a regular cloned git repo, so you can make changes
 directly here, and push PRs from it as normal. It's configured to track the
-`main` branch, but is pinned to a particular commit via the `lit.sha` file. To
-update the current commit, run:
-
-```sh
-cd packages/lit-dev-tools
-npm run monorepo:update
-```
+`main` branch, but is pinned to a particular commit. To update the current
+commit, update the `sha` field in
+[`packages/lit-dev-tools-cjs/src/api-docs/configs/lit-2.ts`](https://github.com/lit/lit.dev/blob/main/packages/lit-dev-tools-cjs/src/api-docs/configs/lit-2.ts).
 
 ### Serve production mode
 

packages/lit-dev-api/api-data/.gitkeep

@@ -6,14 +6,8 @@
   "author": "Google LLC",
   "license": "BSD-3-Clause",
   "scripts": {
-    "build": "(ls lit 2>/dev/null || npm run monorepo:init) && npm run build:generate",
-    "build:generate": "node ../lit-dev-tools-cjs/lib/generate-api-docs-data.js",
-    "build:watch": "chokidar 'lit/packages/*/src/**/*.ts' '../lit-dev-tools-cjs/lib/generate-api-docs-data.js' -c 'npm run monorepo:build && npm run build:generate' --initial",
-    "monorepo:init": "npm run monorepo:clone && npm run monorepo:install && npm run monorepo:build",
-    "monorepo:clone": "git clone https://github.com/lit/lit --branch main lit && cd lit && git checkout $(cat ../lit.sha)",
-    "monorepo:update": "cd lit && git checkout main && git pull && git rev-parse HEAD > ../lit.sha && cd .. && npm run monorepo:install && npm run monorepo:build",
-    "monorepo:install": "cd lit && npm ci && npm run bootstrap",
-    "monorepo:build": "cd lit && npx lerna run build:ts --scope lit --include-dependencies"
+    "build": "node ../lit-dev-tools-cjs/lib/api-docs/generate.js",
+    "build:watch": "chokidar 'lit/packages/*/src/**/*.ts' '../lit-dev-tools-cjs/lib/api-docs/**/*.js' -c 'npm run build' --initial"
   },
   "dependencies": {
     "chokidar-cli": "^2.1.0",

packages/lit-dev-api/api-data/pages.json packages/lit-dev-api/api-data/lit-2/pages.json

@@ -69,7 +69,7 @@ module.exports = function (eleventyConfig) {
   }
   eleventyConfig.addPassthroughCopy('api/**/*');
 
-  eleventyConfig.addWatchTarget('../lit-dev-api/api-data');
+  eleventyConfig.addWatchTarget('../lit-dev-api/api-data/*/*.json');
 
   // Placeholder shortcode for TODOs
   // Formatting is intentional: outdenting the HTML causes the
@@ -246,7 +246,7 @@ ${content}
 
   // Don't use require() because of Node caching in watch mode.
   const apiSymbolMap = JSON.parse(
-    fsSync.readFileSync('../lit-dev-api/api-data/symbols.json', 'utf8')
+    fsSync.readFileSync('../lit-dev-api/api-data/lit-2/symbols.json', 'utf8')
   );
 
   /**

packages/lit-dev-api/api-data/symbols.json packages/lit-dev-api/api-data/lit-2/symbols.json

@@ -18,6 +18,7 @@ module.exports = async () =>
         '..',
         'lit-dev-api',
         'api-data',
+        'lit-2',
         'pages.json'
       ),
       'utf8'

packages/lit-dev-api/lit.sha

@@ -9,11 +9,11 @@ import * as pathlib from 'path';
 import type {ApiDocsConfig} from '../types.js';
 
 const root = pathlib.resolve(__dirname, '..', '..', '..', '..', '..');
-const workDir = pathlib.join(root, 'packages', 'lit-dev-api');
-const gitDir = pathlib.join(workDir, 'lit');
+const dataDir = pathlib.join(root, 'packages', 'lit-dev-api', 'api-data');
+const workDir = pathlib.join(dataDir, 'lit-2');
+const gitDir = pathlib.join(workDir, 'repo');
 const litDir = pathlib.join(gitDir, 'packages', 'lit');
 const srcDir = pathlib.join(litDir, 'src');
-const outDir = pathlib.join(workDir, 'api-data');
 
 /**
  * lit.dev API docs configuration for Lit 2.x
@@ -23,10 +23,25 @@ export const lit2Config: ApiDocsConfig = {
   commit: 'f8ee010bc515e4bb319e98408d38ef3d971cc08b',
   gitDir,
   tsConfigPath: pathlib.join(litDir, 'tsconfig.json'),
-  pagesOutPath: pathlib.resolve(outDir, 'pages.json'),
-  symbolsOutPath: pathlib.resolve(outDir, 'symbols.json'),
+  pagesOutPath: pathlib.resolve(workDir, 'pages.json'),
+  symbolsOutPath: pathlib.resolve(workDir, 'symbols.json'),
   typedocRoot: pathlib.join(root, 'packages'),
 
+  extraSetupCommands: [
+    {cmd: 'npm', args: ['run', 'bootstrap']},
+    {
+      cmd: 'npx',
+      args: [
+        'lerna',
+        'run',
+        'build:ts',
+        '--scope',
+        'lit',
+        '--include-dependencies',
+      ],
+    },
+  ],
+
   entrypointModules: [
     pathlib.join(srcDir, 'async-directive.ts'),
     pathlib.join(srcDir, 'decorators.ts'),

packages/lit-dev-api/package.json

@@ -0,0 +1,136 @@
+/**
+ * @license
+ * Copyright 2021 Google LLC
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
+
+import * as typedoc from 'typedoc';
+import * as fs from 'fs/promises';
+import * as pathlib from 'path';
+import {execFile} from 'child_process';
+import {promisify} from 'util';
+import {ApiDocsTransformer} from './transformer.js';
+import {lit2Config} from './configs/lit-2.js';
+
+import type {ApiDocsConfig} from './types.js';
+
+const execFileAsync = promisify(execFile);
+
+const configs = [lit2Config];
+
+/**
+ * Check whether the given file path exists.
+ */
+const fileExists = async (path: string): Promise<boolean> => {
+  try {
+    await fs.stat(path);
+    return true;
+  } catch (err) {
+    if ((err as {code?: string}).code !== 'ENOENT') {
+      throw err;
+    }
+  }
+  return false;
+};
+
+/**
+ * Return the SHA of the given commit reference.
+ */
+const shaOfReference = async (
+  gitDir: string,
+  reference: string
+): Promise<string> => {
+  const {stdout} = await execFileAsync(
+    'git',
+    ['show', '-s', '--format=%H', reference],
+    {
+      cwd: gitDir,
+    }
+  );
+  return stdout.trim();
+};
+
+/**
+ * Clone the given Git repo URL at the given commit into the given directory. If
+ * the directory already exists, do nothing.
+ */
+const cloneIfNeeded = async (repo: string, commit: string, dir: string) => {
+  if (await fileExists(dir)) {
+    console.log(`${dir} already exists, skipping git clone`);
+    const expectedSha = await shaOfReference(dir, commit);
+    const actualSha = await shaOfReference(dir, 'HEAD');
+    if (actualSha !== expectedSha) {
+      throw new Error(
+        `Git repo ${dir} is at commit ${actualSha}, but is configured for ${commit}. ` +
+          `Update the config, or delete ${dir} and re-run this script to fix.`
+      );
+    }
+    return;
+  }
+  console.log(`cloning git repo ${repo} to ${dir}`);
+  await execFileAsync('git', ['clone', repo, dir]);
+  console.log(`checking out commit ${commit}`);
+  await execFileAsync('git', ['checkout', commit], {cwd: dir});
+};
+
+/**
+ * Run NPM install and other given setup commands. If a node_modules/ directory
+ * already exists in the given directory, do nothing.
+ */
+const setupIfNeeded = async (
+  dir: string,
+  extraCommands?: Array<{cmd: string; args: string[]}>
+) => {
+  if (await fileExists(pathlib.join(dir, 'node_modules'))) {
+    console.log(`${dir}/node_modules already exists, skipping setup`);
+    return;
+  }
+  console.log(`running npm ci in ${dir}`);
+  await execFileAsync('npm', ['ci'], {cwd: dir});
+  for (const {cmd, args} of extraCommands ?? []) {
+    console.log(`running ${cmd} ${args.join(' ')} in ${dir}`);
+    await execFileAsync(cmd, args, {cwd: dir});
+  }
+};
+
+const analyze = async (config: ApiDocsConfig) => {
+  await cloneIfNeeded(config.repo, config.commit, config.gitDir);
+  await setupIfNeeded(config.gitDir, config.extraSetupCommands);
+
+  const app = new typedoc.Application();
+  app.options.addReader(new typedoc.TSConfigReader());
+  app.bootstrap({
+    tsconfig: config.tsConfigPath,
+    entryPoints: config.entrypointModules,
+  });
+  const root = app.convert();
+  if (!root) {
+    throw new Error('TypeDoc.Application.convert() returned undefined');
+  }
+
+  const json = await app.serializer.projectToObject(root);
+  const transformer = new ApiDocsTransformer(json, config);
+  const {pages, symbolMap} = await transformer.transform();
+
+  await fs.mkdir(pathlib.dirname(config.pagesOutPath), {recursive: true});
+  await fs.writeFile(
+    config.pagesOutPath,
+    JSON.stringify(pages, null, 2),
+    'utf8'
+  );
+  console.log(`Wrote ${config.pagesOutPath}`);
+
+  await fs.mkdir(pathlib.dirname(config.symbolsOutPath), {recursive: true});
+  await fs.writeFile(
+    config.symbolsOutPath,
+    JSON.stringify(symbolMap, null, 2),
+    'utf8'
+  );
+  console.log(`Wrote ${config.symbolsOutPath}`);
+};
+
+const main = async () => {
+  await Promise.all(configs.map((config) => analyze(config)));
+};
+
+main();

packages/lit-dev-content/.eleventy.js

@@ -16,8 +16,7 @@ import {
   ExtendedSourceReference,
   Location,
   ExternalLocation,
-} from './api-docs/types.js';
-import {lit2Config} from './api-docs/configs/lit-2.js';
+} from './types.js';
 
 const findIndexOrInfinity = <T>(
   array: ReadonlyArray<T>,
@@ -82,7 +81,7 @@ type SymbolMap = {
  * Eleventy template, and generate a symbol map that can be used to locate an
  * API within our custom page structure.
  */
-class Transformer {
+export class ApiDocsTransformer {
   private config: ApiDocsConfig;
   private project: typedoc.JSONOutput.ProjectReflection;
   private symbolMap: SymbolMap = {};
@@ -722,35 +721,3 @@ class Transformer {
     return pagesArray;
   }
 }
-
-async function main() {
-  const app = new typedoc.Application();
-  app.options.addReader(new typedoc.TSConfigReader());
-  app.bootstrap({
-    tsconfig: lit2Config.tsConfigPath,
-    entryPoints: lit2Config.entrypointModules,
-  });
-  const root = app.convert();
-  if (!root) {
-    throw new Error('TypeDoc.Application.convert() returned undefined');
-  }
-
-  const json = await app.serializer.projectToObject(root);
-  const transformer = new Transformer(json, lit2Config);
-  const {pages, symbolMap} = await transformer.transform();
-
-  await fs.writeFile(
-    lit2Config.pagesOutPath,
-    JSON.stringify(pages, null, 2),
-    'utf8'
-  );
-  console.log(`Wrote ${lit2Config.pagesOutPath}`);
-  await fs.writeFile(
-    lit2Config.symbolsOutPath,
-    JSON.stringify(symbolMap, null, 2),
-    'utf8'
-  );
-  console.log(`Wrote ${lit2Config.symbolsOutPath}`);
-}
-
-main();

packages/lit-dev-content/site/_data/api-docs-pages.js

@@ -65,6 +65,12 @@ export interface ApiDocsConfig {
    */
   typedocRoot: string;
 
+  /**
+   * Extra setup/build commands to run after NPM install and before running
+   * TypeDoc.
+   */
+  extraSetupCommands?: Array<{cmd: string; args: string[]}>;
+
   /**
    * Entrypoint TypeScript modules for TypeDoc to analyze.
    *