chore(core): enable package exports in smithy/core (#1281)

* chore(core): enable package exports in smithy/core

* add CLI dispatcher

* add cbor submodule stub, update Inliner.js script

* enforce files field in package.json files

* chore(core): move core components to core submodules

* add readme for core submodules

* linting

* accidental recursion

* restore existing core/src

* remove unused metadata

* formatting

* handle non-folder in submodules

* pare down metadata

* allow non-submodules in core

Author: kuhe

.changeset/thin-adults-exist.md

@@ -0,0 +1,5 @@
+---
+"@smithy/core": minor
+---
+
+enable package.json exports in core

packages/core/README.md

@@ -5,8 +5,41 @@
 
 > An internal package. You probably shouldn't use this package, at least directly.
 
-## Usage
-
 This package provides common or core functionality for generic Smithy clients.
 
 You do not need to explicitly install this package, since it will be installed during code generation if used.
+
+## Development of `@smithy/core` submodules
+
+Core submodules are organized for distribution via the `package.json` `exports` field.
+
+`exports` is supported by default by the latest Node.js, webpack, and esbuild. For react-native, it can be
+enabled via instructions found at [reactnative.dev/blog](https://reactnative.dev/blog/2023/06/21/package-exports-support), but we also provide a compatibility redirect.
+
+Think of `@smithy/core` as a mono-package within the monorepo.
+It preserves the benefits of modularization, for example to optimize Node.js initialization speed,
+while making it easier to have a consistent version of core dependencies, reducing package sprawl when
+installing a Smithy runtime client.
+
+### Guide for submodules
+
+- Each `index.ts` file corresponding to the pattern `./src/submodules/<MODULE_NAME>/index.ts` will be
+  published as a separate `dist-cjs` bundled submodule index using the `Inliner.js` build script.
+- create a folder as `./src/submodules/<SUBMODULE>` including an `index.ts` file and a `README.md` file.
+  - The linter will throw an error on missing submodule metadata in `package.json` and the various `tsconfig.json` files, but it will automatically fix them if possible.
+- a submodule is equivalent to a standalone `@smithy/<pkg>` package in that importing it in Node.js will resolve a separate bundle.
+- submodules may not relatively import files from other submodules. Instead, directly use the `@scope/pkg/submodule` name as the import.
+  - The linter will check for this and throw an error.
+- To the extent possible, correctly declaring submodule metadata is validated by the linter in `@smithy/core`.
+  The linter runs during `yarn build` and also as `yarn lint`.
+
+### When should I create an `@smithy/core/submodule` vs. `@smithy/new-package`?
+
+Keep in mind that the core package is installed by all downstream clients.
+
+If the component functionality is upstream of multiple clients, it is
+a good candidate for a core submodule. For example, if `middleware-retry` had been written
+after the support for submodules was added, it would have been a submodule.
+
+If the component's functionality is downstream of a client (rare), or only expected to be used by a very small
+subset of clients, it could be written as a standalone package.

packages/core/package.json

@@ -2,20 +2,33 @@
   "name": "@smithy/core",
   "version": "2.0.1",
   "scripts": {
-    "build": "concurrently 'yarn:build:cjs' 'yarn:build:es' 'yarn:build:types && yarn build:types:downlevel'",
+    "build": "yarn lint && concurrently 'yarn:build:cjs' 'yarn:build:es' 'yarn:build:types && yarn build:types:downlevel'",
     "build:cjs": "node ../../scripts/inline core",
     "build:es": "yarn g:tsc -p tsconfig.es.json",
     "build:types": "yarn g:tsc -p tsconfig.types.json",
     "build:types:downlevel": "downlevel-dts dist-types dist-types/ts3.4",
     "stage-release": "rimraf ./.release && yarn pack && mkdir ./.release && tar zxvf ./package.tgz --directory ./.release && rm ./package.tgz",
     "clean": "rimraf ./dist-* && rimraf *.tsbuildinfo || exit 0",
-    "lint": "eslint -c ../../.eslintrc.js \"src/**/*.ts\"",
+    "lint": "npx eslint -c ../../.eslintrc.js \"src/**/*.ts\" && node ./scripts/lint",
     "format": "prettier --config ../../prettier.config.js --ignore-path ../.prettierignore --write \"**/*.{ts,md,json}\"",
     "test": "yarn g:jest"
   },
   "main": "./dist-cjs/index.js",
   "module": "./dist-es/index.js",
   "types": "./dist-types/index.d.ts",
+  "exports": {
+    ".": {
+      "node": "./dist-cjs/index.js",
+      "import": "./dist-es/index.js",
+      "require": "./dist-cjs/index.js",
+      "types": "./dist-types/index.d.ts"
+    },
+    "./package.json": {
+      "node": "./package.json",
+      "import": "./package.json",
+      "require": "./package.json"
+    }
+  },
   "author": {
     "name": "AWS Smithy Team",
     "email": "",

packages/core/scripts/lint.js

@@ -0,0 +1,98 @@
+const fs = require("fs");
+const path = require("path");
+
+const root = path.join(__dirname, "..");
+
+const pkgJson = require(path.join(root, "package.json"));
+const tsconfigs = {
+  cjs: require(path.join(root, "tsconfig.cjs.json")),
+  es: require(path.join(root, "tsconfig.es.json")),
+  types: require(path.join(root, "tsconfig.types.json")),
+};
+const submodules = fs.readdirSync(path.join(root, "src", "submodules"));
+
+const errors = [];
+
+for (const submodule of submodules) {
+  const submodulePath = path.join(root, "src", "submodules", submodule);
+  if (fs.existsSync(submodulePath) && fs.lstatSync(submodulePath).isDirectory()) {
+    // package.json metadata.
+    if (!pkgJson.exports[`./${submodule}`]) {
+      errors.push(`${submodule} submodule is missing exports statement in package.json`);
+      pkgJson.exports[`./${submodule}`] = {
+        node: `./dist-cjs/submodules/${submodule}/index.js`,
+        import: `./dist-es/submodules/${submodule}/index.js`,
+        require: `./dist-cjs/submodules/${submodule}/index.js`,
+        types: `./dist-types/submodules/${submodule}/index.d.ts`,
+      };
+      fs.writeFileSync(path.join(root, "package.json"), JSON.stringify(pkgJson, null, 2) + "\n");
+    }
+    if (!pkgJson.files.includes(`./${submodule}.js`)) {
+      pkgJson.files.push(`./${submodule}.js`);
+      errors.push(`package.json files array missing ${submodule}.js compatibility redirect file.`);
+      fs.writeFileSync(path.join(root, "package.json"), JSON.stringify(pkgJson, null, 2) + "\n");
+    }
+    // tsconfig metadata.
+    for (const [kind, tsconfig] of Object.entries(tsconfigs)) {
+      if (!tsconfig.compilerOptions.paths?.[`@smithy/core/${submodule}`]) {
+        errors.push(`${submodule} submodule is missing paths entry in tsconfig.${kind}.json`);
+
+        tsconfig.compilerOptions.paths[`@smithy/core/${submodule}`] = [`./src/submodules/${submodule}/index.ts`];
+        fs.writeFileSync(path.join(root, `tsconfig.${kind}.json`), JSON.stringify(tsconfig, null, 2) + "\n");
+      }
+    }
+    // compatibility redirect file.
+    const compatibilityRedirectFile = path.join(root, `${submodule}.js`);
+    if (!fs.existsSync(compatibilityRedirectFile)) {
+      errors.push(`${submodule} is missing compatibility redirect file in the package root folder.`);
+      fs.writeFileSync(
+        compatibilityRedirectFile,
+        `
+/**
+ * Do not edit:
+ * This is a compatibility redirect for contexts that do not understand package.json exports field.
+ */
+module.exports = require("./dist-cjs/submodules/${submodule}/index.js");
+`
+      );
+    }
+  }
+}
+
+/**
+ * Check for cross-submodule relative imports.
+ */
+
+const walk = require("../../../scripts/utils/walk");
+
+(async () => {
+  for await (const item of walk(path.join(root, "src", "submodules"))) {
+    // depth within the submodule where 1 is at the root of the submodule.
+    const depth = item.split("core/src/submodules/")[1].split("/").length - 1;
+    const sourceCode = fs.readFileSync(item, "utf-8");
+
+    const relativeImports = [];
+    relativeImports.push(
+      ...new Set(
+        [...(sourceCode.toString().match(/(from |import\()"(.*?)";/g) || [])]
+          .map((_) => _.replace(/from "/g, "").replace(/";$/, ""))
+          .filter((_) => _.startsWith("."))
+      )
+    );
+
+    for (const i of relativeImports) {
+      const relativeImportDepth = i.split("..").length - 1;
+      if (relativeImportDepth >= depth) {
+        errors.push(
+          `relative import ${i} in ${item
+            .split("packages/")
+            .pop()} crosses submodule boundaries. Use @scope/package/submodule import instead.`
+        );
+      }
+    }
+  }
+})().then(() => {
+  if (errors.length) {
+    throw new Error(errors.join("\n"));
+  }
+});

packages/core/src/submodules/.gitkeep

@@ -0,0 +1 @@
+placeholder until submodules exist

packages/core/tsconfig.cjs.json

@@ -2,7 +2,8 @@
   "compilerOptions": {
     "baseUrl": ".",
     "outDir": "dist-cjs",
-    "rootDir": "src"
+    "rootDir": "src",
+    "paths": {}
   },
   "extends": "../../tsconfig.cjs.json",
   "include": ["src/"]

packages/core/tsconfig.es.json

@@ -3,7 +3,8 @@
     "baseUrl": ".",
     "lib": ["dom"],
     "outDir": "dist-es",
-    "rootDir": "src"
+    "rootDir": "src",
+    "paths": {}
   },
   "extends": "../../tsconfig.es.json",
   "include": ["src/"]

packages/core/tsconfig.types.json

@@ -2,7 +2,8 @@
   "compilerOptions": {
     "baseUrl": ".",
     "declarationDir": "dist-types",
-    "rootDir": "src"
+    "rootDir": "src",
+    "paths": {}
   },
   "extends": "../../tsconfig.types.json",
   "include": ["src/"]

scripts/cli-dispatcher/index.js

@@ -0,0 +1,139 @@
+#!/usr/bin/env node
+
+const fs = require("fs");
+const path = require("path");
+const readline = require("readline");
+
+const findFolders = require("./lib/findFolders");
+const findScripts = require("./lib/findScripts");
+const Package = require("./lib/Package");
+
+/**
+ * This script takes your command line arguments and infers the
+ * package in which to execute them.
+ *
+ * It is supposed to save time moving among packages folders
+ * for building and running other test commands.
+ */
+async function main() {
+  console.log("CLI dispatcher");
+
+  const root = path.join(__dirname, "..", "..");
+  const argv = process.argv;
+
+  const packages = fs.readdirSync(path.join(root, "packages"));
+  const private = fs.readdirSync(path.join(root, "private"));
+
+  const allPackages = [
+    ...packages.map((p) => new Package(p, path.join(root, "packages", p))),
+    ...private.map((p) => new Package(p, path.join(root, "private", p))),
+  ];
+
+  const [node, dispatcher, ...rest] = argv;
+  const flags = rest.filter((f) => f.startsWith("--"));
+  const options = {
+    dry: flags.includes("--dry"),
+    help: flags.includes("--help") || rest.length === 0,
+    confirm: flags.includes("--c"),
+  };
+
+  if (options.help) {
+    console.info(`
+    Usage:
+      b [package query words] - [command query words]
+      b cor - b t
+
+      matches to:
+      (cd packages/core && yarn build:types)
+
+      Query words are substrings that match against the package name and npm scripts.
+      The substrings must appear in order for a match.
+      Match priority goes to whole-word matching and initial matching.
+
+    Options:
+      --dry
+        dry run with no command execution.
+      --help
+        show this message.
+      --c
+        ask for confirmation before executing command.
+`);
+    return 0;
+  }
+
+  const nonFlags = rest.filter((_) => !_.startsWith("--"));
+  const separatorIndex = rest.indexOf("-") !== -1 ? rest.indexOf("-") : rest.length;
+  const query = nonFlags.slice(0, separatorIndex);
+  const commands = nonFlags.slice(separatorIndex + 1);
+
+  const matchedPackages = findFolders(allPackages, ...query);
+
+  if (matchedPackages.length === 0) {
+    console.error("No matching packages for query:", query);
+    return 0;
+  }
+
+  console.log("query:", ...query);
+  console.log(
+    "matches:",
+    matchedPackages.map((_) => _.name)
+  );
+
+  const [target] = matchedPackages;
+
+  const targetPkgJson = require(path.join(target.location, "package.json"));
+  const matchedScripts = findScripts(Object.keys(targetPkgJson.scripts || {}), ...commands);
+  const [script] = matchedScripts;
+
+  if (commands.length === 0) {
+    console.info("No commands entered");
+    return 0;
+  }
+
+  if (matchedScripts.length === 0) {
+    console.error("No matching scripts for command query:", commands);
+    return 0;
+  }
+
+  console.log("commands:", ...commands);
+  console.log("matched commands:", matchedScripts);
+
+  const command = `yarn ${script} in ${target.location}`;
+
+  if (options.dry) {
+    console.log("DRYRUN:", command);
+    return 0;
+  }
+
+  const execute = async () => {
+    const { spawnProcess } = require("../utils/spawn-process");
+    console.info("Running:", "yarn", script);
+    console.info("Location:", target.location);
+    await spawnProcess("yarn", [script], {
+      cwd: target.location,
+      stdio: "inherit",
+    });
+    return;
+  };
+
+  if (options.confirm) {
+    const rl = readline.createInterface({
+      input: process.stdin,
+      output: process.stdout,
+    });
+
+    rl.question(`run script "${script}" in ${target.location} (y)/n?:`, async (confirm) => {
+      if (confirm.toLowerCase().trim() === "y" || confirm === "") {
+        await execute();
+      }
+      rl.close();
+    });
+    return 0;
+  }
+
+  await execute();
+
+  return 0;
+}
+
+main().catch(console.error);

scripts/cli-dispatcher/lib/Package.js

@@ -0,0 +1,6 @@
+module.exports = class Package {
+  constructor(name, location) {
+    this.name = name;
+    this.location = location;
+  }
+};

scripts/cli-dispatcher/lib/findFolders.js

@@ -0,0 +1,19 @@
+const matcher = require("./matcher");
+const matchSorter = require("./matchSorter");
+
+/**
+ * @param allPackages {Package[]} - list of all packages.
+ * @param query {string} - query for the package list.
+ * @returns the folders matching the args.
+ */
+module.exports = function findFolders(allPackages, ...query) {
+  const folders = [];
+  for (const pkg of allPackages) {
+    const { name } = pkg;
+    const isMatch = matcher(name, ...query);
+    if (isMatch) {
+      folders.push(pkg);
+    }
+  }
+  return matchSorter(folders, ...query);
+};