chore: make all packages build to be consistent and compatible with c…

…js and esm (#426)

* Update ts config for all packages

* Update package.json files

* Add file prefixes to all imports

* Update the ts-node config to use esm

* Upgrade typescript

* Upgrade typescript settings

* Fix the TS warnings and issues

* Fix import paths for types

* Update import path

* Update the import paths in tests

* Fix lint errors

* Fix lint formatting on config files

* Update the benchmark to use ts loader

* Disable the browser tests

* Fix the loader for the esm

* fix: convert prettierrc to yaml

* fix: __dirname to ESM

* fix: revert TS version to fix download-spec-tests ts-node issue

* feat: use spec-test-util for ESM and wget compatibility

* fix: remove ts-expect-error after downgrading TS version

* fix: update SPEC_TEST_LOCATION

* chore: update eslint deps to remove lint error

* chore: lint code per updated lint rules from package update

* fix: convert to node 20 compatible dirname

* fix: convert to node 20 compatible dirname

* Cleanup benchmark loader for ts

* Update the hasher for benchmark

* Revert "Update the hasher for benchmark"

This reverts commit fd9adcc.

* Update the hasher for benchmark

* Update package output dirs

* Update hasher path

---------

Co-authored-by: matthewkeil <[email protected]>

.gitignore

@@ -11,6 +11,7 @@ packages/ssz/spec-tests
 
 benchmark_data/
 .failedTest.txt
+*.tsbuildinfo
 
 node_modules
 .yarn/*

package.json

@@ -13,7 +13,7 @@
     "check-types": "lerna run check-types",
     "coverage": "lerna run coverage",
     "test": "lerna run test",
-    "benchmark:files": "NODE_OPTIONS=--max_old_space_size=4096 benchmark --config .benchrc.yaml",
+    "benchmark:files": "NODE_OPTIONS='--max_old_space_size=4096 --loader=ts-node/esm' benchmark --config .benchrc.yaml",
     "benchmark": "yarn benchmark:files 'packages/*/test/perf/**/*.test.ts'",
     "benchmark:local": "yarn benchmark --local",
     "check-readme": "lerna run check-readme"
@@ -28,9 +28,9 @@
     "@dapplion/benchmark": "^0.2.2",
     "@types/chai": "^4.2.15",
     "@types/mocha": "^8.2.2",
-    "@types/node": "^14.14.17",
-    "@typescript-eslint/eslint-plugin": "^4.19.0",
-    "@typescript-eslint/parser": "^4.19.0",
+    "@types/node": "^22.10.2",
+    "@typescript-eslint/eslint-plugin": "^6",
+    "@typescript-eslint/parser": "^6",
     "babel-loader": "^8.2.2",
     "chai": "^4.3.4",
     "eslint": "^7.30.0",
@@ -49,14 +49,14 @@
     "mocha": "^8.3.2",
     "nyc": "^15.0.0",
     "prettier": "^2.2.1",
-    "ts-loader": "^9.2.8",
-    "ts-node": "^9.1.1",
-    "typescript": "~4.2.3",
+    "ts-loader": "^9.5.1",
+    "ts-node": "^10.9.2",
+    "typescript": "~5.4.2",
     "webpack": "^5.75.0",
     "webpack-cli": "^5.0.1",
     "webpack-dev-server": "^3.11.2"
   },
   "resolutions": {
     "@types/react": "^17.0"
   }
-}
+}

packages/as-sha256/.mocharc.yaml

@@ -0,0 +1,7 @@
+colors: true
+require:
+- ts-node/register
+extension: 
+  - "ts"
+loader: 
+  - ts-node/esm

packages/as-sha256/.prettierrc

@@ -0,0 +1,9 @@
+printWidth: 120
+tabWidth: 2
+useTabs: false
+semi: true
+singleQuote: false
+quoteProps: "as-needed"
+trailingComma: "es5"
+bracketSpacing: false
+arrowParens: "always"

packages/as-sha256/package.json

@@ -12,17 +12,10 @@
     "type": "git",
     "url": "git+https://github.com/chainsafe/ssz.git"
   },
-  "main": "lib/index.js",
-  "typesVersions": {
-    "*": {
-      "*": [
-        "*",
-        "lib/*",
-        "lib/*/index"
-      ]
-    }
-  },
-  "types": "lib/index.d.ts",
+  "type": "module",
+  "main": "./lib/cjs/index.js",
+  "module": "./lib/index.js",
+  "types": "./lib/index.d.ts",
   "files": [
     "lib",
     "build"
@@ -31,17 +24,19 @@
     "clean": "rm -rf ./dist",
     "lint": "echo 'no linting for this package'",
     "check-types": "echo 'no type check for this package'",
-    "generate": "rm -rf ./dist && node -r ts-node/register ./scripts/codegen.ts",
-    "build": "yarn asbuild:untouched && yarn asbuild:optimized && yarn build:lib",
+    "generate": "rm -rf ./dist && node --loader ts-node/esm ./scripts/codegen.ts",
+    "build": "yarn asbuild:untouched && yarn asbuild:optimized && yarn build:cjs && yarn build:esm && yarn build:types",
     "asbuild:untouched": "asc assembly/index.ts -o build/untouched.wasm -t build/untouched.wat --runtime minimal --target debug --enable simd",
     "asbuild:optimized": "asc assembly/index.ts -o build/optimized.wasm -t build/optimized.wat --runtime minimal --target release -O3z --noAssert --enable simd",
-    "build:lib": "tsc -p tsconfig.build.json",
+    "build:esm": "tsc -p tsconfig.build.esm.json",
+    "build:cjs": "tsc -p tsconfig.build.cjs.json && echo '{\"type\": \"commonjs\"}' > ./lib/cjs/package.json",
+    "build:types": "tsc -p tsconfig.build.types.json",
     "build:web": "webpack --mode production --entry ./index.js --output ./dist/as-sha256.min.js",
     "test": "yarn run test:unit",
-    "test:unit": "yarn run test:unit:node && yarn run test:unit:browser",
-    "test:unit:node": "mocha -r ts-node/register test/unit/*.test.ts",
-    "test:unit:browser": "karma start --single-run --browsers ChromeHeadless,FirefoxHeadless karma.config.js",
-    "benchmark": "node -r ts-node/register ./node_modules/.bin/benchmark 'test/perf/*.test.ts'",
+    "test:unit": "yarn run test:unit:node",
+    "test:unit:node": "yarn mocha test/unit/*.test.ts",
+    "test:unit:browser": "karma start --single-run --browsers ChromeHeadless,FirefoxHeadless karma.config.cjs",
+    "benchmark": "node --loader ts-node/esm ./node_modules/.bin/benchmark 'test/perf/*.test.ts'",
     "benchmark:local": "yarn benchmark --local",
     "test:ci": "yarn test:as-ci"
   },

packages/as-sha256/src/index.ts

@@ -1,7 +1,7 @@
-import {allocUnsafe} from "./alloc";
-import {newInstance} from "./wasm";
-import {HashObject, byteArrayIntoHashObject, byteArrayToHashObject, hashObjectToByteArray} from "./hashObject";
-import SHA256 from "./sha256";
+import {allocUnsafe} from "./alloc.js";
+import {newInstance} from "./wasm.js";
+import {HashObject, byteArrayIntoHashObject, byteArrayToHashObject, hashObjectToByteArray} from "./hashObject.js";
+import SHA256 from "./sha256.js";
 export {HashObject, byteArrayToHashObject, hashObjectToByteArray, byteArrayIntoHashObject, SHA256};
 
 const ctx = newInstance();

packages/as-sha256/src/sha256.ts

@@ -1,4 +1,4 @@
-import {newInstance, WasmContext} from "./wasm";
+import {newInstance, WasmContext} from "./wasm.js";
 
 /**
  * Class based SHA256

packages/as-sha256/src/wasm.ts

@@ -1,4 +1,4 @@
-import {wasmCode} from "./wasmCode";
+import {wasmCode} from "./wasmCode.js";
 
 const _module = new WebAssembly.Module(wasmCode);
 

packages/as-sha256/test/perf/index.test.ts

@@ -1,5 +1,5 @@
 import {itBench, setBenchOpts} from "@dapplion/benchmark";
-import * as sha256 from "../../src";
+import * as sha256 from "../../src/index.js";
 
 // Feb 2024 Mac M1
 // digestTwoHashObjects vs digest64 vs digest

packages/as-sha256/test/perf/simd.test.ts

@@ -1,6 +1,6 @@
 import {itBench, setBenchOpts} from "@dapplion/benchmark";
-import * as sha256 from "../../src";
-import {byteArrayToHashObject} from "../../src/hashObject";
+import * as sha256 from "../../src/index.js";
+import {byteArrayToHashObject} from "../../src/hashObject.js";
 
 /**
  * This really depends on cpu, on a test ubuntu node, batch*() is 2x faster than digest64

packages/as-sha256/test/unit/index.test.ts

@@ -1,5 +1,5 @@
 import {Buffer} from "buffer";
-import * as sha256 from "../../src";
+import * as sha256 from "../../src/index.js";
 import {expect} from "chai";
 
 describe("sha256", function () {

packages/as-sha256/test/unit/simd.test.ts

@@ -1,7 +1,7 @@
 import {expect} from "chai";
 import crypto from "crypto";
-import {byteArrayToHashObject, hashObjectToByteArray} from "../../src/hashObject";
-import * as sha256 from "../../src";
+import {byteArrayToHashObject, hashObjectToByteArray} from "../../src/hashObject.js";
+import * as sha256 from "../../src/index.js";
 
 describe("Test SIMD implementation of as-sha256", () => {
   it("testHash4UintArray64s", () => {

packages/as-sha256/tsconfig.build.cjs.json

@@ -0,0 +1,8 @@
+{
+  "extends": "../../tsconfig.build.cjs.json",
+  "include": ["src"],
+  "compilerOptions": {
+    "outDir": "lib/cjs",
+    "typeRoots": ["../../node_modules/@types", "./node_modules/@types", "./types"],
+  }
+}

packages/as-sha256/tsconfig.build.esm.json

@@ -0,0 +1,8 @@
+{
+  "extends": "../../tsconfig.build.esm.json",
+  "include": ["src"],
+  "compilerOptions": {
+    "outDir": "lib",
+    "typeRoots": ["../../node_modules/@types", "./node_modules/@types", "./types"],
+  }
+}

packages/as-sha256/tsconfig.build.types.json

@@ -0,0 +1,8 @@
+{
+  "extends": "../../tsconfig.build.types.json",
+  "include": ["src"],
+  "compilerOptions": {
+    "outDir": "lib",
+    "typeRoots": ["../../node_modules/@types", "./node_modules/@types", "./types"],
+  }
+}

packages/as-sha256/tsconfig.json

@@ -1,7 +1,7 @@
 {
+  "extends": "../../tsconfig.json",
   "include": ["src", "test"],
-  "extends": "./tsconfig.build.json",
   "compilerOptions": {
-    "incremental": true
+    "typeRoots": ["../../node_modules/@types", "./node_modules/@types", "./types"],
   }
 }

packages/persistent-merkle-tree/.mocharc.yaml

@@ -2,3 +2,5 @@ colors: true
 require:
 - ts-node/register
 - ../../setHasher.mjs
+loader: 
+  - ts-node/esm

packages/persistent-merkle-tree/package.json

@@ -2,29 +2,26 @@
   "name": "@chainsafe/persistent-merkle-tree",
   "version": "0.8.0",
   "description": "Merkle tree implemented as a persistent datastructure",
-  "main": "lib/index.js",
-  "typesVersions": {
-    "*": {
-      "*": [
-        "*",
-        "lib/*",
-        "lib/*/index"
-      ]
-    }
-  },
+  "type": "module",
+  "module": "./lib/index.js",
+  "main": "./lib/cjs/index.js",
+  "types": "./lib/index.d.ts",
   "files": [
     "lib"
   ],
   "scripts": {
     "check-types": "tsc --noEmit",
     "clean": "rm -rf lib",
-    "build": "tsc",
+    "build": "yarn build:cjs && yarn build:esm && yarn build:types",
+    "build:esm": "tsc -p tsconfig.build.esm.json",
+    "build:cjs": "tsc -p tsconfig.build.cjs.json && echo '{\"type\": \"commonjs\"}' > ./lib/cjs/package.json",
+    "build:types": "tsc -p tsconfig.build.types.json",
     "lint": "eslint --color --ext .ts src/",
     "lint:fix": "yarn run lint --fix",
-    "benchmark:files": "node --max-old-space-size=4096 --expose-gc -r ts-node/register ../../node_modules/.bin/benchmark",
+    "benchmark:files": "node --max-old-space-size=4096 --expose-gc --loader ts-node/esm ../../node_modules/.bin/benchmark",
     "benchmark": "yarn benchmark:files 'test/perf/*.test.ts'",
     "benchmark:local": "yarn benchmark --local",
-    "test": "mocha -r ts-node/register 'test/unit/**/*.test.ts'"
+    "test": "mocha 'test/unit/**/*.test.ts'"
   },
   "pre-push": [
     "lint"

packages/persistent-merkle-tree/src/hashComputation.ts

@@ -1,4 +1,4 @@
-import type {Node} from "./node";
+import type {Node} from "./node.js";
 
 /**
  * HashComputation to be later used to compute hash of nodes from bottom up.

packages/persistent-merkle-tree/src/hasher/as-sha256.ts

@@ -5,10 +5,10 @@ import {
   batchHash4HashObjectInputs,
   hashInto,
 } from "@chainsafe/as-sha256";
-import type {Hasher} from "./types";
-import {Node} from "../node";
-import type {HashComputationLevel} from "../hashComputation";
-import {BLOCK_SIZE, doDigestNLevel, doMerkleizeBlockArray, doMerkleizeBlocksBytes} from "./util";
+import type {Hasher} from "./types.js";
+import {Node} from "../node.js";
+import type {HashComputationLevel} from "../hashComputation.js";
+import {BLOCK_SIZE, doDigestNLevel, doMerkleizeBlockArray, doMerkleizeBlocksBytes} from "./util.js";
 
 /**
  * hashInto() function of as-sha256 loop through every 256 bytes

packages/persistent-merkle-tree/src/hasher/hashtree.ts

@@ -1,9 +1,9 @@
 import {hashInto} from "@chainsafe/hashtree";
-import {Hasher, HashObject} from "./types";
-import {Node} from "../node";
-import type {HashComputationLevel} from "../hashComputation";
-import {byteArrayIntoHashObject} from "@chainsafe/as-sha256/lib/hashObject";
-import {doDigestNLevel, doMerkleizeBlockArray, doMerkleizeBlocksBytes} from "./util";
+import {Hasher, HashObject} from "./types.js";
+import {Node} from "../node.js";
+import type {HashComputationLevel} from "../hashComputation.js";
+import {byteArrayIntoHashObject} from "@chainsafe/as-sha256";
+import {doDigestNLevel, doMerkleizeBlockArray, doMerkleizeBlocksBytes} from "./util.js";
 
 /**
  * Best SIMD implementation is in 512 bits = 64 bytes

packages/persistent-merkle-tree/src/hasher/index.ts

@@ -1,9 +1,9 @@
-import {Hasher} from "./types";
-import {hasher as nobleHasher} from "./noble";
-import type {HashComputationLevel} from "../hashComputation";
+import {Hasher} from "./types.js";
+import {hasher as nobleHasher} from "./noble.js";
+import type {HashComputationLevel} from "../hashComputation.js";
 
-export * from "./types";
-export * from "./util";
+export * from "./types.js";
+export * from "./util.js";
 
 /**
  * Hasher used across the SSZ codebase, by default, this does not support batch hash.

packages/persistent-merkle-tree/src/hasher/noble.ts

@@ -1,13 +1,13 @@
 import {sha256} from "@noble/hashes/sha256";
 import {digest64HashObjects, byteArrayIntoHashObject} from "@chainsafe/as-sha256";
-import type {Hasher} from "./types";
+import type {Hasher} from "./types.js";
 import {
   BLOCK_SIZE,
   doDigestNLevel,
   doMerkleizeBlockArray,
   doMerkleizeBlocksBytes,
   hashObjectToUint8Array,
-} from "./util";
+} from "./util.js";
 
 const digest64 = (a: Uint8Array, b: Uint8Array): Uint8Array => sha256.create().update(a).update(b).digest();
 const hashInto = (input: Uint8Array, output: Uint8Array): void => {