diff --git a/cli/CHANGELOG.md b/cli/CHANGELOG.md index b01b2ac8..19ef294c 100644 --- a/cli/CHANGELOG.md +++ b/cli/CHANGELOG.md @@ -5,6 +5,7 @@ - Refactor `mops install` command - Reduce install threads to 12 (was 16) - Reduce install threads to 6 when install called from `mops sources` +- Install dependencies directly to global cache, copy to local cache only final resolved dependencies ## 0.41.1 - Fix bin path for npm diff --git a/cli/cache.ts b/cli/cache.ts index 58c99a19..bbf921ed 100644 --- a/cli/cache.ts +++ b/cli/cache.ts @@ -3,15 +3,33 @@ import path from 'node:path'; import ncp from 'ncp'; import getFolderSize from 'get-folder-size'; -import {globalCacheDir} from './mops.js'; +import {getDependencyType, globalCacheDir, parseGithubURL} from './mops.js'; -export let isCached = (pkgId : string) => { - let dir = path.join(globalCacheDir, 'packages', pkgId); +export let getDepCacheDir = (cacheName : string) => { + return path.join(globalCacheDir, 'packages', cacheName); +}; + +export let isDepCached = (cacheName : string) => { + let dir = getDepCacheDir(cacheName); return fs.existsSync(dir); }; -export let addCache = (pkgId : string, source : string) => { - let dest = path.join(globalCacheDir, 'packages', pkgId); +export function getDepCacheName(name : string, version : string) { + let depType = getDependencyType(version); + return depType === 'mops' ? getMopsDepCacheName(name, version) : getGithubDepCacheName(name, version); +} + +export function getMopsDepCacheName(name : string, version : string) { + return `${name}@${version}`; +} + +export function getGithubDepCacheName(name : string, repo : string) { + const {branch, commitHash} = parseGithubURL(repo); + return `_github/${name}#${branch}` + (commitHash ? `@${commitHash}` : ''); +} + +export let addCache = (cacheName : string, source : string) => { + let dest = path.join(globalCacheDir, 'packages', cacheName); fs.mkdirSync(dest, {recursive: true}); return new Promise((resolve, reject) => { @@ -24,8 +42,8 @@ export let addCache = (pkgId : string, source : string) => { }); }; -export let copyCache = (pkgId : string, dest : string) => { - let source = path.join(globalCacheDir, 'packages', pkgId); +export let copyCache = (cacheName : string, dest : string) => { + let source = path.join(globalCacheDir, 'packages', cacheName); fs.mkdirSync(dest, {recursive: true}); return new Promise((resolve, reject) => { diff --git a/cli/commands/add.ts b/cli/commands/add.ts index af0be4f7..b962e2a7 100644 --- a/cli/commands/add.ts +++ b/cli/commands/add.ts @@ -6,9 +6,10 @@ import {checkConfigFile, getGithubCommit, parseGithubURL, readConfig, writeConfi import {getHighestVersion} from '../api/getHighestVersion.js'; import {installMopsDep} from './install/install-mops-dep.js'; import {installFromGithub} from '../vessel.js'; -import {notifyInstalls} from '../notify-installs.js'; import {checkIntegrity} from '../integrity.js'; import {checkRequirements} from '../check-requirements.js'; +import {syncLocalCache} from './install/sync-local-cache.js'; +import {notifyInstalls} from '../notify-installs.js'; type AddOptions = { verbose ?: boolean; @@ -87,8 +88,6 @@ export async function add(name : string, {verbose = false, dev = false, lock} : }; } - let installedPackages = {}; - if (pkgDetails.repo) { await installFromGithub(pkgDetails.name, pkgDetails.repo, {verbose: verbose}); } @@ -97,7 +96,6 @@ export async function add(name : string, {verbose = false, dev = false, lock} : if (res === false) { return; } - installedPackages = {...installedPackages, ...res}; } const depsProp = dev ? 'dev-dependencies' : 'dependencies'; @@ -116,8 +114,11 @@ export async function add(name : string, {verbose = false, dev = false, lock} : if (lock !== 'ignore') { logUpdate('Checking integrity...'); } + + let installedPackages = await syncLocalCache(); + await Promise.all([ - notifyInstalls(Object.keys(installedPackages)), + notifyInstalls(installedPackages), checkIntegrity(lock), ]); diff --git a/cli/commands/install/install-all.ts b/cli/commands/install/install-all.ts index ae4f2024..91d2d42a 100644 --- a/cli/commands/install/install-all.ts +++ b/cli/commands/install/install-all.ts @@ -2,10 +2,11 @@ import process from 'node:process'; import chalk from 'chalk'; import {createLogUpdate} from 'log-update'; import {checkConfigFile, readConfig} from '../../mops.js'; -import {notifyInstalls} from '../../notify-installs.js'; import {checkIntegrity} from '../../integrity.js'; import {installDeps} from './install-deps.js'; import {checkRequirements} from '../../check-requirements.js'; +import {syncLocalCache} from './sync-local-cache.js'; +import {notifyInstalls} from '../../notify-installs.js'; type InstallAllOptions = { verbose ?: boolean; @@ -28,7 +29,6 @@ export async function installAll({verbose = false, silent = false, threads, lock if (!res) { return; } - let installedDeps = res; let logUpdate = createLogUpdate(process.stdout, {showCursor: true}); @@ -36,8 +36,10 @@ export async function installAll({verbose = false, silent = false, threads, lock logUpdate('Checking integrity...'); } + let installedPackages = await syncLocalCache(); + await Promise.all([ - notifyInstalls(Object.keys(installedDeps)), + notifyInstalls(installedPackages), checkIntegrity(lock), ]); diff --git a/cli/commands/install/install-dep.ts b/cli/commands/install/install-dep.ts index 21dbcd88..d90646a6 100644 --- a/cli/commands/install/install-dep.ts +++ b/cli/commands/install/install-dep.ts @@ -3,6 +3,7 @@ import {installFromGithub} from '../../vessel.js'; import {installMopsDep} from './install-mops-dep.js'; import {Dependency} from '../../types.js'; import {installLocalDep} from './install-local-dep.js'; +import {getRootDir} from '../../mops.js'; type InstallDepOptions = { verbose ?: boolean; @@ -19,6 +20,7 @@ export async function installDep(dep : Dependency, {verbose, silent, threads} : } else if (dep.path) { let depPath = dep.path; + parentPkgPath = parentPkgPath || getRootDir(); if (parentPkgPath) { depPath = path.resolve(parentPkgPath, dep.path); } diff --git a/cli/commands/install/install-mops-dep.ts b/cli/commands/install/install-mops-dep.ts index 43dd1525..6d0e457a 100644 --- a/cli/commands/install/install-mops-dep.ts +++ b/cli/commands/install/install-mops-dep.ts @@ -4,11 +4,12 @@ import path from 'node:path'; import {Buffer} from 'node:buffer'; import {createLogUpdate} from 'log-update'; import chalk from 'chalk'; +import {deleteSync} from 'del'; import {checkConfigFile, formatDir, progressBar, readConfig} from '../../mops.js'; import {getHighestVersion} from '../../api/getHighestVersion.js'; import {storageActor} from '../../api/actors.js'; import {parallel} from '../../parallel.js'; -import {addCache, copyCache, isCached} from '../../cache.js'; +import {getDepCacheDir, getMopsDepCacheName, isDepCached} from '../../cache.js'; import {downloadFile, getPackageFilesInfo} from '../../api/downloadPackageFiles.js'; import {installDeps} from './install-deps.js'; @@ -46,6 +47,8 @@ export async function installMopsDep(pkg : string, version = '', {verbose, silen } let dir = formatDir(pkg, version); + let cacheName = getMopsDepCacheName(pkg, version); + let cacheDir = getDepCacheDir(cacheName); let alreadyInstalled = false; // already installed @@ -54,8 +57,7 @@ export async function installMopsDep(pkg : string, version = '', {verbose, silen alreadyInstalled = true; } // copy from cache - else if (isCached(`${pkg}@${version}`)) { - await copyCache(`${pkg}@${version}`, dir); + else if (isDepCached(cacheName)) { silent || logUpdate(`${dep ? 'Dependency' : 'Installing'} ${pkg}@${version} (global cache)`); } // download @@ -79,10 +81,17 @@ export async function installMopsDep(pkg : string, version = '', {verbose, silen progress(); }); - // write files to disk - for (let [filePath, data] of filesData.entries()) { - fs.mkdirSync(path.join(dir, path.dirname(filePath)), {recursive: true}); - fs.writeFileSync(path.join(dir, filePath), Buffer.from(data)); + // write files to global cache + try { + for (let [filePath, data] of filesData.entries()) { + fs.mkdirSync(path.join(cacheDir, path.dirname(filePath)), {recursive: true}); + fs.writeFileSync(path.join(cacheDir, filePath), Buffer.from(data)); + } + } + catch (err) { + console.error(chalk.red('Error: ') + err); + deleteSync([cacheDir], {force: true}); + return false; } } catch (err) { @@ -90,9 +99,6 @@ export async function installMopsDep(pkg : string, version = '', {verbose, silen return false; } - // add to cache - await addCache(`${pkg}@${version}`, dir); - progress(); } @@ -104,7 +110,7 @@ export async function installMopsDep(pkg : string, version = '', {verbose, silen } // install dependencies - let config = readConfig(path.join(dir, 'mops.toml')); + let config = readConfig(path.join(cacheDir, 'mops.toml')); let res = await installDeps(Object.values(config.dependencies || {}), {silent, verbose}); if (!res) { diff --git a/cli/commands/install/sync-local-cache.ts b/cli/commands/install/sync-local-cache.ts new file mode 100644 index 00000000..53637c22 --- /dev/null +++ b/cli/commands/install/sync-local-cache.ts @@ -0,0 +1,34 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import {copyCache, getDepCacheName} from '../../cache.js'; +import {getDependencyType, getRootDir} from '../../mops.js'; +import {resolvePackages} from '../../resolve-packages.js'; + +export async function syncLocalCache({verbose = false} = {}) : Promise> { + let resolvedPackages = await resolvePackages(); + let rootDir = getRootDir(); + + verbose && console.log('Syncing local cache...'); + + let installedDeps : Record = {}; + + await Promise.all(Object.entries(resolvedPackages).map(([name, value]) => { + let depType = getDependencyType(value); + + if (depType === 'mops' || depType === 'github') { + let cacheName = getDepCacheName(name, value); + let dest = path.join(rootDir, '.mops', cacheName); + + if (!fs.existsSync(dest)) { + if (depType === 'mops') { + installedDeps[name] = value; + } + return copyCache(cacheName, path.join(rootDir, '.mops', cacheName)); + } + } + + return Promise.resolve(); + })); + + return installedDeps; +} diff --git a/cli/commands/remove.ts b/cli/commands/remove.ts index f714ea68..72ba5fb4 100644 --- a/cli/commands/remove.ts +++ b/cli/commands/remove.ts @@ -1,9 +1,12 @@ import fs from 'node:fs'; import {deleteSync} from 'del'; import chalk from 'chalk'; -import {formatDir, formatGithubDir, checkConfigFile, readConfig, writeConfig} from '../mops.js'; +import {checkConfigFile, getRootDir, readConfig, writeConfig} from '../mops.js'; import {Config, Dependency} from '../types.js'; import {checkIntegrity} from '../integrity.js'; +import {getDepCacheDir, getDepCacheName} from '../cache.js'; +import path from 'node:path'; +import {syncLocalCache} from './install/sync-local-cache.js'; type RemoveOptions = { verbose ?: boolean; @@ -12,7 +15,6 @@ type RemoveOptions = { lock ?: 'update' | 'ignore'; }; -// eslint-disable-next-line @typescript-eslint/no-unused-vars export async function remove(name : string, {dev = false, verbose = false, dryRun = false, lock} : RemoveOptions = {}) { if (!checkConfigFile()) { return; @@ -31,13 +33,12 @@ export async function remove(name : string, {dev = false, verbose = false, dryRu } function getTransitiveDependenciesOf(name : string, version : string | undefined, repo ?: string) { - let pkgDir = ''; - if (repo) { - pkgDir = formatGithubDir(name, repo); - } - else if (version) { - pkgDir = formatDir(name, version); + let value = version || repo; + if (!value) { + return []; } + let cacheName = getDepCacheName(name, value); + let pkgDir = getDepCacheDir(cacheName); let configFile = pkgDir + '/mops.toml'; if (!fs.existsSync(configFile)) { verbose && console.log('no config', configFile); @@ -78,16 +79,11 @@ export async function remove(name : string, {dev = false, verbose = false, dryRu verbose && console.log(`Ignored transitive dependency ${depId} (other deps depend on it)`); continue; } - let pkgDir; - if (dep.repo) { - pkgDir = formatGithubDir(dep.name, dep.repo); - } - else if (dep.version) { - pkgDir = formatDir(dep.name, dep.version); - } - if (pkgDir && fs.existsSync(pkgDir)) { - dryRun || deleteSync([`${pkgDir}`], {force: true}); - verbose && console.log(`Removed local cache ${pkgDir}`); + let cacheName = getDepCacheName(dep.name, dep.version || dep.repo || ''); + let localCacheDir = path.join(getRootDir(), '.mops', cacheName); + if (localCacheDir && fs.existsSync(localCacheDir)) { + dryRun || deleteSync([localCacheDir], {force: true}); + verbose && console.log(`Removed local cache ${localCacheDir}`); } } @@ -100,6 +96,7 @@ export async function remove(name : string, {dev = false, verbose = false, dryRu } dryRun || writeConfig(config); + await syncLocalCache(); await checkIntegrity(lock); console.log(chalk.green('Package removed ') + `${name} = "${version}"`); diff --git a/cli/notify-installs.ts b/cli/notify-installs.ts index a9696a58..b488b1ff 100644 --- a/cli/notify-installs.ts +++ b/cli/notify-installs.ts @@ -1,12 +1,16 @@ import {getDependencyType} from './mops.js'; import {mainActor} from './api/actors.js'; -import {resolvePackages} from './resolve-packages.js'; -export async function notifyInstalls(names : string[]) { - let resolvedPackages = await resolvePackages(); - let packages : [string, string][] = names.map(name => [name, resolvedPackages[name] as string]); +export async function notifyInstalls(installedDeps : Record) { + let packages = Object.entries(installedDeps).filter(([_, version]) => getDependencyType(version) === 'mops'); if (packages.length) { let actor = await mainActor(); - await actor.notifyInstalls(packages.filter(([_, version]) => getDependencyType(version) === 'mops')); + + try { + await actor.notifyInstalls(packages); + } + catch (err) { + // verbose && console.error('Failed to notify installs:', err); + } } } \ No newline at end of file diff --git a/cli/resolve-packages.ts b/cli/resolve-packages.ts index 08ccf5fc..f92c5803 100644 --- a/cli/resolve-packages.ts +++ b/cli/resolve-packages.ts @@ -1,9 +1,10 @@ import process from 'node:process'; import path from 'node:path'; import chalk from 'chalk'; -import {checkConfigFile, formatDir, formatGithubDir, getRootDir, parseGithubURL, readConfig} from './mops.js'; +import {checkConfigFile, getRootDir, parseGithubURL, readConfig} from './mops.js'; import {VesselConfig, readVesselConfig} from './vessel.js'; import {Config, Dependency} from './types.js'; +import {getDepCacheDir, getDepCacheName} from './cache.js'; export async function resolvePackages({verbose = false} = {}) : Promise> { if (!checkConfigFile()) { @@ -76,25 +77,25 @@ export async function resolvePackages({verbose = false} = {}) : Promise { if (existsSync(filePath)) { @@ -150,38 +150,35 @@ export const downloadFromGithub = async (repo : string, dest : string, onProgres }; export const installFromGithub = async (name : string, repo : string, {verbose = false, dep = false, silent = false} = {}) => { - let {branch, commitHash} = parseGithubURL(repo); let dir = formatGithubDir(name, repo); - let cacheName = `_github/${name}#${branch}` + (commitHash ? `@${commitHash}` : ''); + let cacheName = getGithubDepCacheName(name, repo); + let cacheDir = getDepCacheDir(cacheName); let logUpdate = createLogUpdate(process.stdout, {showCursor: true}); if (existsSync(dir)) { silent || logUpdate(`${dep ? 'Dependency' : 'Installing'} ${repo} (local cache)`); } - else if (isCached(cacheName)) { - await copyCache(cacheName, dir); + else if (isDepCached(cacheName)) { silent || logUpdate(`${dep ? 'Dependency' : 'Installing'} ${repo} (global cache)`); } else { - mkdirSync(dir, {recursive: true}); let progress = (step : number, total : number) => { silent || logUpdate(`${dep ? 'Dependency' : 'Installing'} ${repo} ${progressBar(step, total)}`); }; - progress(0, 2 * (1024 ** 2)); + progress(0, 1024 * 500); + + mkdirSync(cacheDir, {recursive: true}); try { - await downloadFromGithub(repo, dir, progress); + await downloadFromGithub(repo, cacheDir, progress); } catch (err) { - deleteSync([dir]); + deleteSync([cacheDir]); process.exit(1); } - - // add to cache - await addCache(cacheName, dir); } if (verbose) {