Merge pull request #218 from ZenVoich/cli/global-cache-install-resolve

install/resolve via global cache

cli/CHANGELOG.md

@@ -5,6 +5,7 @@
 - Refactor `mops install` command
 - Reduce install threads to 12 (was 16)
 - Reduce install threads to 6 when install called from `mops sources`
+- Install dependencies directly to global cache, copy to local cache only final resolved dependencies
 
 ## 0.41.1
 - Fix bin path for npm

cli/cache.ts

@@ -3,15 +3,33 @@ import path from 'node:path';
 import ncp from 'ncp';
 import getFolderSize from 'get-folder-size';
 
-import {globalCacheDir} from './mops.js';
+import {getDependencyType, globalCacheDir, parseGithubURL} from './mops.js';
 
-export let isCached = (pkgId : string) => {
-	let dir = path.join(globalCacheDir, 'packages', pkgId);
+export let getDepCacheDir = (cacheName : string) => {
+	return path.join(globalCacheDir, 'packages', cacheName);
+};
+
+export let isDepCached = (cacheName : string) => {
+	let dir = getDepCacheDir(cacheName);
 	return fs.existsSync(dir);
 };
 
-export let addCache = (pkgId : string, source : string) => {
-	let dest = path.join(globalCacheDir, 'packages', pkgId);
+export function getDepCacheName(name : string, version : string) {
+	let depType = getDependencyType(version);
+	return depType === 'mops' ? getMopsDepCacheName(name, version) : getGithubDepCacheName(name, version);
+}
+
+export function getMopsDepCacheName(name : string, version : string) {
+	return `${name}@${version}`;
+}
+
+export function getGithubDepCacheName(name : string, repo : string) {
+	const {branch, commitHash} = parseGithubURL(repo);
+	return `_github/${name}#${branch}` + (commitHash ? `@${commitHash}` : '');
+}
+
+export let addCache = (cacheName : string, source : string) => {
+	let dest = path.join(globalCacheDir, 'packages', cacheName);
 	fs.mkdirSync(dest, {recursive: true});
 
 	return new Promise<void>((resolve, reject) => {
@@ -24,8 +42,8 @@ export let addCache = (pkgId : string, source : string) => {
 	});
 };
 
-export let copyCache = (pkgId : string, dest : string) => {
-	let source = path.join(globalCacheDir, 'packages', pkgId);
+export let copyCache = (cacheName : string, dest : string) => {
+	let source = path.join(globalCacheDir, 'packages', cacheName);
 	fs.mkdirSync(dest, {recursive: true});
 
 	return new Promise<void>((resolve, reject) => {

cli/commands/add.ts

@@ -6,9 +6,10 @@ import {checkConfigFile, getGithubCommit, parseGithubURL, readConfig, writeConfi
 import {getHighestVersion} from '../api/getHighestVersion.js';
 import {installMopsDep} from './install/install-mops-dep.js';
 import {installFromGithub} from '../vessel.js';
-import {notifyInstalls} from '../notify-installs.js';
 import {checkIntegrity} from '../integrity.js';
 import {checkRequirements} from '../check-requirements.js';
+import {syncLocalCache} from './install/sync-local-cache.js';
+import {notifyInstalls} from '../notify-installs.js';
 
 type AddOptions = {
 	verbose ?: boolean;
@@ -87,8 +88,6 @@ export async function add(name : string, {verbose = false, dev = false, lock} :
 		};
 	}
 
-	let installedPackages = {};
-
 	if (pkgDetails.repo) {
 		await installFromGithub(pkgDetails.name, pkgDetails.repo, {verbose: verbose});
 	}
@@ -97,7 +96,6 @@ export async function add(name : string, {verbose = false, dev = false, lock} :
 		if (res === false) {
 			return;
 		}
-		installedPackages = {...installedPackages, ...res};
 	}
 
 	const depsProp = dev ? 'dev-dependencies' : 'dependencies';
@@ -116,8 +114,11 @@ export async function add(name : string, {verbose = false, dev = false, lock} :
 	if (lock !== 'ignore') {
 		logUpdate('Checking integrity...');
 	}
+
+	let installedPackages = await syncLocalCache();
+
 	await Promise.all([
-		notifyInstalls(Object.keys(installedPackages)),
+		notifyInstalls(installedPackages),
 		checkIntegrity(lock),
 	]);
 

cli/commands/install/install-all.ts

@@ -2,10 +2,11 @@ import process from 'node:process';
 import chalk from 'chalk';
 import {createLogUpdate} from 'log-update';
 import {checkConfigFile, readConfig} from '../../mops.js';
-import {notifyInstalls} from '../../notify-installs.js';
 import {checkIntegrity} from '../../integrity.js';
 import {installDeps} from './install-deps.js';
 import {checkRequirements} from '../../check-requirements.js';
+import {syncLocalCache} from './sync-local-cache.js';
+import {notifyInstalls} from '../../notify-installs.js';
 
 type InstallAllOptions = {
 	verbose ?: boolean;
@@ -28,16 +29,17 @@ export async function installAll({verbose = false, silent = false, threads, lock
 	if (!res) {
 		return;
 	}
-	let installedDeps = res;
 
 	let logUpdate = createLogUpdate(process.stdout, {showCursor: true});
 
 	if (!silent && lock !== 'ignore') {
 		logUpdate('Checking integrity...');
 	}
 
+	let installedPackages = await syncLocalCache();
+
 	await Promise.all([
-		notifyInstalls(Object.keys(installedDeps)),
+		notifyInstalls(installedPackages),
 		checkIntegrity(lock),
 	]);
 

cli/commands/install/install-dep.ts

@@ -3,6 +3,7 @@ import {installFromGithub} from '../../vessel.js';
 import {installMopsDep} from './install-mops-dep.js';
 import {Dependency} from '../../types.js';
 import {installLocalDep} from './install-local-dep.js';
+import {getRootDir} from '../../mops.js';
 
 type InstallDepOptions = {
 	verbose ?: boolean;
@@ -19,6 +20,7 @@ export async function installDep(dep : Dependency, {verbose, silent, threads} :
 	}
 	else if (dep.path) {
 		let depPath = dep.path;
+		parentPkgPath = parentPkgPath || getRootDir();
 		if (parentPkgPath) {
 			depPath = path.resolve(parentPkgPath, dep.path);
 		}

cli/commands/install/install-mops-dep.ts

@@ -4,11 +4,12 @@ import path from 'node:path';
 import {Buffer} from 'node:buffer';
 import {createLogUpdate} from 'log-update';
 import chalk from 'chalk';
+import {deleteSync} from 'del';
 import {checkConfigFile, formatDir, progressBar, readConfig} from '../../mops.js';
 import {getHighestVersion} from '../../api/getHighestVersion.js';
 import {storageActor} from '../../api/actors.js';
 import {parallel} from '../../parallel.js';
-import {addCache, copyCache, isCached} from '../../cache.js';
+import {getDepCacheDir, getMopsDepCacheName, isDepCached} from '../../cache.js';
 import {downloadFile, getPackageFilesInfo} from '../../api/downloadPackageFiles.js';
 import {installDeps} from './install-deps.js';
 
@@ -46,6 +47,8 @@ export async function installMopsDep(pkg : string, version = '', {verbose, silen
 	}
 
 	let dir = formatDir(pkg, version);
+	let cacheName = getMopsDepCacheName(pkg, version);
+	let cacheDir = getDepCacheDir(cacheName);
 	let alreadyInstalled = false;
 
 	// already installed
@@ -54,8 +57,7 @@ export async function installMopsDep(pkg : string, version = '', {verbose, silen
 		alreadyInstalled = true;
 	}
 	// copy from cache
-	else if (isCached(`${pkg}@${version}`)) {
-		await copyCache(`${pkg}@${version}`, dir);
+	else if (isDepCached(cacheName)) {
 		silent || logUpdate(`${dep ? 'Dependency' : 'Installing'} ${pkg}@${version} (global cache)`);
 	}
 	// download
@@ -79,20 +81,24 @@ export async function installMopsDep(pkg : string, version = '', {verbose, silen
 				progress();
 			});
 
-			// write files to disk
-			for (let [filePath, data] of filesData.entries()) {
-				fs.mkdirSync(path.join(dir, path.dirname(filePath)), {recursive: true});
-				fs.writeFileSync(path.join(dir, filePath), Buffer.from(data));
+			// write files to global cache
+			try {
+				for (let [filePath, data] of filesData.entries()) {
+					fs.mkdirSync(path.join(cacheDir, path.dirname(filePath)), {recursive: true});
+					fs.writeFileSync(path.join(cacheDir, filePath), Buffer.from(data));
+				}
+			}
+			catch (err) {
+				console.error(chalk.red('Error: ') + err);
+				deleteSync([cacheDir], {force: true});
+				return false;
 			}
 		}
 		catch (err) {
 			console.error(chalk.red('Error: ') + err);
 			return false;
 		}
 
-		// add to cache
-		await addCache(`${pkg}@${version}`, dir);
-
 		progress();
 	}
 
@@ -104,7 +110,7 @@ export async function installMopsDep(pkg : string, version = '', {verbose, silen
 	}
 
 	// install dependencies
-	let config = readConfig(path.join(dir, 'mops.toml'));
+	let config = readConfig(path.join(cacheDir, 'mops.toml'));
 	let res = await installDeps(Object.values(config.dependencies || {}), {silent, verbose});
 
 	if (!res) {

cli/commands/install/sync-local-cache.ts

@@ -0,0 +1,34 @@
+import fs from 'node:fs';
+import path from 'node:path';
+import {copyCache, getDepCacheName} from '../../cache.js';
+import {getDependencyType, getRootDir} from '../../mops.js';
+import {resolvePackages} from '../../resolve-packages.js';
+
+export async function syncLocalCache({verbose = false} = {}) : Promise<Record<string, string>> {
+	let resolvedPackages = await resolvePackages();
+	let rootDir = getRootDir();
+
+	verbose && console.log('Syncing local cache...');
+
+	let installedDeps : Record<string, string> = {};
+
+	await Promise.all(Object.entries(resolvedPackages).map(([name, value]) => {
+		let depType = getDependencyType(value);
+
+		if (depType === 'mops' || depType === 'github') {
+			let cacheName = getDepCacheName(name, value);
+			let dest = path.join(rootDir, '.mops', cacheName);
+
+			if (!fs.existsSync(dest)) {
+				if (depType === 'mops') {
+					installedDeps[name] = value;
+				}
+				return copyCache(cacheName, path.join(rootDir, '.mops', cacheName));
+			}
+		}
+
+		return Promise.resolve();
+	}));
+
+	return installedDeps;
+}

cli/commands/remove.ts

@@ -1,9 +1,12 @@
 import fs from 'node:fs';
 import {deleteSync} from 'del';
 import chalk from 'chalk';
-import {formatDir, formatGithubDir, checkConfigFile, readConfig, writeConfig} from '../mops.js';
+import {checkConfigFile, getRootDir, readConfig, writeConfig} from '../mops.js';
 import {Config, Dependency} from '../types.js';
 import {checkIntegrity} from '../integrity.js';
+import {getDepCacheDir, getDepCacheName} from '../cache.js';
+import path from 'node:path';
+import {syncLocalCache} from './install/sync-local-cache.js';
 
 type RemoveOptions = {
 	verbose ?: boolean;
@@ -12,7 +15,6 @@ type RemoveOptions = {
 	lock ?: 'update' | 'ignore';
 };
 
-// eslint-disable-next-line @typescript-eslint/no-unused-vars
 export async function remove(name : string, {dev = false, verbose = false, dryRun = false, lock} : RemoveOptions = {}) {
 	if (!checkConfigFile()) {
 		return;
@@ -31,13 +33,12 @@ export async function remove(name : string, {dev = false, verbose = false, dryRu
 	}
 
 	function getTransitiveDependenciesOf(name : string, version : string | undefined, repo ?: string) {
-		let pkgDir = '';
-		if (repo) {
-			pkgDir = formatGithubDir(name, repo);
-		}
-		else if (version) {
-			pkgDir = formatDir(name, version);
+		let value = version || repo;
+		if (!value) {
+			return [];
 		}
+		let cacheName = getDepCacheName(name, value);
+		let pkgDir = getDepCacheDir(cacheName);
 		let configFile = pkgDir + '/mops.toml';
 		if (!fs.existsSync(configFile)) {
 			verbose && console.log('no config', configFile);
@@ -78,16 +79,11 @@ export async function remove(name : string, {dev = false, verbose = false, dryRu
 			verbose && console.log(`Ignored transitive dependency ${depId} (other deps depend on it)`);
 			continue;
 		}
-		let pkgDir;
-		if (dep.repo) {
-			pkgDir = formatGithubDir(dep.name, dep.repo);
-		}
-		else if (dep.version) {
-			pkgDir = formatDir(dep.name, dep.version);
-		}
-		if (pkgDir && fs.existsSync(pkgDir)) {
-			dryRun || deleteSync([`${pkgDir}`], {force: true});
-			verbose && console.log(`Removed local cache ${pkgDir}`);
+		let cacheName = getDepCacheName(dep.name, dep.version || dep.repo || '');
+		let localCacheDir = path.join(getRootDir(), '.mops', cacheName);
+		if (localCacheDir && fs.existsSync(localCacheDir)) {
+			dryRun || deleteSync([localCacheDir], {force: true});
+			verbose && console.log(`Removed local cache ${localCacheDir}`);
 		}
 	}
 
@@ -100,6 +96,7 @@ export async function remove(name : string, {dev = false, verbose = false, dryRu
 	}
 	dryRun || writeConfig(config);
 
+	await syncLocalCache();
 	await checkIntegrity(lock);
 
 	console.log(chalk.green('Package removed ') + `${name} = "${version}"`);

cli/notify-installs.ts

@@ -1,12 +1,16 @@
 import {getDependencyType} from './mops.js';
 import {mainActor} from './api/actors.js';
-import {resolvePackages} from './resolve-packages.js';
 
-export async function notifyInstalls(names : string[]) {
-	let resolvedPackages = await resolvePackages();
-	let packages : [string, string][] = names.map(name => [name, resolvedPackages[name] as string]);
+export async function notifyInstalls(installedDeps : Record<string, string>) {
+	let packages = Object.entries(installedDeps).filter(([_, version]) => getDependencyType(version) === 'mops');
 	if (packages.length) {
 		let actor = await mainActor();
-		await actor.notifyInstalls(packages.filter(([_, version]) => getDependencyType(version) === 'mops'));
+
+		try {
+			await actor.notifyInstalls(packages);
+		}
+		catch (err) {
+			// verbose && console.error('Failed to notify installs:', err);
+		}
 	}
 }

cli/resolve-packages.ts

@@ -1,9 +1,10 @@
 import process from 'node:process';
 import path from 'node:path';
 import chalk from 'chalk';
-import {checkConfigFile, formatDir, formatGithubDir, getRootDir, parseGithubURL, readConfig} from './mops.js';
+import {checkConfigFile, getRootDir, parseGithubURL, readConfig} from './mops.js';
 import {VesselConfig, readVesselConfig} from './vessel.js';
 import {Config, Dependency} from './types.js';
+import {getDepCacheDir, getDepCacheName} from './cache.js';
 
 export async function resolvePackages({verbose = false} = {}) : Promise<Record<string, string>> {
 	if (!checkConfigFile()) {
@@ -76,25 +77,25 @@ export async function resolvePackages({verbose = false} = {}) : Promise<Record<s
 			}
 
 			let nestedConfig;
-			let nestedDir = '';
+			let localNestedDir = '';
 
 			// read nested config
 			if (repo) {
-				nestedDir = formatGithubDir(name, repo);
-				nestedConfig = await readVesselConfig(nestedDir, {silent: true}) || {};
+				let cacheDir = getDepCacheName(name, repo);
+				nestedConfig = await readVesselConfig(getDepCacheDir(cacheDir), {silent: true}) || {};
 			}
 			else if (pkgDetails.path) {
-				nestedDir = path.resolve(configDir, pkgDetails.path);
-				nestedConfig = readConfig(nestedDir + '/mops.toml');
+				localNestedDir = path.resolve(configDir, pkgDetails.path);
+				nestedConfig = readConfig(localNestedDir + '/mops.toml');
 			}
 			else if (version) {
-				nestedDir = formatDir(name, version);
-				nestedConfig = readConfig(nestedDir + '/mops.toml');
+				let cacheDir = getDepCacheName(name, version);
+				nestedConfig = readConfig(getDepCacheDir(cacheDir) + '/mops.toml');
 			}
 
 			// collect nested deps
 			if (nestedConfig) {
-				await collectDeps(nestedConfig, nestedDir);
+				await collectDeps(nestedConfig, localNestedDir);
 			}
 
 			if (!versions[name]) {