Skip to content

Commit bde2a20

Browse files
ZenVoichZenVoich
authored
Merge pull request #218 from ZenVoich/cli/global-cache-install-resolve
install/resolve via global cache
2 parents 2195738 + 58fe4a7 commit bde2a20

11 files changed

Lines changed: 133 additions & 70 deletions

File tree

‎cli/CHANGELOG.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
- Refactor `mops install` command
66
- Reduce install threads to 12 (was 16)
77
- Reduce install threads to 6 when install called from `mops sources`
8+
- Install dependencies directly to global cache, copy to local cache only final resolved dependencies
89

910
## 0.41.1
1011
- Fix bin path for npm

‎cli/cache.ts‎

Lines changed: 25 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -3,15 +3,33 @@ import path from 'node:path';
33
import ncp from 'ncp';
44
import getFolderSize from 'get-folder-size';
55

6-
import {globalCacheDir} from './mops.js';
6+
import {getDependencyType, globalCacheDir, parseGithubURL} from './mops.js';
77

8-
export let isCached = (pkgId : string) => {
9-
let dir = path.join(globalCacheDir, 'packages', pkgId);
8+
export let getDepCacheDir = (cacheName : string) => {
9+
return path.join(globalCacheDir, 'packages', cacheName);
10+
};
11+
12+
export let isDepCached = (cacheName : string) => {
13+
let dir = getDepCacheDir(cacheName);
1014
return fs.existsSync(dir);
1115
};
1216

13-
export let addCache = (pkgId : string, source : string) => {
14-
let dest = path.join(globalCacheDir, 'packages', pkgId);
17+
export function getDepCacheName(name : string, version : string) {
18+
let depType = getDependencyType(version);
19+
return depType === 'mops' ? getMopsDepCacheName(name, version) : getGithubDepCacheName(name, version);
20+
}
21+
22+
export function getMopsDepCacheName(name : string, version : string) {
23+
return `${name}@${version}`;
24+
}
25+
26+
export function getGithubDepCacheName(name : string, repo : string) {
27+
const {branch, commitHash} = parseGithubURL(repo);
28+
return `_github/${name}#${branch}` + (commitHash ? `@${commitHash}` : '');
29+
}
30+
31+
export let addCache = (cacheName : string, source : string) => {
32+
let dest = path.join(globalCacheDir, 'packages', cacheName);
1533
fs.mkdirSync(dest, {recursive: true});
1634

1735
return new Promise<void>((resolve, reject) => {
@@ -24,8 +42,8 @@ export let addCache = (pkgId : string, source : string) => {
2442
});
2543
};
2644

27-
export let copyCache = (pkgId : string, dest : string) => {
28-
let source = path.join(globalCacheDir, 'packages', pkgId);
45+
export let copyCache = (cacheName : string, dest : string) => {
46+
let source = path.join(globalCacheDir, 'packages', cacheName);
2947
fs.mkdirSync(dest, {recursive: true});
3048

3149
return new Promise<void>((resolve, reject) => {

‎cli/commands/add.ts‎

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -6,9 +6,10 @@ import {checkConfigFile, getGithubCommit, parseGithubURL, readConfig, writeConfi
66
import {getHighestVersion} from '../api/getHighestVersion.js';
77
import {installMopsDep} from './install/install-mops-dep.js';
88
import {installFromGithub} from '../vessel.js';
9-
import {notifyInstalls} from '../notify-installs.js';
109
import {checkIntegrity} from '../integrity.js';
1110
import {checkRequirements} from '../check-requirements.js';
11+
import {syncLocalCache} from './install/sync-local-cache.js';
12+
import {notifyInstalls} from '../notify-installs.js';
1213

1314
type AddOptions = {
1415
verbose ?: boolean;
@@ -87,8 +88,6 @@ export async function add(name : string, {verbose = false, dev = false, lock} :
8788
};
8889
}
8990

90-
let installedPackages = {};
91-
9291
if (pkgDetails.repo) {
9392
await installFromGithub(pkgDetails.name, pkgDetails.repo, {verbose: verbose});
9493
}
@@ -97,7 +96,6 @@ export async function add(name : string, {verbose = false, dev = false, lock} :
9796
if (res === false) {
9897
return;
9998
}
100-
installedPackages = {...installedPackages, ...res};
10199
}
102100

103101
const depsProp = dev ? 'dev-dependencies' : 'dependencies';
@@ -116,8 +114,11 @@ export async function add(name : string, {verbose = false, dev = false, lock} :
116114
if (lock !== 'ignore') {
117115
logUpdate('Checking integrity...');
118116
}
117+
118+
let installedPackages = await syncLocalCache();
119+
119120
await Promise.all([
120-
notifyInstalls(Object.keys(installedPackages)),
121+
notifyInstalls(installedPackages),
121122
checkIntegrity(lock),
122123
]);
123124

‎cli/commands/install/install-all.ts‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2,10 +2,11 @@ import process from 'node:process';
22
import chalk from 'chalk';
33
import {createLogUpdate} from 'log-update';
44
import {checkConfigFile, readConfig} from '../../mops.js';
5-
import {notifyInstalls} from '../../notify-installs.js';
65
import {checkIntegrity} from '../../integrity.js';
76
import {installDeps} from './install-deps.js';
87
import {checkRequirements} from '../../check-requirements.js';
8+
import {syncLocalCache} from './sync-local-cache.js';
9+
import {notifyInstalls} from '../../notify-installs.js';
910

1011
type InstallAllOptions = {
1112
verbose ?: boolean;
@@ -28,16 +29,17 @@ export async function installAll({verbose = false, silent = false, threads, lock
2829
if (!res) {
2930
return;
3031
}
31-
let installedDeps = res;
3232

3333
let logUpdate = createLogUpdate(process.stdout, {showCursor: true});
3434

3535
if (!silent && lock !== 'ignore') {
3636
logUpdate('Checking integrity...');
3737
}
3838

39+
let installedPackages = await syncLocalCache();
40+
3941
await Promise.all([
40-
notifyInstalls(Object.keys(installedDeps)),
42+
notifyInstalls(installedPackages),
4143
checkIntegrity(lock),
4244
]);
4345

‎cli/commands/install/install-dep.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ import {installFromGithub} from '../../vessel.js';
33
import {installMopsDep} from './install-mops-dep.js';
44
import {Dependency} from '../../types.js';
55
import {installLocalDep} from './install-local-dep.js';
6+
import {getRootDir} from '../../mops.js';
67

78
type InstallDepOptions = {
89
verbose ?: boolean;
@@ -19,6 +20,7 @@ export async function installDep(dep : Dependency, {verbose, silent, threads} :
1920
}
2021
else if (dep.path) {
2122
let depPath = dep.path;
23+
parentPkgPath = parentPkgPath || getRootDir();
2224
if (parentPkgPath) {
2325
depPath = path.resolve(parentPkgPath, dep.path);
2426
}

‎cli/commands/install/install-mops-dep.ts‎

Lines changed: 17 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -4,11 +4,12 @@ import path from 'node:path';
44
import {Buffer} from 'node:buffer';
55
import {createLogUpdate} from 'log-update';
66
import chalk from 'chalk';
7+
import {deleteSync} from 'del';
78
import {checkConfigFile, formatDir, progressBar, readConfig} from '../../mops.js';
89
import {getHighestVersion} from '../../api/getHighestVersion.js';
910
import {storageActor} from '../../api/actors.js';
1011
import {parallel} from '../../parallel.js';
11-
import {addCache, copyCache, isCached} from '../../cache.js';
12+
import {getDepCacheDir, getMopsDepCacheName, isDepCached} from '../../cache.js';
1213
import {downloadFile, getPackageFilesInfo} from '../../api/downloadPackageFiles.js';
1314
import {installDeps} from './install-deps.js';
1415

@@ -46,6 +47,8 @@ export async function installMopsDep(pkg : string, version = '', {verbose, silen
4647
}
4748

4849
let dir = formatDir(pkg, version);
50+
let cacheName = getMopsDepCacheName(pkg, version);
51+
let cacheDir = getDepCacheDir(cacheName);
4952
let alreadyInstalled = false;
5053

5154
// already installed
@@ -54,8 +57,7 @@ export async function installMopsDep(pkg : string, version = '', {verbose, silen
5457
alreadyInstalled = true;
5558
}
5659
// copy from cache
57-
else if (isCached(`${pkg}@${version}`)) {
58-
await copyCache(`${pkg}@${version}`, dir);
60+
else if (isDepCached(cacheName)) {
5961
silent || logUpdate(`${dep ? 'Dependency' : 'Installing'} ${pkg}@${version} (global cache)`);
6062
}
6163
// download
@@ -79,20 +81,24 @@ export async function installMopsDep(pkg : string, version = '', {verbose, silen
7981
progress();
8082
});
8183

82-
// write files to disk
83-
for (let [filePath, data] of filesData.entries()) {
84-
fs.mkdirSync(path.join(dir, path.dirname(filePath)), {recursive: true});
85-
fs.writeFileSync(path.join(dir, filePath), Buffer.from(data));
84+
// write files to global cache
85+
try {
86+
for (let [filePath, data] of filesData.entries()) {
87+
fs.mkdirSync(path.join(cacheDir, path.dirname(filePath)), {recursive: true});
88+
fs.writeFileSync(path.join(cacheDir, filePath), Buffer.from(data));
89+
}
90+
}
91+
catch (err) {
92+
console.error(chalk.red('Error: ') + err);
93+
deleteSync([cacheDir], {force: true});
94+
return false;
8695
}
8796
}
8897
catch (err) {
8998
console.error(chalk.red('Error: ') + err);
9099
return false;
91100
}
92101

93-
// add to cache
94-
await addCache(`${pkg}@${version}`, dir);
95-
96102
progress();
97103
}
98104

@@ -104,7 +110,7 @@ export async function installMopsDep(pkg : string, version = '', {verbose, silen
104110
}
105111

106112
// install dependencies
107-
let config = readConfig(path.join(dir, 'mops.toml'));
113+
let config = readConfig(path.join(cacheDir, 'mops.toml'));
108114
let res = await installDeps(Object.values(config.dependencies || {}), {silent, verbose});
109115

110116
if (!res) {

‎cli/commands/install/sync-local-cache.ts‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
import fs from 'node:fs';
2+
import path from 'node:path';
3+
import {copyCache, getDepCacheName} from '../../cache.js';
4+
import {getDependencyType, getRootDir} from '../../mops.js';
5+
import {resolvePackages} from '../../resolve-packages.js';
6+
7+
export async function syncLocalCache({verbose = false} = {}) : Promise<Record<string, string>> {
8+
let resolvedPackages = await resolvePackages();
9+
let rootDir = getRootDir();
10+
11+
verbose && console.log('Syncing local cache...');
12+
13+
let installedDeps : Record<string, string> = {};
14+
15+
await Promise.all(Object.entries(resolvedPackages).map(([name, value]) => {
16+
let depType = getDependencyType(value);
17+
18+
if (depType === 'mops' || depType === 'github') {
19+
let cacheName = getDepCacheName(name, value);
20+
let dest = path.join(rootDir, '.mops', cacheName);
21+
22+
if (!fs.existsSync(dest)) {
23+
if (depType === 'mops') {
24+
installedDeps[name] = value;
25+
}
26+
return copyCache(cacheName, path.join(rootDir, '.mops', cacheName));
27+
}
28+
}
29+
30+
return Promise.resolve();
31+
}));
32+
33+
return installedDeps;
34+
}

‎cli/commands/remove.ts‎

Lines changed: 15 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,12 @@
11
import fs from 'node:fs';
22
import {deleteSync} from 'del';
33
import chalk from 'chalk';
4-
import {formatDir, formatGithubDir, checkConfigFile, readConfig, writeConfig} from '../mops.js';
4+
import {checkConfigFile, getRootDir, readConfig, writeConfig} from '../mops.js';
55
import {Config, Dependency} from '../types.js';
66
import {checkIntegrity} from '../integrity.js';
7+
import {getDepCacheDir, getDepCacheName} from '../cache.js';
8+
import path from 'node:path';
9+
import {syncLocalCache} from './install/sync-local-cache.js';
710

811
type RemoveOptions = {
912
verbose ?: boolean;
@@ -12,7 +15,6 @@ type RemoveOptions = {
1215
lock ?: 'update' | 'ignore';
1316
};
1417

15-
// eslint-disable-next-line @typescript-eslint/no-unused-vars
1618
export async function remove(name : string, {dev = false, verbose = false, dryRun = false, lock} : RemoveOptions = {}) {
1719
if (!checkConfigFile()) {
1820
return;
@@ -31,13 +33,12 @@ export async function remove(name : string, {dev = false, verbose = false, dryRu
3133
}
3234

3335
function getTransitiveDependenciesOf(name : string, version : string | undefined, repo ?: string) {
34-
let pkgDir = '';
35-
if (repo) {
36-
pkgDir = formatGithubDir(name, repo);
37-
}
38-
else if (version) {
39-
pkgDir = formatDir(name, version);
36+
let value = version || repo;
37+
if (!value) {
38+
return [];
4039
}
40+
let cacheName = getDepCacheName(name, value);
41+
let pkgDir = getDepCacheDir(cacheName);
4142
let configFile = pkgDir + '/mops.toml';
4243
if (!fs.existsSync(configFile)) {
4344
verbose && console.log('no config', configFile);
@@ -78,16 +79,11 @@ export async function remove(name : string, {dev = false, verbose = false, dryRu
7879
verbose && console.log(`Ignored transitive dependency ${depId} (other deps depend on it)`);
7980
continue;
8081
}
81-
let pkgDir;
82-
if (dep.repo) {
83-
pkgDir = formatGithubDir(dep.name, dep.repo);
84-
}
85-
else if (dep.version) {
86-
pkgDir = formatDir(dep.name, dep.version);
87-
}
88-
if (pkgDir && fs.existsSync(pkgDir)) {
89-
dryRun || deleteSync([`${pkgDir}`], {force: true});
90-
verbose && console.log(`Removed local cache ${pkgDir}`);
82+
let cacheName = getDepCacheName(dep.name, dep.version || dep.repo || '');
83+
let localCacheDir = path.join(getRootDir(), '.mops', cacheName);
84+
if (localCacheDir && fs.existsSync(localCacheDir)) {
85+
dryRun || deleteSync([localCacheDir], {force: true});
86+
verbose && console.log(`Removed local cache ${localCacheDir}`);
9187
}
9288
}
9389

@@ -100,6 +96,7 @@ export async function remove(name : string, {dev = false, verbose = false, dryRu
10096
}
10197
dryRun || writeConfig(config);
10298

99+
await syncLocalCache();
103100
await checkIntegrity(lock);
104101

105102
console.log(chalk.green('Package removed ') + `${name} = "${version}"`);

‎cli/notify-installs.ts‎

Lines changed: 9 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,16 @@
11
import {getDependencyType} from './mops.js';
22
import {mainActor} from './api/actors.js';
3-
import {resolvePackages} from './resolve-packages.js';
43

5-
export async function notifyInstalls(names : string[]) {
6-
let resolvedPackages = await resolvePackages();
7-
let packages : [string, string][] = names.map(name => [name, resolvedPackages[name] as string]);
4+
export async function notifyInstalls(installedDeps : Record<string, string>) {
5+
let packages = Object.entries(installedDeps).filter(([_, version]) => getDependencyType(version) === 'mops');
86
if (packages.length) {
97
let actor = await mainActor();
10-
await actor.notifyInstalls(packages.filter(([_, version]) => getDependencyType(version) === 'mops'));
8+
9+
try {
10+
await actor.notifyInstalls(packages);
11+
}
12+
catch (err) {
13+
// verbose && console.error('Failed to notify installs:', err);
14+
}
1115
}
1216
}

‎cli/resolve-packages.ts‎

Lines changed: 10 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,10 @@
11
import process from 'node:process';
22
import path from 'node:path';
33
import chalk from 'chalk';
4-
import {checkConfigFile, formatDir, formatGithubDir, getRootDir, parseGithubURL, readConfig} from './mops.js';
4+
import {checkConfigFile, getRootDir, parseGithubURL, readConfig} from './mops.js';
55
import {VesselConfig, readVesselConfig} from './vessel.js';
66
import {Config, Dependency} from './types.js';
7+
import {getDepCacheDir, getDepCacheName} from './cache.js';
78

89
export async function resolvePackages({verbose = false} = {}) : Promise<Record<string, string>> {
910
if (!checkConfigFile()) {
@@ -76,25 +77,25 @@ export async function resolvePackages({verbose = false} = {}) : Promise<Record<s
7677
}
7778

7879
let nestedConfig;
79-
let nestedDir = '';
80+
let localNestedDir = '';
8081

8182
// read nested config
8283
if (repo) {
83-
nestedDir = formatGithubDir(name, repo);
84-
nestedConfig = await readVesselConfig(nestedDir, {silent: true}) || {};
84+
let cacheDir = getDepCacheName(name, repo);
85+
nestedConfig = await readVesselConfig(getDepCacheDir(cacheDir), {silent: true}) || {};
8586
}
8687
else if (pkgDetails.path) {
87-
nestedDir = path.resolve(configDir, pkgDetails.path);
88-
nestedConfig = readConfig(nestedDir + '/mops.toml');
88+
localNestedDir = path.resolve(configDir, pkgDetails.path);
89+
nestedConfig = readConfig(localNestedDir + '/mops.toml');
8990
}
9091
else if (version) {
91-
nestedDir = formatDir(name, version);
92-
nestedConfig = readConfig(nestedDir + '/mops.toml');
92+
let cacheDir = getDepCacheName(name, version);
93+
nestedConfig = readConfig(getDepCacheDir(cacheDir) + '/mops.toml');
9394
}
9495

9596
// collect nested deps
9697
if (nestedConfig) {
97-
await collectDeps(nestedConfig, nestedDir);
98+
await collectDeps(nestedConfig, localNestedDir);
9899
}
99100

100101
if (!versions[name]) {

0 commit comments

Comments
 (0)