Skip to content

Commit b5919d4

Browse files
hadarshjfroghadarshjfrog
authored
Add skipAutoInstall for Pip and Poetry package managers (#383)
1 parent efb8cc0 commit b5919d4

File tree

3 files changed

+192
-97
lines changed

3 files changed

+192
-97
lines changed

Diff for: ‎commands/audit/sca/python/python.go

+67-57
Original file line numberDiff line numberDiff line change
@@ -9,16 +9,16 @@ import (
99
biutils "github.com/jfrog/build-info-go/utils"
1010
"github.com/jfrog/build-info-go/utils/pythonutils"
1111
"github.com/jfrog/gofrog/datastructures"
12-
utils "github.com/jfrog/jfrog-cli-artifactory/artifactory/commands/python"
12+
artifactoryutils "github.com/jfrog/jfrog-cli-artifactory/artifactory/commands/python"
1313
"github.com/jfrog/jfrog-cli-core/v2/utils/config"
1414
"github.com/jfrog/jfrog-cli-core/v2/utils/coreutils"
1515
"github.com/jfrog/jfrog-cli-security/commands/audit/sca"
16-
xrayutils2 "github.com/jfrog/jfrog-cli-security/utils"
16+
"github.com/jfrog/jfrog-cli-security/utils"
1717
"github.com/jfrog/jfrog-cli-security/utils/techutils"
1818
"github.com/jfrog/jfrog-client-go/utils/errorutils"
1919
"github.com/jfrog/jfrog-client-go/utils/io/fileutils"
2020
"github.com/jfrog/jfrog-client-go/utils/log"
21-
xrayUtils "github.com/jfrog/jfrog-client-go/xray/services/utils"
21+
clientutils "github.com/jfrog/jfrog-client-go/xray/services/utils"
2222

2323
"os"
2424
"os/exec"
@@ -34,42 +34,33 @@ const (
3434
CurationPipMinimumVersion = "23.0.0"
3535
)
3636

37-
type AuditPython struct {
38-
Server *config.ServerDetails
39-
Tool pythonutils.PythonTool
40-
RemotePypiRepo string
41-
PipRequirementsFile string
42-
InstallCommandArgs []string
43-
IsCurationCmd bool
44-
}
45-
46-
func BuildDependencyTree(auditPython *AuditPython) (dependencyTree []*xrayUtils.GraphNode, uniqueDeps []string, downloadUrls map[string]string, err error) {
47-
dependenciesGraph, directDependenciesList, pipUrls, errGetTree := getDependencies(auditPython)
37+
func BuildDependencyTree(params utils.AuditParams, technology techutils.Technology) (dependencyTree []*clientutils.GraphNode, uniqueDeps []string, downloadUrls map[string]string, err error) {
38+
dependenciesGraph, directDependenciesList, pipUrls, errGetTree := getDependencies(params, technology)
4839
if errGetTree != nil {
4940
err = errGetTree
5041
return
5142
}
5243
downloadUrls = pipUrls
53-
directDependencies := []*xrayUtils.GraphNode{}
44+
directDependencies := []*clientutils.GraphNode{}
5445
uniqueDepsSet := datastructures.MakeSet[string]()
5546
for _, rootDep := range directDependenciesList {
56-
directDependency := &xrayUtils.GraphNode{
47+
directDependency := &clientutils.GraphNode{
5748
Id: PythonPackageTypeIdentifier + rootDep,
58-
Nodes: []*xrayUtils.GraphNode{},
49+
Nodes: []*clientutils.GraphNode{},
5950
}
6051
populatePythonDependencyTree(directDependency, dependenciesGraph, uniqueDepsSet)
6152
directDependencies = append(directDependencies, directDependency)
6253
}
63-
root := &xrayUtils.GraphNode{
54+
root := &clientutils.GraphNode{
6455
Id: "root",
6556
Nodes: directDependencies,
6657
}
67-
dependencyTree = []*xrayUtils.GraphNode{root}
58+
dependencyTree = []*clientutils.GraphNode{root}
6859
uniqueDeps = uniqueDepsSet.ToSlice()
6960
return
7061
}
7162

72-
func getDependencies(auditPython *AuditPython) (dependenciesGraph map[string][]string, directDependencies []string, pipUrls map[string]string, err error) {
63+
func getDependencies(params utils.AuditParams, technology techutils.Technology) (dependenciesGraph map[string][]string, directDependencies []string, pipUrls map[string]string, err error) {
7364
wd, err := os.Getwd()
7465
if errorutils.CheckError(err) != nil {
7566
return
@@ -100,24 +91,28 @@ func getDependencies(auditPython *AuditPython) (dependenciesGraph map[string][]s
10091
return
10192
}
10293

103-
restoreEnv, err := runPythonInstall(auditPython)
104-
defer func() {
105-
err = errors.Join(err, restoreEnv())
106-
}()
107-
if err != nil {
108-
return
94+
pythonTool := pythonutils.PythonTool(technology)
95+
if technology == techutils.Pipenv || !params.SkipAutoInstall() {
96+
var restoreEnv func() error
97+
restoreEnv, err = runPythonInstall(params, pythonTool)
98+
defer func() {
99+
err = errors.Join(err, restoreEnv())
100+
}()
101+
if err != nil {
102+
return
103+
}
109104
}
110105

111106
localDependenciesPath, err := config.GetJfrogDependenciesPath()
112107
if err != nil {
113108
return
114109
}
115-
dependenciesGraph, directDependencies, err = pythonutils.GetPythonDependencies(auditPython.Tool, tempDirPath, localDependenciesPath, log.GetLogger())
110+
dependenciesGraph, directDependencies, err = pythonutils.GetPythonDependencies(pythonTool, tempDirPath, localDependenciesPath, log.GetLogger())
116111
if err != nil {
117112
sca.LogExecutableVersion("python")
118-
sca.LogExecutableVersion(string(auditPython.Tool))
113+
sca.LogExecutableVersion(string(pythonTool))
119114
}
120-
if !auditPython.IsCurationCmd {
115+
if !params.IsCurationCmd() {
121116
return
122117
}
123118
pipUrls, errProcessed := processPipDownloadsUrlsFromReportFile()
@@ -181,29 +176,34 @@ type pypiMetaData struct {
181176
Version string `json:"version"`
182177
}
183178

184-
func runPythonInstall(auditPython *AuditPython) (restoreEnv func() error, err error) {
185-
switch auditPython.Tool {
179+
func runPythonInstall(params utils.AuditParams, tool pythonutils.PythonTool) (restoreEnv func() error, err error) {
180+
switch tool {
186181
case pythonutils.Pip:
187-
return installPipDeps(auditPython)
182+
return installPipDeps(params)
188183
case pythonutils.Pipenv:
189-
return installPipenvDeps(auditPython)
184+
return installPipenvDeps(params)
190185
case pythonutils.Poetry:
191-
return installPoetryDeps(auditPython)
186+
return installPoetryDeps(params)
192187
}
193188
return
194189
}
195190

196-
func installPoetryDeps(auditPython *AuditPython) (restoreEnv func() error, err error) {
191+
func installPoetryDeps(params utils.AuditParams) (restoreEnv func() error, err error) {
197192
restoreEnv = func() error {
198193
return nil
199194
}
200-
if auditPython.RemotePypiRepo != "" {
201-
rtUrl, username, password, err := utils.GetPypiRepoUrlWithCredentials(auditPython.Server, auditPython.RemotePypiRepo, false)
195+
if params.DepsRepo() != "" {
196+
var serverDetails *config.ServerDetails
197+
serverDetails, err = params.ServerDetails()
198+
if err != nil {
199+
return restoreEnv, err
200+
}
201+
rtUrl, username, password, err := artifactoryutils.GetPypiRepoUrlWithCredentials(serverDetails, params.DepsRepo(), false)
202202
if err != nil {
203203
return restoreEnv, err
204204
}
205205
if password != "" {
206-
err = utils.ConfigPoetryRepo(rtUrl.Scheme+"://"+rtUrl.Host+rtUrl.Path, username, password, auditPython.RemotePypiRepo)
206+
err = artifactoryutils.ConfigPoetryRepo(rtUrl.Scheme+"://"+rtUrl.Host+rtUrl.Path, username, password, params.DepsRepo())
207207
if err != nil {
208208
return restoreEnv, err
209209
}
@@ -214,7 +214,7 @@ func installPoetryDeps(auditPython *AuditPython) (restoreEnv func() error, err e
214214
return restoreEnv, err
215215
}
216216

217-
func installPipenvDeps(auditPython *AuditPython) (restoreEnv func() error, err error) {
217+
func installPipenvDeps(params utils.AuditParams) (restoreEnv func() error, err error) {
218218
// Set virtualenv path to venv dir
219219
err = os.Setenv("WORKON_HOME", ".jfrog")
220220
if err != nil {
@@ -223,46 +223,56 @@ func installPipenvDeps(auditPython *AuditPython) (restoreEnv func() error, err e
223223
restoreEnv = func() error {
224224
return os.Unsetenv("WORKON_HOME")
225225
}
226-
if auditPython.RemotePypiRepo != "" {
227-
return restoreEnv, runPipenvInstallFromRemoteRegistry(auditPython.Server, auditPython.RemotePypiRepo)
226+
if params.DepsRepo() != "" {
227+
var serverDetails *config.ServerDetails
228+
serverDetails, err = params.ServerDetails()
229+
if err != nil {
230+
return
231+
}
232+
return restoreEnv, runPipenvInstallFromRemoteRegistry(serverDetails, params.DepsRepo())
228233
}
229234
// Run 'pipenv install -d'
230235
_, err = executeCommand("pipenv", "install", "-d")
231236
return restoreEnv, err
232237
}
233238

234-
func installPipDeps(auditPython *AuditPython) (restoreEnv func() error, err error) {
239+
func installPipDeps(params utils.AuditParams) (restoreEnv func() error, err error) {
235240
restoreEnv, err = SetPipVirtualEnvPath()
236241
if err != nil {
237242
return
238243
}
239244

240245
remoteUrl := ""
241-
if auditPython.RemotePypiRepo != "" {
242-
remoteUrl, err = utils.GetPypiRepoUrl(auditPython.Server, auditPython.RemotePypiRepo, auditPython.IsCurationCmd)
246+
if params.DepsRepo() != "" {
247+
var serverDetails *config.ServerDetails
248+
serverDetails, err = params.ServerDetails()
249+
if err != nil {
250+
return
251+
}
252+
remoteUrl, err = artifactoryutils.GetPypiRepoUrl(serverDetails, params.DepsRepo(), params.IsCurationCmd())
243253
if err != nil {
244254
return
245255
}
246256
}
247257

248258
var curationCachePip string
249259
var reportFileName string
250-
if auditPython.IsCurationCmd {
260+
if params.IsCurationCmd() {
251261
// upgrade pip version to 23.0.0, as it is required for the curation command.
252262
if err = upgradePipVersion(CurationPipMinimumVersion); err != nil {
253263
log.Warn(fmt.Sprintf("Failed to upgrade pip version, err: %v", err))
254264
}
255-
if curationCachePip, err = xrayutils2.GetCurationPipCacheFolder(); err != nil {
265+
if curationCachePip, err = utils.GetCurationPipCacheFolder(); err != nil {
256266
return
257267
}
258268
reportFileName = pythonReportFile
259269
}
260270

261-
pipInstallArgs := getPipInstallArgs(auditPython.PipRequirementsFile, remoteUrl, curationCachePip, reportFileName, auditPython.InstallCommandArgs...)
271+
pipInstallArgs := getPipInstallArgs(params.PipRequirementsFile(), remoteUrl, curationCachePip, reportFileName, params.InstallCommandArgs()...)
262272
var reqErr error
263273
_, err = executeCommand("python", pipInstallArgs...)
264-
if err != nil && auditPython.PipRequirementsFile == "" {
265-
pipInstallArgs = getPipInstallArgs("requirements.txt", remoteUrl, curationCachePip, reportFileName, auditPython.InstallCommandArgs...)
274+
if err != nil && params.PipRequirementsFile() == "" {
275+
pipInstallArgs = getPipInstallArgs("requirements.txt", remoteUrl, curationCachePip, reportFileName, params.InstallCommandArgs()...)
266276
_, reqErr = executeCommand("python", pipInstallArgs...)
267277
if reqErr != nil {
268278
// Return Pip install error and log the requirements fallback error.
@@ -272,7 +282,7 @@ func installPipDeps(auditPython *AuditPython) (restoreEnv func() error, err erro
272282
}
273283
}
274284
if err != nil || reqErr != nil {
275-
if msgToUser := sca.GetMsgToUserForCurationBlock(auditPython.IsCurationCmd, techutils.Pip, errors.Join(err, reqErr).Error()); msgToUser != "" {
285+
if msgToUser := sca.GetMsgToUserForCurationBlock(params.IsCurationCmd(), techutils.Pip, errors.Join(err, reqErr).Error()); msgToUser != "" {
276286
err = errors.Join(err, errors.New(msgToUser))
277287
}
278288
}
@@ -318,7 +328,7 @@ func getPipInstallArgs(requirementsFile, remoteUrl, cacheFolder, reportFileName
318328
args = append(args, "-r", requirementsFile)
319329
}
320330
if remoteUrl != "" {
321-
args = append(args, utils.GetPypiRemoteRegistryFlag(pythonutils.Pip), remoteUrl)
331+
args = append(args, artifactoryutils.GetPypiRemoteRegistryFlag(pythonutils.Pip), remoteUrl)
322332
}
323333
if cacheFolder != "" {
324334
args = append(args, "--cache-dir", cacheFolder)
@@ -359,7 +369,7 @@ func parseCustomArgs(remoteUrl, cacheFolder, reportFileName string, customArgs .
359369
continue
360370
}
361371
}
362-
if remoteUrl != "" && strings.Contains(customArgs[i], utils.GetPypiRemoteRegistryFlag(pythonutils.Pip)) {
372+
if remoteUrl != "" && strings.Contains(customArgs[i], artifactoryutils.GetPypiRemoteRegistryFlag(pythonutils.Pip)) {
363373
log.Warn("The remote registry flag is not supported in the custom arguments list. skipping...")
364374
i++
365375
continue
@@ -370,11 +380,11 @@ func parseCustomArgs(remoteUrl, cacheFolder, reportFileName string, customArgs .
370380
}
371381

372382
func runPipenvInstallFromRemoteRegistry(server *config.ServerDetails, depsRepoName string) (err error) {
373-
rtUrl, err := utils.GetPypiRepoUrl(server, depsRepoName, false)
383+
rtUrl, err := artifactoryutils.GetPypiRepoUrl(server, depsRepoName, false)
374384
if err != nil {
375385
return err
376386
}
377-
args := []string{"install", "-d", utils.GetPypiRemoteRegistryFlag(pythonutils.Pipenv), rtUrl}
387+
args := []string{"install", "-d", artifactoryutils.GetPypiRemoteRegistryFlag(pythonutils.Pipenv), rtUrl}
378388
_, err = executeCommand("pipenv", args...)
379389
return err
380390
}
@@ -424,17 +434,17 @@ func SetPipVirtualEnvPath() (restoreEnv func() error, err error) {
424434
return
425435
}
426436

427-
func populatePythonDependencyTree(currNode *xrayUtils.GraphNode, dependenciesGraph map[string][]string, uniqueDepsSet *datastructures.Set[string]) {
437+
func populatePythonDependencyTree(currNode *clientutils.GraphNode, dependenciesGraph map[string][]string, uniqueDepsSet *datastructures.Set[string]) {
428438
if currNode.NodeHasLoop() {
429439
return
430440
}
431441
uniqueDepsSet.Add(currNode.Id)
432442
currDepChildren := dependenciesGraph[strings.TrimPrefix(currNode.Id, PythonPackageTypeIdentifier)]
433443
// Recursively create & append all node's dependencies.
434444
for _, dependency := range currDepChildren {
435-
childNode := &xrayUtils.GraphNode{
445+
childNode := &clientutils.GraphNode{
436446
Id: PythonPackageTypeIdentifier + dependency,
437-
Nodes: []*xrayUtils.GraphNode{},
447+
Nodes: []*clientutils.GraphNode{},
438448
Parent: currNode,
439449
}
440450
currNode.Nodes = append(currNode.Nodes, childNode)

0 commit comments

Comments
 (0)