Enable using local repository instead cloning a remote repo - ScanRepository (#767)

Author: eranturgeman

azure_test.go

@@ -20,25 +20,30 @@ func buildAzureReposClient(t *testing.T, azureToken string) vcsclient.VcsClient
 	return azureClient
 }
 
-func buildAzureReposIntegrationTestDetails(t *testing.T) *IntegrationTestDetails {
+func buildAzureReposIntegrationTestDetails(t *testing.T, useLocalRepo bool) *IntegrationTestDetails {
 	integrationRepoToken := getIntegrationToken(t, azureIntegrationTokenEnv)
-	testDetails := NewIntegrationTestDetails(integrationRepoToken, string(utils.AzureRepos), azureGitCloneUrl, "frogbot-test")
+	testDetails := NewIntegrationTestDetails(integrationRepoToken, string(utils.AzureRepos), azureGitCloneUrl, "frogbot-test", useLocalRepo)
 	testDetails.ApiEndpoint = azureApiEndpoint
 	return testDetails
 }
 
-func azureReposTestsInit(t *testing.T) (vcsclient.VcsClient, *IntegrationTestDetails) {
-	testDetails := buildAzureReposIntegrationTestDetails(t)
+func azureReposTestsInit(t *testing.T, useLocalRepo bool) (vcsclient.VcsClient, *IntegrationTestDetails) {
+	testDetails := buildAzureReposIntegrationTestDetails(t, useLocalRepo)
 	azureClient := buildAzureReposClient(t, testDetails.GitToken)
 	return azureClient, testDetails
 }
 
 func TestAzureRepos_ScanPullRequestIntegration(t *testing.T) {
-	azureClient, testDetails := azureReposTestsInit(t)
+	azureClient, testDetails := azureReposTestsInit(t, false)
 	runScanPullRequestCmd(t, azureClient, testDetails)
 }
 
 func TestAzureRepos_ScanRepositoryIntegration(t *testing.T) {
-	azureClient, testDetails := azureReposTestsInit(t)
+	azureClient, testDetails := azureReposTestsInit(t, false)
+	runScanRepositoryCmd(t, azureClient, testDetails)
+}
+
+func TestAzureRepos_ScanRepositoryWithLocalDirIntegration(t *testing.T) {
+	azureClient, testDetails := azureReposTestsInit(t, true)
 	runScanRepositoryCmd(t, azureClient, testDetails)
 }

bitbucket_server_test.go

@@ -26,9 +26,9 @@ func buildBitbucketServerClient(t *testing.T, bitbucketServerToken string) vcscl
 	return bbClient
 }
 
-func buildBitbucketServerIntegrationTestDetails(t *testing.T) *IntegrationTestDetails {
+func buildBitbucketServerIntegrationTestDetails(t *testing.T, useLocalRepo bool) *IntegrationTestDetails {
 	integrationRepoToken := getIntegrationToken(t, bitbucketServerIntegrationTokenEnv)
-	testDetails := NewIntegrationTestDetails(integrationRepoToken, string(utils.BitbucketServer), bitbucketServerGitCloneUrl, "FROG")
+	testDetails := NewIntegrationTestDetails(integrationRepoToken, string(utils.BitbucketServer), bitbucketServerGitCloneUrl, "FROG", useLocalRepo)
 	testDetails.ApiEndpoint = bitbucketServerApiEndpoint
 	return testDetails
 }
@@ -53,19 +53,24 @@ func waitForConnection(t *testing.T) {
 	require.NoError(t, retryExecutor.Execute())
 }
 
-func bitbucketServerTestsInit(t *testing.T) (vcsclient.VcsClient, *IntegrationTestDetails) {
-	testDetails := buildBitbucketServerIntegrationTestDetails(t)
+func bitbucketServerTestsInit(t *testing.T, useLocalRepo bool) (vcsclient.VcsClient, *IntegrationTestDetails) {
+	testDetails := buildBitbucketServerIntegrationTestDetails(t, useLocalRepo)
 	bbClient := buildBitbucketServerClient(t, testDetails.GitToken)
 	waitForConnection(t)
 	return bbClient, testDetails
 }
 
 func TestBitbucketServer_ScanPullRequestIntegration(t *testing.T) {
-	bbClient, testDetails := bitbucketServerTestsInit(t)
+	bbClient, testDetails := bitbucketServerTestsInit(t, false)
 	runScanPullRequestCmd(t, bbClient, testDetails)
 }
 
 func TestBitbucketServer_ScanRepositoryIntegration(t *testing.T) {
-	bbClient, testDetails := bitbucketServerTestsInit(t)
+	bbClient, testDetails := bitbucketServerTestsInit(t, false)
+	runScanRepositoryCmd(t, bbClient, testDetails)
+}
+
+func TestBitbucketServer_ScanRepositoryWithLocalDirIntegration(t *testing.T) {
+	bbClient, testDetails := bitbucketServerTestsInit(t, true)
 	runScanRepositoryCmd(t, bbClient, testDetails)
 }

github_test.go

@@ -19,23 +19,28 @@ func buildGitHubClient(t *testing.T, githubToken string) vcsclient.VcsClient {
 	return githubClient
 }
 
-func buildGitHubIntegrationTestDetails(t *testing.T) *IntegrationTestDetails {
+func buildGitHubIntegrationTestDetails(t *testing.T, useLocalRepo bool) *IntegrationTestDetails {
 	integrationRepoToken := getIntegrationToken(t, githubIntegrationTokenEnv)
-	return NewIntegrationTestDetails(integrationRepoToken, string(utils.GitHub), githubGitCloneUrl, "frogbot-test")
+	return NewIntegrationTestDetails(integrationRepoToken, string(utils.GitHub), githubGitCloneUrl, "frogbot-test", useLocalRepo)
 }
 
-func githubTestsInit(t *testing.T) (vcsclient.VcsClient, *IntegrationTestDetails) {
-	testDetails := buildGitHubIntegrationTestDetails(t)
+func githubTestsInit(t *testing.T, useLocalRepo bool) (vcsclient.VcsClient, *IntegrationTestDetails) {
+	testDetails := buildGitHubIntegrationTestDetails(t, useLocalRepo)
 	githubClient := buildGitHubClient(t, testDetails.GitToken)
 	return githubClient, testDetails
 }
 
 func TestGitHub_ScanPullRequestIntegration(t *testing.T) {
-	githubClient, testDetails := githubTestsInit(t)
+	githubClient, testDetails := githubTestsInit(t, false)
 	runScanPullRequestCmd(t, githubClient, testDetails)
 }
 
 func TestGitHub_ScanRepositoryIntegration(t *testing.T) {
-	githubClient, testDetails := githubTestsInit(t)
+	githubClient, testDetails := githubTestsInit(t, false)
+	runScanRepositoryCmd(t, githubClient, testDetails)
+}
+
+func TestGitHub_ScanRepositoryWithLocalDirIntegration(t *testing.T) {
+	githubClient, testDetails := githubTestsInit(t, true)
 	runScanRepositoryCmd(t, githubClient, testDetails)
 }

gitlab_test.go

@@ -19,23 +19,28 @@ func buildGitLabClient(t *testing.T, gitlabToken string) vcsclient.VcsClient {
 	return azureClient
 }
 
-func buildGitLabIntegrationTestDetails(t *testing.T) *IntegrationTestDetails {
+func buildGitLabIntegrationTestDetails(t *testing.T, useLocalRepo bool) *IntegrationTestDetails {
 	integrationRepoToken := getIntegrationToken(t, gitlabIntegrationTokenEnv)
-	return NewIntegrationTestDetails(integrationRepoToken, string(utils.GitLab), gitlabGitCloneUrl, "frogbot-test2")
+	return NewIntegrationTestDetails(integrationRepoToken, string(utils.GitLab), gitlabGitCloneUrl, "frogbot-test2", useLocalRepo)
 }
 
-func gitlabTestsInit(t *testing.T) (vcsclient.VcsClient, *IntegrationTestDetails) {
-	testDetails := buildGitLabIntegrationTestDetails(t)
+func gitlabTestsInit(t *testing.T, useLocalRepo bool) (vcsclient.VcsClient, *IntegrationTestDetails) {
+	testDetails := buildGitLabIntegrationTestDetails(t, useLocalRepo)
 	gitlabClient := buildGitLabClient(t, testDetails.GitToken)
 	return gitlabClient, testDetails
 }
 
 func TestGitLab_ScanPullRequestIntegration(t *testing.T) {
-	gitlabClient, testDetails := gitlabTestsInit(t)
+	gitlabClient, testDetails := gitlabTestsInit(t, false)
 	runScanPullRequestCmd(t, gitlabClient, testDetails)
 }
 
 func TestGitLab_ScanRepositoryIntegration(t *testing.T) {
-	gitlabClient, testDetails := gitlabTestsInit(t)
+	gitlabClient, testDetails := gitlabTestsInit(t, false)
+	runScanRepositoryCmd(t, gitlabClient, testDetails)
+}
+
+func TestGitLab_ScanRepositoryWithLocalDirIntegration(t *testing.T) {
+	gitlabClient, testDetails := gitlabTestsInit(t, true)
 	runScanRepositoryCmd(t, gitlabClient, testDetails)
 }

integrationutils.go

@@ -3,6 +3,8 @@ package main
 import (
 	"context"
 	"fmt"
+	"github.com/go-git/go-git/v5"
+	githttp "github.com/go-git/go-git/v5/plumbing/transport/http"
 	"github.com/jfrog/frogbot/v2/scanpullrequest"
 	"github.com/jfrog/frogbot/v2/scanrepository"
 	"github.com/jfrog/frogbot/v2/utils"
@@ -36,17 +38,19 @@ type IntegrationTestDetails struct {
 	ApiEndpoint      string
 	PullRequestID    string
 	CustomBranchName string
+	UseLocalRepo     bool
 }
 
-func NewIntegrationTestDetails(token, gitProvider, gitCloneUrl, repoOwner string) *IntegrationTestDetails {
+func NewIntegrationTestDetails(token, gitProvider, gitCloneUrl, repoOwner string, useLocalRepo bool) *IntegrationTestDetails {
 	return &IntegrationTestDetails{
-		GitProject:  repoName,
-		RepoOwner:   repoOwner,
-		RepoName:    repoName,
-		GitToken:    token,
-		GitUsername: "frogbot",
-		GitProvider: gitProvider,
-		GitCloneURL: gitCloneUrl,
+		GitProject:   repoName,
+		RepoOwner:    repoOwner,
+		RepoName:     repoName,
+		GitToken:     token,
+		GitUsername:  "frogbot",
+		GitProvider:  gitProvider,
+		GitCloneURL:  gitCloneUrl,
+		UseLocalRepo: useLocalRepo,
 	}
 }
 
@@ -71,18 +75,23 @@ func setIntegrationTestEnvs(t *testing.T, testDetails *IntegrationTestDetails) f
 	// Frogbot sanitizes all the environment variables that start with 'JF',
 	// so we restore them at the end of the test to avoid collisions with other tests
 	envRestoreFunc := getJfrogEnvRestoreFunc(t)
+	useLocalRepo := "false"
+	if testDetails.UseLocalRepo {
+		useLocalRepo = "true"
+	}
 	unsetEnvs := utils.SetEnvsAndAssertWithCallback(t, map[string]string{
-		utils.RequirementsFileEnv:   "requirements.txt",
-		utils.GitPullRequestIDEnv:   testDetails.PullRequestID,
-		utils.GitProvider:           testDetails.GitProvider,
-		utils.GitTokenEnv:           testDetails.GitToken,
-		utils.GitRepoEnv:            testDetails.RepoName,
-		utils.GitRepoOwnerEnv:       testDetails.RepoOwner,
-		utils.BranchNameTemplateEnv: testDetails.CustomBranchName,
-		utils.GitApiEndpointEnv:     testDetails.ApiEndpoint,
-		utils.GitProjectEnv:         testDetails.GitProject,
-		utils.GitUsernameEnv:        testDetails.GitUsername,
-		utils.GitBaseBranchEnv:      mainBranch,
+		utils.RequirementsFileEnv:      "requirements.txt",
+		utils.GitPullRequestIDEnv:      testDetails.PullRequestID,
+		utils.GitProvider:              testDetails.GitProvider,
+		utils.GitTokenEnv:              testDetails.GitToken,
+		utils.GitRepoEnv:               testDetails.RepoName,
+		utils.GitRepoOwnerEnv:          testDetails.RepoOwner,
+		utils.BranchNameTemplateEnv:    testDetails.CustomBranchName,
+		utils.GitApiEndpointEnv:        testDetails.ApiEndpoint,
+		utils.GitProjectEnv:            testDetails.GitProject,
+		utils.GitUsernameEnv:           testDetails.GitUsername,
+		utils.GitBaseBranchEnv:         mainBranch,
+		utils.GitUseLocalRepositoryEnv: useLocalRepo,
 	})
 	return func() {
 		envRestoreFunc()
@@ -173,11 +182,28 @@ func runScanPullRequestCmd(t *testing.T, client vcsclient.VcsClient, testDetails
 }
 
 func runScanRepositoryCmd(t *testing.T, client vcsclient.VcsClient, testDetails *IntegrationTestDetails) {
-	_, restoreFunc := utils.ChangeToTempDirWithCallback(t)
+	testTempDir, restoreFunc := utils.ChangeToTempDirWithCallback(t)
 	defer func() {
 		assert.NoError(t, restoreFunc())
 	}()
 
+	// When testing using local repository clone the repository before the test starts so we can work with it as if it existed locally
+	if testDetails.UseLocalRepo {
+		cloneOptions := &git.CloneOptions{
+			URL: testDetails.GitCloneURL,
+			Auth: &githttp.BasicAuth{
+				Username: testDetails.GitUsername,
+				Password: testDetails.GitToken,
+			},
+			RemoteName:    "origin",
+			ReferenceName: utils.GetFullBranchName("main"),
+			SingleBranch:  true,
+			Depth:         1,
+			Tags:          git.NoTags,
+		}
+		_, err := git.PlainClone(testTempDir, false, cloneOptions)
+		require.NoError(t, err)
+	}
 	timestamp := getTimestamp()
 	// Add a timestamp to the fixing pull requests, to identify them later
 	testDetails.CustomBranchName = "frogbot-{IMPACTED_PACKAGE}-{BRANCH_NAME_HASH}-" + timestamp

scanrepository/scanrepository.go

@@ -4,6 +4,8 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	biutils "github.com/jfrog/build-info-go/utils"
+	"os"
 	"path/filepath"
 	"regexp"
 	"strings"
@@ -80,17 +82,17 @@ func (cfp *ScanRepositoryCmd) scanAndFixBranch(repository *utils.Repository) (er
 		cfp.analyticsService.UpdateAndSendXscAnalyticsGeneralEventFinalize(err)
 	}()
 
-	clonedRepoDir, restoreBaseDir, err := cfp.cloneRepositoryAndCheckoutToBranch()
+	repoDir, restoreBaseDir, err := cfp.cloneRepositoryOrUseLocalAndCheckoutToBranch()
 	if err != nil {
 		return
 	}
-	cfp.baseWd = clonedRepoDir
+	cfp.baseWd = repoDir
 	defer func() {
 		// On dry run don't delete the folder as we want to validate results
 		if cfp.dryRun {
 			return
 		}
-		err = errors.Join(err, restoreBaseDir(), fileutils.RemoveTempDir(clonedRepoDir))
+		err = errors.Join(err, restoreBaseDir(), fileutils.RemoveTempDir(repoDir))
 	}()
 
 	// If MSI exists we always need to report events
@@ -476,7 +478,7 @@ func (cfp *ScanRepositoryCmd) preparePullRequestDetails(vulnerabilitiesDetails .
 	return pullRequestTitle, prBody, extraComments, nil
 }
 
-func (cfp *ScanRepositoryCmd) cloneRepositoryAndCheckoutToBranch() (tempWd string, restoreDir func() error, err error) {
+func (cfp *ScanRepositoryCmd) cloneRepositoryOrUseLocalAndCheckoutToBranch() (tempWd string, restoreDir func() error, err error) {
 	if cfp.dryRun {
 		tempWd = filepath.Join(cfp.dryRunRepoPath, cfp.scanDetails.RepoName)
 	} else {
@@ -487,13 +489,29 @@ func (cfp *ScanRepositoryCmd) cloneRepositoryAndCheckoutToBranch() (tempWd strin
 	}
 	log.Debug("Created temp working directory:", tempWd)
 
-	// Clone the content of the repo to the new working directory
-	if err = cfp.gitManager.Clone(tempWd, cfp.scanDetails.BaseBranch()); err != nil {
-		return
+	if cfp.scanDetails.UseLocalRepository {
+		var curDir string
+		if curDir, err = os.Getwd(); err != nil {
+			return
+		}
+		if err = biutils.CopyDir(curDir, tempWd, true, nil); err != nil {
+			return
+		}
+		// 'CD' into the temp working directory
+		restoreDir, err = utils.Chdir(tempWd)
+		if err != nil {
+			return
+		}
+		// Set the current copied local dir as the local git repository we are working with
+		err = cfp.gitManager.SetLocalRepository()
+	} else {
+		// Clone the content of the repo to the new working directory
+		if err = cfp.gitManager.Clone(tempWd, cfp.scanDetails.BaseBranch()); err != nil {
+			return
+		}
+		// 'CD' into the temp working directory
+		restoreDir, err = utils.Chdir(tempWd)
 	}
-
-	// 'CD' into the temp working directory
-	restoreDir, err = utils.Chdir(tempWd)
 	return
 }
 

utils/consts.go

@@ -34,11 +34,12 @@ const (
 	JfrogConfigProfileEnv  = "JF_CONFIG_PROFILE"
 
 	// Git environment variables
-	GitProvider     = "JF_GIT_PROVIDER"
-	GitRepoOwnerEnv = "JF_GIT_OWNER"
-	GitRepoEnv      = "JF_GIT_REPO"
-	GitProjectEnv   = "JF_GIT_PROJECT"
-	GitUsernameEnv  = "JF_GIT_USERNAME"
+	GitProvider              = "JF_GIT_PROVIDER"
+	GitRepoOwnerEnv          = "JF_GIT_OWNER"
+	GitRepoEnv               = "JF_GIT_REPO"
+	GitProjectEnv            = "JF_GIT_PROJECT"
+	GitUsernameEnv           = "JF_GIT_USERNAME"
+	GitUseLocalRepositoryEnv = "JF_USE_LOCAL_REPOSITORY"
 
 	// Git naming template environment variables
 	BranchNameTemplateEnv       = "JF_BRANCH_NAME_TEMPLATE"

utils/git.go

@@ -92,7 +92,7 @@ func (gm *GitManager) SetRemoteGitUrl(remoteHttpsGitUrl string) (*GitManager, er
 	}
 
 	if gm.localGitRepository == nil {
-		if _, err = gm.SetLocalRepository(); err != nil {
+		if _, err = gm.SetLocalRepositoryAndRemoteName(); err != nil {
 			return gm, err
 		}
 	}
@@ -115,14 +115,20 @@ func (gm *GitManager) SetRemoteGitUrl(remoteHttpsGitUrl string) (*GitManager, er
 	return gm, nil
 }
 
-func (gm *GitManager) SetLocalRepository() (*GitManager, error) {
+func (gm *GitManager) SetLocalRepositoryAndRemoteName() (*GitManager, error) {
 	var err error
 	// Re-initialize the repository and update remoteName
 	gm.remoteName = vcsutils.RemoteName
-	gm.localGitRepository, err = git.PlainOpen(".")
+	err = gm.SetLocalRepository()
 	return gm, err
 }
 
+func (gm *GitManager) SetLocalRepository() error {
+	var err error
+	gm.localGitRepository, err = git.PlainOpen(".")
+	return err
+}
+
 func (gm *GitManager) SetGitParams(gitParams *Git) (*GitManager, error) {
 	var err error
 	if gm.customTemplates, err = loadCustomTemplates(gitParams.CommitMessageTemplate, gitParams.BranchNameTemplate, gitParams.PullRequestTitleTemplate); err != nil {

utils/params.go

@@ -302,6 +302,7 @@ type Git struct {
 	AggregateFixes           bool     `yaml:"aggregateFixes,omitempty"`
 	PullRequestDetails       vcsclient.PullRequestInfo
 	RepositoryCloneUrl       string
+	UseLocalRepository       bool
 }
 
 func (g *Git) setDefaultsIfNeeded(gitParamsFromEnv *Git, commandName string) (err error) {
@@ -373,6 +374,11 @@ func (g *Git) extractScanRepositoryEnvParams(gitParamsFromEnv *Git) (err error)
 			return
 		}
 	}
+	if !g.UseLocalRepository {
+		if g.UseLocalRepository, err = getBoolEnv(GitUseLocalRepositoryEnv, false); err != nil {
+			return
+		}
+	}
 	return
 }