From 2d7b2db0389c0404a770b70cdb8ab46545aa4d33 Mon Sep 17 00:00:00 2001 From: Phillip Nguyen Date: Fri, 13 Dec 2024 20:49:17 +0000 Subject: [PATCH] Reuse the same HTTP client for all requests. * create a client.Downloader wrapper around http.Client * don't pass around the proxyServer URL --- client/client.go | 182 ++++++++++++++++++++++-------------------- client/client_test.go | 18 ++++- download/download.go | 35 ++++---- googet_available.go | 7 +- googet_download.go | 11 ++- googet_install.go | 15 ++-- googet_latest.go | 7 +- googet_remove.go | 8 +- googet_update.go | 9 ++- googet_verify.go | 10 ++- install/install.go | 20 ++--- remove/remove.go | 12 +-- remove/remove_test.go | 7 +- verify/verify.go | 19 +---- 14 files changed, 201 insertions(+), 159 deletions(-) diff --git a/client/client.go b/client/client.go index 039eeae..c483786 100644 --- a/client/client.go +++ b/client/client.go @@ -102,11 +102,43 @@ type Repo struct { // RepoMap describes each repo's packages as seen from a client. type RepoMap map[string]Repo +// Downloader is a wrapper around http.Client +type Downloader struct { + HTTPClient *http.Client + UsingProxyServer bool +} + +// NewDownloader returns a Downloader optionally using a specified proxyServer. +func NewDownloader(proxyServer string) (*Downloader, error) { + httpClient := http.DefaultClient + proxy := http.ProxyFromEnvironment + if proxyServer != "" { + proxyURL, err := url.Parse(proxyServer) + if err != nil { + return nil, err + } + proxy = http.ProxyURL(proxyURL) + } + httpClient.Transport = &http.Transport{ + Proxy: proxy, + DialContext: (&net.Dialer{ + Timeout: 30 * time.Second, + KeepAlive: 30 * time.Second, + }).DialContext, + ForceAttemptHTTP2: true, + MaxIdleConns: 100, + IdleConnTimeout: 60 * time.Second, + TLSHandshakeTimeout: 10 * time.Second, + ExpectContinueTimeout: 1 * time.Second, + } + return &Downloader{HTTPClient: httpClient, UsingProxyServer: proxyServer != ""}, nil +} + // AvailableVersions builds a RepoMap from a list of sources. -func AvailableVersions(ctx context.Context, srcs map[string]priority.Value, cacheDir string, cacheLife time.Duration, proxyServer string) RepoMap { +func (d *Downloader) AvailableVersions(ctx context.Context, srcs map[string]priority.Value, cacheDir string, cacheLife time.Duration) RepoMap { rm := make(RepoMap) for r, pri := range srcs { - rf, err := unmarshalRepoPackages(ctx, r, cacheDir, cacheLife, proxyServer) + rf, err := d.unmarshalRepoPackages(ctx, r, cacheDir, cacheLife) if err != nil { logger.Errorf("error reading repo %q: %v", r, err) continue @@ -119,56 +151,10 @@ func AvailableVersions(ctx context.Context, srcs map[string]priority.Value, cach return rm } -func decode(index io.ReadCloser, ct, url, cf string) ([]goolib.RepoSpec, error) { - defer index.Close() - - var dec *json.Decoder - switch ct { - case "application/x-gzip": - gr, err := gzip.NewReader(index) - if err != nil { - return nil, err - } - dec = json.NewDecoder(gr) - case "application/json": - dec = json.NewDecoder(index) - default: - return nil, fmt.Errorf("unsupported content type: %s", ct) - } - - var m []goolib.RepoSpec - for dec.More() { - if err := dec.Decode(&m); err != nil { - return nil, err - } - } - - f, err := oswrap.Create(cf) - if err != nil { - return nil, err - } - j, err := json.Marshal(m) - if err != nil { - return nil, err - } - if _, err := f.Write(j); err != nil { - return nil, err - } - - // The .url files aren't used by googet but help developers and the - // curious figure out which file belongs to which repo/URL. - mf := fmt.Sprintf("%s.url", strings.TrimSuffix(cf, filepath.Ext(cf))) - if err = ioutil.WriteFile(mf, []byte(url), 0644); err != nil { - logger.Errorf("Failed to write '%s': %v", mf, err) - } - - return m, f.Close() -} - // unmarshalRepoPackages gets and unmarshals a repository URL or uses the cached contents // if mtime is less than cacheLife. // Successfully unmarshalled contents will be written to a cache. -func unmarshalRepoPackages(ctx context.Context, p, cacheDir string, cacheLife time.Duration, proxyServer string) ([]goolib.RepoSpec, error) { +func (d *Downloader) unmarshalRepoPackages(ctx context.Context, p, cacheDir string, cacheLife time.Duration) ([]goolib.RepoSpec, error) { pName := strings.TrimPrefix(p, "oauth-") cf := filepath.Join(cacheDir, fmt.Sprintf("%x.rs", sha256.Sum256([]byte(pName)))) @@ -196,34 +182,13 @@ func unmarshalRepoPackages(ctx context.Context, p, cacheDir string, cacheLife ti isGCSURL, bucket, object := goolib.SplitGCSUrl(pName) if isGCSURL { - return unmarshalRepoPackagesGCS(ctx, bucket, object, pName, cf, proxyServer) + return d.unmarshalRepoPackagesGCS(ctx, bucket, object, pName, cf) } - return unmarshalRepoPackagesHTTP(ctx, p, cf, proxyServer) + return d.unmarshalRepoPackagesHTTP(ctx, p, cf) } // Get gets a url using an optional proxy server, retrying once on any error. -func Get(ctx context.Context, path, proxyServer string) (*http.Response, error) { - httpClient := http.DefaultClient - proxy := http.ProxyFromEnvironment - if proxyServer != "" { - proxyURL, err := url.Parse(proxyServer) - if err != nil { - return nil, err - } - proxy = http.ProxyURL(proxyURL) - } - httpClient.Transport = &http.Transport{ - Proxy: proxy, - DialContext: (&net.Dialer{ - Timeout: 30 * time.Second, - KeepAlive: 30 * time.Second, - }).DialContext, - ForceAttemptHTTP2: true, - MaxIdleConns: 100, - IdleConnTimeout: 60 * time.Second, - TLSHandshakeTimeout: 10 * time.Second, - ExpectContinueTimeout: 1 * time.Second, - } +func (d *Downloader) Get(ctx context.Context, path string) (*http.Response, error) { useOauth := strings.HasPrefix(path, "oauth-") path = strings.TrimPrefix(path, "oauth-") req, err := http.NewRequest(http.MethodGet, path, nil) @@ -241,46 +206,43 @@ func Get(ctx context.Context, path, proxyServer string) (*http.Response, error) } req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", token.AccessToken)) } - resp, err := httpClient.Do(req) + resp, err := d.HTTPClient.Do(req) // We retry on any error once as this mitigates some // connection issues in certain situations. if err == nil { return resp, nil } - return httpClient.Do(req) + return d.HTTPClient.Do(req) } -func unmarshalRepoPackagesHTTP(ctx context.Context, repoURL string, cf string, proxyServer string) ([]goolib.RepoSpec, error) { +func (d *Downloader) unmarshalRepoPackagesHTTP(ctx context.Context, repoURL string, cf string) ([]goolib.RepoSpec, error) { indexURL := repoURL + "/index.gz" trimmedIndexURL := strings.TrimPrefix(indexURL, "oauth-") ct := "application/x-gzip" logger.Infof("Fetching %q", trimmedIndexURL) - res, err := Get(ctx, indexURL, proxyServer) + res, err := d.Get(ctx, indexURL) if err != nil { return nil, err } - - if err != nil || res.StatusCode != 200 { - //logger.Infof("Gzipped index returned status: %q, trying plain JSON.", res.Status) + if res.StatusCode != http.StatusOK { indexURL = repoURL + "/index" + trimmedIndexURL = strings.TrimPrefix(indexURL, "oauth-") ct = "application/json" logger.Infof("Fetching %q", trimmedIndexURL) - res, err = Get(ctx, indexURL, proxyServer) + res, err = d.Get(ctx, indexURL) if err != nil { return nil, err } - - if res.StatusCode != 200 { + if res.StatusCode != http.StatusOK { return nil, fmt.Errorf("index GET request returned status: %q", res.Status) } } - return decode(res.Body, ct, repoURL, cf) } -func unmarshalRepoPackagesGCS(ctx context.Context, bucket, object, url, cf string, proxyServer string) ([]goolib.RepoSpec, error) { - if proxyServer != "" { - logger.Errorf("Proxy server not supported with gs:// URLs, skiping repo 'gs://%s/%s'", bucket, object) +func (d *Downloader) unmarshalRepoPackagesGCS(ctx context.Context, bucket, object, url, cf string) ([]goolib.RepoSpec, error) { + if d.UsingProxyServer { + logger.Errorf("Proxy server not supported with gs:// URLs, skipping repo 'gs://%s/%s'", bucket, object) var empty []goolib.RepoSpec return empty, nil } @@ -315,6 +277,52 @@ func unmarshalRepoPackagesGCS(ctx context.Context, bucket, object, url, cf strin return decode(r, "application/json", url, cf) } +func decode(index io.ReadCloser, ct, url, cf string) ([]goolib.RepoSpec, error) { + defer index.Close() + + var dec *json.Decoder + switch ct { + case "application/x-gzip": + gr, err := gzip.NewReader(index) + if err != nil { + return nil, err + } + dec = json.NewDecoder(gr) + case "application/json": + dec = json.NewDecoder(index) + default: + return nil, fmt.Errorf("unsupported content type: %s", ct) + } + + var m []goolib.RepoSpec + for dec.More() { + if err := dec.Decode(&m); err != nil { + return nil, err + } + } + + f, err := oswrap.Create(cf) + if err != nil { + return nil, err + } + j, err := json.Marshal(m) + if err != nil { + return nil, err + } + if _, err := f.Write(j); err != nil { + return nil, err + } + + // The .url files aren't used by googet but help developers and the + // curious figure out which file belongs to which repo/URL. + mf := fmt.Sprintf("%s.url", strings.TrimSuffix(cf, filepath.Ext(cf))) + if err = ioutil.WriteFile(mf, []byte(url), 0644); err != nil { + logger.Errorf("Failed to write '%s': %v", mf, err) + } + + return m, f.Close() +} + // FindRepoSpec returns the RepoSpec in repo whose PackageSpec matches pi. func FindRepoSpec(pi goolib.PackageInfo, repo Repo) (goolib.RepoSpec, error) { for _, p := range repo.Packages { diff --git a/client/client_test.go b/client/client_test.go index 9cf2884..e39f24d 100644 --- a/client/client_test.go +++ b/client/client_test.go @@ -268,7 +268,11 @@ func TestUnmarshalRepoPackagesJSON(t *testing.T) { })) defer ts.Close() - got, err := unmarshalRepoPackages(context.Background(), ts.URL, tempDir, cacheLife, proxyServer) + d, err := NewDownloader(proxyServer) + if err != nil { + t.Fatalf("NewDownloader(%s): %v", proxyServer, err) + } + got, err := d.unmarshalRepoPackages(context.Background(), ts.URL, tempDir, cacheLife) if err != nil { t.Fatalf("Error running unmarshalRepoPackages: %v", err) } @@ -313,7 +317,11 @@ func TestUnmarshalRepoPackagesGzip(t *testing.T) { })) defer ts.Close() - got, err := unmarshalRepoPackages(context.Background(), ts.URL, tempDir, cacheLife, proxyServer) + d, err := NewDownloader(proxyServer) + if err != nil { + t.Fatalf("NewDownloader(%s): %v", proxyServer, err) + } + got, err := d.unmarshalRepoPackages(context.Background(), ts.URL, tempDir, cacheLife) if err != nil { t.Fatalf("Error running unmarshalRepoPackages: %v", err) } @@ -351,7 +359,11 @@ func TestUnmarshalRepoPackagesCache(t *testing.T) { } // No http server as this should use the cached content. - got, err := unmarshalRepoPackages(context.Background(), url, tempDir, cacheLife, proxyServer) + d, err := NewDownloader(proxyServer) + if err != nil { + t.Fatalf("NewDownloader(%s): %v", proxyServer, err) + } + got, err := d.unmarshalRepoPackages(context.Background(), url, tempDir, cacheLife) if err != nil { t.Fatalf("Error running unmarshalRepoPackages: %v", err) } diff --git a/download/download.go b/download/download.go index 01cccb6..84bb60b 100644 --- a/download/download.go +++ b/download/download.go @@ -23,6 +23,7 @@ import ( "errors" "fmt" "io" + "net/http" "net/url" "os" "path" @@ -37,34 +38,30 @@ import ( "github.com/google/logger" ) -const ( - httpOK = 200 -) - // Package downloads a package from the given url, // the provided SHA256 checksum will be checked during download. -func Package(ctx context.Context, pkgURL, dst, chksum, proxyServer string) error { +func Package(ctx context.Context, pkgURL, dst, chksum string, downloader *client.Downloader) error { if err := oswrap.RemoveAll(dst); err != nil { return err } isGCSURL, bucket, object := goolib.SplitGCSUrl(pkgURL) if isGCSURL { - return packageGCS(ctx, bucket, object, dst, chksum, "") + return packageGCS(ctx, bucket, object, dst, chksum) } - return packageHTTP(ctx, pkgURL, dst, chksum, proxyServer) + return packageHTTP(ctx, pkgURL, dst, chksum, downloader) } // Downloads a package from an HTTP(s) server -func packageHTTP(ctx context.Context, pkgURL, dst, chksum string, proxyServer string) error { - resp, err := client.Get(ctx, pkgURL, proxyServer) +func packageHTTP(ctx context.Context, pkgURL, dst, chksum string, downloader *client.Downloader) error { + resp, err := downloader.Get(ctx, pkgURL) if err != nil { return err } defer resp.Body.Close() - if resp.StatusCode != httpOK { - return fmt.Errorf("Invalid return code from server, got: %d, want: %d", resp.StatusCode, httpOK) + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("invalid return code from server, got: %d, want: %d", resp.StatusCode, http.StatusOK) } logger.Infof("Downloading %q", pkgURL) @@ -72,11 +69,7 @@ func packageHTTP(ctx context.Context, pkgURL, dst, chksum string, proxyServer st } // Downloads a package from Google Cloud Storage -func packageGCS(ctx context.Context, bucket, object string, dst, chksum string, proxyServer string) error { - if proxyServer != "" { - return fmt.Errorf("Proxy server not supported with GCS URLs") - } - +func packageGCS(ctx context.Context, bucket, object string, dst, chksum string) error { client, err := storage.NewClient(ctx) if err != nil { return err @@ -94,7 +87,7 @@ func packageGCS(ctx context.Context, bucket, object string, dst, chksum string, } // FromRepo downloads a package from a repo. -func FromRepo(ctx context.Context, rs goolib.RepoSpec, repo, dir string, proxyServer string) (string, error) { +func FromRepo(ctx context.Context, rs goolib.RepoSpec, repo, dir string, downloader *client.Downloader) (string, error) { repoURL, err := url.Parse(repo) if err != nil { return "", err @@ -114,11 +107,11 @@ func FromRepo(ctx context.Context, rs goolib.RepoSpec, repo, dir string, proxySe pn := goolib.PackageInfo{Name: rs.PackageSpec.Name, Arch: rs.PackageSpec.Arch, Ver: rs.PackageSpec.Version}.PkgName() dst := filepath.Join(dir, filepath.Base(pn)) - return dst, Package(ctx, pkgURL.String(), dst, rs.Checksum, proxyServer) + return dst, Package(ctx, pkgURL.String(), dst, rs.Checksum, downloader) } // Latest downloads the latest available version of a package. -func Latest(ctx context.Context, name, dir string, rm client.RepoMap, archs []string, proxyServer string) (string, error) { +func Latest(ctx context.Context, name, dir string, rm client.RepoMap, archs []string, downloader *client.Downloader) (string, error) { ver, repo, arch, err := client.FindRepoLatest(goolib.PackageInfo{Name: name, Arch: "", Ver: ""}, rm, archs) if err != nil { return "", err @@ -127,7 +120,7 @@ func Latest(ctx context.Context, name, dir string, rm client.RepoMap, archs []st if err != nil { return "", err } - return FromRepo(ctx, rs, repo, dir, proxyServer) + return FromRepo(ctx, rs, repo, dir, downloader) } func download(r io.Reader, dst, chksum string) (err error) { @@ -159,7 +152,7 @@ func download(r io.Reader, dst, chksum string) (err error) { } // ExtractPkg takes a path to a package and extracts it to a directory based on the -// package name, it returns the path to the extraced directory. +// package name, it returns the path to the extracted directory. func ExtractPkg(src string) (dst string, err error) { dst = strings.TrimSuffix(src, filepath.Ext(src)) if src == "" || dst == "" { diff --git a/googet_available.go b/googet_available.go index 53141ae..1e24e1b 100644 --- a/googet_available.go +++ b/googet_available.go @@ -73,8 +73,13 @@ func (cmd *availableCmd) Execute(ctx context.Context, f *flag.FlagSet, _ ...inte logger.Fatal("No repos defined, create a .repo file or pass using the -sources flag.") } + downloader, err := client.NewDownloader(proxyServer) + if err != nil { + logger.Fatal(err) + } + m := make(map[string][]string) - rm := client.AvailableVersions(ctx, repos, filepath.Join(rootDir, cacheDir), cacheLife, proxyServer) + rm := downloader.AvailableVersions(ctx, repos, filepath.Join(rootDir, cacheDir), cacheLife) for r, repo := range rm { for _, p := range repo.Packages { m[r] = append(m[r], p.PackageSpec.Name+"."+p.PackageSpec.Arch+"."+p.PackageSpec.Version) diff --git a/googet_download.go b/googet_download.go index 2192e1a..1d2c06d 100644 --- a/googet_download.go +++ b/googet_download.go @@ -58,7 +58,12 @@ func (cmd *downloadCmd) Execute(ctx context.Context, flags *flag.FlagSet, _ ...i logger.Fatal("No repos defined, create a .repo file or pass using the -sources flag.") } - rm := client.AvailableVersions(ctx, repos, filepath.Join(rootDir, cacheDir), cacheLife, proxyServer) + downloader, err := client.NewDownloader(proxyServer) + if err != nil { + logger.Fatal(err) + } + + rm := downloader.AvailableVersions(ctx, repos, filepath.Join(rootDir, cacheDir), cacheLife) exitCode := subcommands.ExitSuccess dir := cmd.downloadDir @@ -72,7 +77,7 @@ func (cmd *downloadCmd) Execute(ctx context.Context, flags *flag.FlagSet, _ ...i for _, arg := range flags.Args() { pi := goolib.PkgNameSplit(arg) if pi.Ver == "" { - if _, err := download.Latest(ctx, pi.Name, dir, rm, archs, proxyServer); err != nil { + if _, err := download.Latest(ctx, pi.Name, dir, rm, archs, downloader); err != nil { logger.Errorf("error downloading %s, %v", pi.Name, err) exitCode = subcommands.ExitFailure } @@ -97,7 +102,7 @@ func (cmd *downloadCmd) Execute(ctx context.Context, flags *flag.FlagSet, _ ...i exitCode = subcommands.ExitFailure continue } - if _, err := download.FromRepo(ctx, rs, repo, dir, proxyServer); err != nil { + if _, err := download.FromRepo(ctx, rs, repo, dir, downloader); err != nil { logger.Errorf("error downloading %s.%s %s, %v", pi.Name, pi.Arch, pi.Ver, err) exitCode = subcommands.ExitFailure continue diff --git a/googet_install.go b/googet_install.go index 87e9ec1..08fe206 100644 --- a/googet_install.go +++ b/googet_install.go @@ -80,6 +80,11 @@ func (cmd *installCmd) Execute(ctx context.Context, flags *flag.FlagSet, _ ...in logger.Fatal(err) } + downloader, err := client.NewDownloader(proxyServer) + if err != nil { + logger.Fatal(err) + } + var rm client.RepoMap for _, arg := range args { if ext := filepath.Ext(arg); ext == ".goo" { @@ -102,7 +107,7 @@ func (cmd *installCmd) Execute(ctx context.Context, flags *flag.FlagSet, _ ...in pi := goolib.PkgNameSplit(arg) if cmd.reinstall { - if err := reinstall(ctx, pi, *state, cmd.redownload); err != nil { + if err := reinstall(ctx, pi, *state, cmd.redownload, downloader); err != nil { logger.Errorf("Error reinstalling %s: %v", pi.Name, err) exitCode = subcommands.ExitFailure continue @@ -116,7 +121,7 @@ func (cmd *installCmd) Execute(ctx context.Context, flags *flag.FlagSet, _ ...in if repos == nil { logger.Fatal("No repos defined, create a .repo file or pass using the -sources flag.") } - rm = client.AvailableVersions(ctx, repos, filepath.Join(rootDir, cacheDir), cacheLife, proxyServer) + rm = downloader.AvailableVersions(ctx, repos, filepath.Join(rootDir, cacheDir), cacheLife) } if pi.Ver == "" { v, _, a, err := client.FindRepoLatest(pi, rm, archs) @@ -161,7 +166,7 @@ func (cmd *installCmd) Execute(ctx context.Context, flags *flag.FlagSet, _ ...in continue } } - if err := install.FromRepo(ctx, pi, r, cache, rm, archs, state, cmd.dbOnly, proxyServer); err != nil { + if err := install.FromRepo(ctx, pi, r, cache, rm, archs, state, cmd.dbOnly, downloader); err != nil { logger.Errorf("Error installing %s.%s.%s: %v", pi.Name, pi.Arch, pi.Ver, err) exitCode = subcommands.ExitFailure continue @@ -173,7 +178,7 @@ func (cmd *installCmd) Execute(ctx context.Context, flags *flag.FlagSet, _ ...in return exitCode } -func reinstall(ctx context.Context, pi goolib.PackageInfo, state client.GooGetState, rd bool) error { +func reinstall(ctx context.Context, pi goolib.PackageInfo, state client.GooGetState, rd bool, downloader *client.Downloader) error { ps, err := state.GetPackageState(pi) if err != nil { return fmt.Errorf("cannot reinstall something that is not already installed") @@ -184,7 +189,7 @@ func reinstall(ctx context.Context, pi goolib.PackageInfo, state client.GooGetSt return nil } } - if err := install.Reinstall(ctx, ps, state, rd, proxyServer); err != nil { + if err := install.Reinstall(ctx, ps, state, rd, downloader); err != nil { return fmt.Errorf("error reinstalling %s, %v", pi.Name, err) } return nil diff --git a/googet_latest.go b/googet_latest.go index 1a84491..ce8f577 100644 --- a/googet_latest.go +++ b/googet_latest.go @@ -55,7 +55,12 @@ func (cmd *latestCmd) Execute(ctx context.Context, flags *flag.FlagSet, _ ...int logger.Fatal("No repos defined, create a .repo file or pass using the -sources flag.") } - rm := client.AvailableVersions(ctx, repos, filepath.Join(rootDir, cacheDir), cacheLife, proxyServer) + downloader, err := client.NewDownloader(proxyServer) + if err != nil { + logger.Fatal(err) + } + + rm := downloader.AvailableVersions(ctx, repos, filepath.Join(rootDir, cacheDir), cacheLife) v, _, a, err := client.FindRepoLatest(pi, rm, archs) if err != nil { logger.Fatal(err) diff --git a/googet_remove.go b/googet_remove.go index b52ea13..1de0874 100644 --- a/googet_remove.go +++ b/googet_remove.go @@ -23,6 +23,7 @@ import ( "os" "path/filepath" + "github.com/google/googet/v2/client" "github.com/google/googet/v2/goolib" "github.com/google/googet/v2/remove" "github.com/google/logger" @@ -52,6 +53,11 @@ func (cmd *removeCmd) Execute(ctx context.Context, flags *flag.FlagSet, _ ...int logger.Error(err) } + downloader, err := client.NewDownloader(proxyServer) + if err != nil { + logger.Fatal(err) + } + for _, arg := range flags.Args() { pi := goolib.PkgNameSplit(arg) var ins []string @@ -83,7 +89,7 @@ func (cmd *removeCmd) Execute(ctx context.Context, flags *flag.FlagSet, _ ...int } } fmt.Printf("Removing %s and all dependencies...\n", pi.Name) - if err = remove.All(ctx, pi, deps, state, cmd.dbOnly, proxyServer); err != nil { + if err = remove.All(ctx, pi, deps, state, cmd.dbOnly, downloader); err != nil { logger.Errorf("error removing %s, %v", arg, err) exitCode = subcommands.ExitFailure continue diff --git a/googet_update.go b/googet_update.go index 57841f9..5241179 100644 --- a/googet_update.go +++ b/googet_update.go @@ -68,7 +68,12 @@ func (cmd *updateCmd) Execute(ctx context.Context, _ *flag.FlagSet, _ ...interfa logger.Fatal("No repos defined, create a .repo file or pass using the -sources flag.") } - rm := client.AvailableVersions(ctx, repos, filepath.Join(rootDir, cacheDir), cacheLife, proxyServer) + downloader, err := client.NewDownloader(proxyServer) + if err != nil { + logger.Fatal(err) + } + + rm := downloader.AvailableVersions(ctx, repos, filepath.Join(rootDir, cacheDir), cacheLife) ud := updates(pm, rm) if ud == nil { fmt.Println("No updates available for any installed packages.") @@ -88,7 +93,7 @@ func (cmd *updateCmd) Execute(ctx context.Context, _ *flag.FlagSet, _ ...interfa if err != nil { logger.Errorf("Error finding repo: %v.", err) } - if err := install.FromRepo(ctx, pi, r, cache, rm, archs, state, cmd.dbOnly, proxyServer); err != nil { + if err := install.FromRepo(ctx, pi, r, cache, rm, archs, state, cmd.dbOnly, downloader); err != nil { logger.Errorf("Error updating %s %s %s: %v", pi.Arch, pi.Name, pi.Ver, err) exitCode = subcommands.ExitFailure continue diff --git a/googet_verify.go b/googet_verify.go index 89b6221..fa8f7d9 100644 --- a/googet_verify.go +++ b/googet_verify.go @@ -22,6 +22,7 @@ import ( "os" "path/filepath" + "github.com/google/googet/v2/client" "github.com/google/googet/v2/goolib" "github.com/google/googet/v2/install" "github.com/google/googet/v2/verify" @@ -58,6 +59,11 @@ func (cmd *verifyCmd) Execute(ctx context.Context, flags *flag.FlagSet, _ ...int logger.Error(err) } + downloader, err := client.NewDownloader(proxyServer) + if err != nil { + logger.Fatal(err) + } + for _, arg := range flags.Args() { pi := goolib.PkgNameSplit(arg) ps, err := state.GetPackageState(pi) @@ -80,7 +86,7 @@ func (cmd *verifyCmd) Execute(ctx context.Context, flags *flag.FlagSet, _ ...int continue } - v, err := verify.Command(ctx, ps, proxyServer) + v, err := verify.Command(ctx, ps, downloader) if err != nil { logger.Errorf("Error running verify command for %s: %v", pkg, err) exitCode = subcommands.ExitFailure @@ -99,7 +105,7 @@ func (cmd *verifyCmd) Execute(ctx context.Context, flags *flag.FlagSet, _ ...int msg := fmt.Sprintf("Verification failed for %s, reinstalling...", pkg) logger.Info(msg) fmt.Println(msg) - if err := install.Reinstall(ctx, ps, *state, false, proxyServer); err != nil { + if err := install.Reinstall(ctx, ps, *state, false, downloader); err != nil { logger.Errorf("Error reinstalling %s, %v", pi.Name, err) } } else if !v { diff --git a/install/install.go b/install/install.go index 244b875..2dc9361 100644 --- a/install/install.go +++ b/install/install.go @@ -67,7 +67,7 @@ func resolveConflicts(ps *goolib.PkgSpec, state *client.GooGetState) error { return nil } -func resolveReplacements(ctx context.Context, ps *goolib.PkgSpec, state *client.GooGetState, dbOnly bool, proxyServer string) error { +func resolveReplacements(ctx context.Context, ps *goolib.PkgSpec, state *client.GooGetState, dbOnly bool, downloader *client.Downloader) error { // Check for and remove any package this replaces. // TODO(ajackura): Make sure no replacements are listed as // dependencies or subdependancies. @@ -82,14 +82,14 @@ func resolveReplacements(ctx context.Context, ps *goolib.PkgSpec, state *client. } deps, _ := remove.EnumerateDeps(pi, *state) logger.Infof("%s replaces %s, removing", ps, pi) - if err := remove.All(ctx, pi, deps, state, dbOnly, proxyServer); err != nil { + if err := remove.All(ctx, pi, deps, state, dbOnly, downloader); err != nil { return err } } return nil } -func installDeps(ctx context.Context, ps *goolib.PkgSpec, cache string, rm client.RepoMap, archs []string, state *client.GooGetState, dbOnly bool, proxyServer string) error { +func installDeps(ctx context.Context, ps *goolib.PkgSpec, cache string, rm client.RepoMap, archs []string, state *client.GooGetState, dbOnly bool, downloader *client.Downloader) error { logger.Infof("Resolving conflicts and dependencies for %s %s version %s", ps.Arch, ps.Name, ps.Version) if err := resolveConflicts(ps, state); err != nil { return err @@ -116,7 +116,7 @@ func installDeps(ctx context.Context, ps *goolib.PkgSpec, cache string, rm clien } if c > -1 { logger.Infof("Dependency found: %s.%s %s is available", pi.Name, arch, v) - if err := FromRepo(ctx, goolib.PackageInfo{Name: pi.Name, Arch: arch, Ver: v}, repo, cache, rm, archs, state, dbOnly, proxyServer); err != nil { + if err := FromRepo(ctx, goolib.PackageInfo{Name: pi.Name, Arch: arch, Ver: v}, repo, cache, rm, archs, state, dbOnly, downloader); err != nil { return err } ins = true @@ -125,22 +125,22 @@ func installDeps(ctx context.Context, ps *goolib.PkgSpec, cache string, rm clien return fmt.Errorf("cannot resolve dependency, %s.%s version %s or greater not installed and not available in any repo", pi.Name, arch, ver) } } - return resolveReplacements(ctx, ps, state, dbOnly, proxyServer) + return resolveReplacements(ctx, ps, state, dbOnly, downloader) } // FromRepo installs a package and all dependencies from a repository. -func FromRepo(ctx context.Context, pi goolib.PackageInfo, repo, cache string, rm client.RepoMap, archs []string, state *client.GooGetState, dbOnly bool, proxyServer string) error { +func FromRepo(ctx context.Context, pi goolib.PackageInfo, repo, cache string, rm client.RepoMap, archs []string, state *client.GooGetState, dbOnly bool, downloader *client.Downloader) error { logger.Infof("Starting install of %s.%s.%s", pi.Name, pi.Arch, pi.Ver) fmt.Printf("Installing %s.%s.%s and dependencies...\n", pi.Name, pi.Arch, pi.Ver) rs, err := client.FindRepoSpec(pi, rm[repo]) if err != nil { return err } - if err := installDeps(ctx, rs.PackageSpec, cache, rm, archs, state, dbOnly, proxyServer); err != nil { + if err := installDeps(ctx, rs.PackageSpec, cache, rm, archs, state, dbOnly, downloader); err != nil { return err } - dst, err := download.FromRepo(ctx, rs, repo, cache, proxyServer) + dst, err := download.FromRepo(ctx, rs, repo, cache, downloader) if err != nil { return err } @@ -250,7 +250,7 @@ func FromDisk(arg, cache string, state *client.GooGetState, dbOnly, ri bool) err } // Reinstall reinstalls and optionally redownloads, a package. -func Reinstall(ctx context.Context, ps client.PackageState, state client.GooGetState, rd bool, proxyServer string) error { +func Reinstall(ctx context.Context, ps client.PackageState, state client.GooGetState, rd bool, downloader *client.Downloader) error { pi := goolib.PackageInfo{Name: ps.PackageSpec.Name, Arch: ps.PackageSpec.Arch, Ver: ps.PackageSpec.Version} logger.Infof("Starting reinstall of %s.%s, version %s", pi.Name, pi.Arch, pi.Ver) fmt.Printf("Reinstalling %s.%s %s and dependencies...\n", pi.Name, pi.Arch, pi.Ver) @@ -284,7 +284,7 @@ func Reinstall(ctx context.Context, ps client.PackageState, state client.GooGetS if ps.DownloadURL == "" { return fmt.Errorf("can not redownload %s.%s.%s, DownloadURL not saved", pi.Name, pi.Arch, pi.Ver) } - if err := download.Package(ctx, ps.DownloadURL, ps.LocalPath, ps.Checksum, proxyServer); err != nil { + if err := download.Package(ctx, ps.DownloadURL, ps.LocalPath, ps.Checksum, downloader); err != nil { return fmt.Errorf("error redownloading package: %v", err) } } diff --git a/remove/remove.go b/remove/remove.go index 1e2571a..9151273 100644 --- a/remove/remove.go +++ b/remove/remove.go @@ -28,7 +28,7 @@ import ( "github.com/google/logger" ) -func uninstallPkg(ctx context.Context, pi goolib.PackageInfo, state *client.GooGetState, dbOnly bool, proxyServer string) error { +func uninstallPkg(ctx context.Context, pi goolib.PackageInfo, state *client.GooGetState, dbOnly bool, downloader *client.Downloader) error { logger.Infof("Executing removal of package %q", pi.Name) ps, err := state.GetPackageState(pi) if err != nil { @@ -65,7 +65,7 @@ func uninstallPkg(ctx context.Context, pi goolib.PackageInfo, state *client.GooG if ps.DownloadURL == "" { return fmt.Errorf("can not redownload %s.%s.%s, DownloadURL not saved", pi.Name, pi.Arch, pi.Ver) } - if err := download.Package(ctx, ps.DownloadURL, ps.LocalPath, ps.Checksum, proxyServer); err != nil { + if err := download.Package(ctx, ps.DownloadURL, ps.LocalPath, ps.Checksum, downloader); err != nil { return fmt.Errorf("error redownloading %s.%s.%s, package may no longer exist in the repo, you can use the '-db_only' flag to remove it form the database: %v", pi.Name, pi.Arch, pi.Ver, err) } } @@ -161,19 +161,19 @@ func EnumerateDeps(pi goolib.PackageInfo, state client.GooGetState) (DepMap, []s return dm, dl } -// All removes a package and all dependant packages. Packages with no dependant packages +// All removes a package and all dependent packages. Packages with no dependent packages // will be removed first. -func All(ctx context.Context, pi goolib.PackageInfo, deps DepMap, state *client.GooGetState, dbOnly bool, proxyServer string) error { +func All(ctx context.Context, pi goolib.PackageInfo, deps DepMap, state *client.GooGetState, dbOnly bool, downloader *client.Downloader) error { for len(deps) > 1 { for dep := range deps { if len(deps[dep]) == 0 { di := goolib.PkgNameSplit(dep) - if err := uninstallPkg(ctx, di, state, dbOnly, proxyServer); err != nil { + if err := uninstallPkg(ctx, di, state, dbOnly, downloader); err != nil { return err } deps.remove(dep) } } } - return uninstallPkg(ctx, pi, state, dbOnly, proxyServer) + return uninstallPkg(ctx, pi, state, dbOnly, downloader) } diff --git a/remove/remove_test.go b/remove/remove_test.go index aba9c13..0695140 100644 --- a/remove/remove_test.go +++ b/remove/remove_test.go @@ -107,8 +107,11 @@ func TestUninstallPkg(t *testing.T) { LocalPath: f.Name(), }, } - - if err := uninstallPkg(context.Background(), goolib.PackageInfo{Name: "foo"}, st, false, ""); err != nil { + d, err := client.NewDownloader("") + if err != nil { + t.Fatalf("NewDownloader: %v", err) + } + if err := uninstallPkg(context.Background(), goolib.PackageInfo{Name: "foo"}, st, false, d); err != nil { t.Fatalf("Error running uninstallPkg: %v", err) } diff --git a/verify/verify.go b/verify/verify.go index 265339f..c6aa559 100644 --- a/verify/verify.go +++ b/verify/verify.go @@ -20,8 +20,6 @@ import ( "context" "fmt" "io" - "net/http" - "net/url" "os" "path/filepath" "strings" @@ -110,7 +108,7 @@ func Files(ps client.PackageState) (bool, error) { // Command runs a packages verify command. // Will only return true if the verify command exits with 0 or an approved // return code. -func Command(ctx context.Context, ps client.PackageState, proxyServer string) (bool, error) { +func Command(ctx context.Context, ps client.PackageState, downloader *client.Downloader) (bool, error) { if ps.PackageSpec.Verify.Path == "" { return true, nil } @@ -140,16 +138,7 @@ func Command(ctx context.Context, ps client.PackageState, proxyServer string) (b if ps.DownloadURL == "" { return false, fmt.Errorf("can not pull package %s from repo, DownloadURL not saved", pkg) } - - httpClient := &http.Client{} - if proxyServer != "" { - proxyURL, err := url.Parse(proxyServer) - if err != nil { - return false, err - } - httpClient.Transport = &http.Transport{Proxy: http.ProxyURL(proxyURL)} - } - resp, err := httpClient.Get(ps.DownloadURL) + resp, err := downloader.Get(ctx, ps.DownloadURL) if err != nil { return false, err } @@ -164,13 +153,13 @@ func Command(ctx context.Context, ps client.PackageState, proxyServer string) (b } f.Close() - // Try just running the extracted command, rextract the full package on any error. + // Try just running the extracted command, re-extract the full package on any error. if err := system.Verify(dir, ps.PackageSpec); err == nil { return true, nil } if rd { - if err := download.Package(ctx, ps.DownloadURL, ps.LocalPath, ps.Checksum, proxyServer); err != nil { + if err := download.Package(ctx, ps.DownloadURL, ps.LocalPath, ps.Checksum, downloader); err != nil { return false, fmt.Errorf("error redownloading package: %v", err) } }