Allow specifying the page size and filter for Backstage entities (#164)

Allows you to customise the `limit` and `filter` parameters that we pass
to the Backstage API. Fixes #60.

cmd/catalog-importer/cmd/import.go

@@ -8,6 +8,7 @@ import (
 
 	"github.com/alecthomas/kingpin/v2"
 	kitlog "github.com/go-kit/kit/log"
+	"github.com/hashicorp/go-cleanhttp"
 	"github.com/incident-io/catalog-importer/v2/config"
 	"github.com/incident-io/catalog-importer/v2/output"
 	"github.com/incident-io/catalog-importer/v2/source"
@@ -69,7 +70,7 @@ func (opt *ImportOptions) Run(ctx context.Context, logger kitlog.Logger) error {
 	}
 
 	logger.Log("msg", "loading entries from files", "files", opt.Files)
-	sourceEntries, err := src.Load(ctx, logger)
+	sourceEntries, err := src.Load(ctx, logger, cleanhttp.DefaultClient())
 	if err != nil {
 		return errors.Wrap(err, "reading source files")
 	}

docs/sources.md

@@ -160,6 +160,13 @@ https://api.roadie.so/api/catalog/entities
 
 And you should append `sign_jwt: false,` to your jsonnet, as explained above.
 
+### Other parameters
+
+You can also specify the following parameters in your source:
+
+* `page_size`: Allows you to customise how many entities we fetch from Backstage at once.
+* `filter`: Allows you to filter entities as per [the Backstage API spec](https://backstage.io/docs/features/software-catalog/software-catalog-api/#get-entities), for example `kind=user,metadata.namespace=default`.
+
 ## `github`
 
 This source can pull files matching a pattern from across repositories in a

go.mod

@@ -18,6 +18,7 @@ require (
 	github.com/google/uuid v1.5.0
 	github.com/hashicorp/go-cleanhttp v0.5.2
 	github.com/hashicorp/go-retryablehttp v0.7.2
+	github.com/jarcoal/httpmock v1.3.1
 	github.com/machinebox/graphql v0.2.2
 	github.com/manifoldco/promptui v0.9.0
 	github.com/oapi-codegen/oapi-codegen/v2 v2.4.1

go.sum

@@ -99,6 +99,8 @@ github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpO
 github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
 github.com/invopop/yaml v0.3.1 h1:f0+ZpmhfBSS4MhG+4HYseMdJhoeeopbSKbq5Rpeelso=
 github.com/invopop/yaml v0.3.1/go.mod h1:PMOp3nn4/12yEZUFfmOuNHJsZToEEOwoWsT+D81KkeA=
+github.com/jarcoal/httpmock v1.3.1 h1:iUx3whfZWVf3jT01hQTO/Eo5sAYtB2/rqaUuOtpInww=
+github.com/jarcoal/httpmock v1.3.1/go.mod h1:3yb8rc4BI7TCBhFY8ng0gjuLKJNquuDNiPaZjnENuYg=
 github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
 github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
 github.com/juju/gnuflag v0.0.0-20171113085948-2ce1bb71843d/go.mod h1:2PavIy+JPciBPrBUjwbNvtwB6RQlve+hkpll6QSNmOE=
@@ -127,6 +129,7 @@ github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D
 github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
 github.com/mattn/go-runewidth v0.0.14 h1:+xnbZSEeDbOIg5/mE6JF0w6n9duR1l3/WmbinWVwUuU=
 github.com/mattn/go-runewidth v0.0.14/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
+github.com/maxatome/go-testdeep v1.12.0 h1:Ql7Go8Tg0C1D/uMMX59LAoYK7LffeJQ6X2T04nTH68g=
 github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ=
 github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw=
 github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 h1:RWengNIwukTxcDr9M+97sNutRR1RKhG96O6jWumTTnw=

source/source.go

@@ -3,10 +3,12 @@ package source
 import (
 	"context"
 	"fmt"
+	"net/http"
 	"reflect"
 
 	kitlog "github.com/go-kit/kit/log"
 	validation "github.com/go-ozzo/ozzo-validation/v4"
+	"github.com/hashicorp/go-cleanhttp"
 )
 
 // SourceEntry is an entry that has been discovered in a source, with the contents of the
@@ -54,7 +56,7 @@ func (s Source) Validate() error {
 
 type SourceBackend interface {
 	String() string
-	Load(ctx context.Context, logger kitlog.Logger) ([]*SourceEntry, error)
+	Load(ctx context.Context, logger kitlog.Logger, client *http.Client) ([]*SourceEntry, error)
 }
 
 func (s Source) Backend() (SourceBackend, error) {
@@ -88,5 +90,5 @@ func (s Source) Load(ctx context.Context, logger kitlog.Logger) ([]*SourceEntry,
 		return nil, err
 	}
 
-	return source.Load(ctx, logger)
+	return source.Load(ctx, logger, cleanhttp.DefaultClient())
 }

source/source_backstage.go

@@ -6,13 +6,14 @@ import (
 	"encoding/json"
 	"fmt"
 	"net/http"
+	"net/url"
+	"strconv"
 	"time"
 
 	kitlog "github.com/go-kit/kit/log"
 	"github.com/go-ozzo/ozzo-validation/is"
 	validation "github.com/go-ozzo/ozzo-validation/v4"
 	"github.com/golang-jwt/jwt"
-	"github.com/hashicorp/go-cleanhttp"
 	"github.com/pkg/errors"
 )
 
@@ -21,6 +22,8 @@ type SourceBackstage struct {
 	Token    Credential `json:"token"`
 	SignJWT  *bool      `json:"sign_jwt"`
 	Header   string     `json:"header"`
+	PageSize int        `json:"page_size"`
+	Filter   string     `json:"filter"`
 }
 
 func (s SourceBackstage) Validate() error {
@@ -36,7 +39,7 @@ func (s SourceBackstage) String() string {
 	return fmt.Sprintf("backstage (endpoint=%s)", s.Endpoint)
 }
 
-func (s SourceBackstage) Load(ctx context.Context, logger kitlog.Logger) ([]*SourceEntry, error) {
+func (s SourceBackstage) Load(ctx context.Context, logger kitlog.Logger, client *http.Client) ([]*SourceEntry, error) {
 	var token string
 	if s.Token != "" {
 		// If not provided or explicitly enabled, sign the token into a JWT and use that as
@@ -53,26 +56,36 @@ func (s SourceBackstage) Load(ctx context.Context, logger kitlog.Logger) ([]*Sou
 		}
 	}
 
-	client := cleanhttp.DefaultClient()
-
 	var (
 		limit  = 100
 		offset = 0
 	)
 
+	if s.PageSize != 0 {
+		limit = s.PageSize
+	}
+
 	entries := []*SourceEntry{}
 	for {
-		req, err := http.NewRequestWithContext(ctx, http.MethodGet, s.Endpoint+fmt.Sprintf("?limit=%d&offset=%d", limit, offset), nil)
+		query := url.Values{}
+		query.Set("limit", strconv.Itoa(limit))
+		query.Set("offset", strconv.Itoa(offset))
+
+		if s.Filter != "" {
+			query.Set("filter", s.Filter)
+		}
+
+		req, err := http.NewRequestWithContext(ctx, http.MethodGet, s.Endpoint+"?"+query.Encode(), nil)
 		if err != nil {
 			return nil, errors.Wrap(err, "building Backstage URL")
 		}
 
 		if token != "" {
 
-			header := s.Header;
+			header := s.Header
 
 			if header == "" {
-				header = "Authorization";
+				header = "Authorization"
 			}
 
 			req.Header.Add(header, fmt.Sprintf("Bearer %s", token))

source/source_backstage_test.go

@@ -0,0 +1,102 @@
+package source_test
+
+import (
+	"context"
+	"net/http"
+	"os"
+
+	kitlog "github.com/go-kit/kit/log"
+	"github.com/hashicorp/go-cleanhttp"
+	"github.com/incident-io/catalog-importer/v2/source"
+	"github.com/jarcoal/httpmock"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("SourceBackstage", func() {
+	var (
+		ctx    context.Context
+		logger kitlog.Logger
+	)
+
+	BeforeEach(func() {
+		ctx = context.Background()
+		logger = kitlog.NewLogfmtLogger(kitlog.NewSyncWriter(os.Stderr))
+	})
+
+	var (
+		s      source.SourceBackstage
+		client *http.Client
+		mock   *httpmock.MockTransport
+	)
+
+	BeforeEach(func() {
+		s = source.SourceBackstage{
+			Endpoint: "https://example.com/api/entities",
+		}
+
+		client = cleanhttp.DefaultClient()
+		mock = httpmock.NewMockTransport()
+		client.Transport = mock
+	})
+
+	Describe("Load", func() {
+		var backstageRequest *http.Request
+
+		BeforeEach(func() {
+			mock.RegisterResponder(
+				http.MethodGet,
+				"https://example.com/api/entities",
+				func(req *http.Request) (*http.Response, error) {
+					backstageRequest = req
+					resp, err := httpmock.NewJsonResponse(http.StatusOK, []map[string]any{})
+					Expect(err).To(Succeed())
+					return resp, nil
+				},
+			)
+		})
+
+		JustBeforeEach(func() {
+			_, err := s.Load(ctx, logger, client)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(backstageRequest).NotTo(BeNil())
+		})
+
+		Context("page size", func() {
+			When("no page size is specified", func() {
+				It("uses the default page size", func() {
+					Expect(backstageRequest.URL.Query().Get("limit")).To(Equal("100"))
+				})
+			})
+
+			When("the page size is overridden", func() {
+				BeforeEach(func() {
+					s.PageSize = 30
+				})
+
+				It("uses the default page size", func() {
+					Expect(backstageRequest.URL.Query().Get("limit")).To(Equal("30"))
+				})
+			})
+		})
+
+		Context("filter", func() {
+			When("no filter is specified", func() {
+				It("uses the default page size", func() {
+					Expect(backstageRequest.URL.Query().Has("filter")).To(BeFalse())
+				})
+			})
+
+			When("a filter is specified", func() {
+				BeforeEach(func() {
+					s.Filter = "kind=user,metadata.namespace=default"
+				})
+
+				It("is included in the request", func() {
+					Expect(backstageRequest.URL.Query().Get("filter")).To(Equal("kind=user,metadata.namespace=default"))
+				})
+			})
+		})
+	})
+})

source/source_exec.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"context"
 	"fmt"
+	"net/http"
 	"os"
 	"os/exec"
 	"strings"
@@ -28,7 +29,7 @@ func (s SourceExec) String() string {
 	return fmt.Sprintf("exec (command=%s)", strings.Join(s.Command, ","))
 }
 
-func (s SourceExec) Load(ctx context.Context, logger kitlog.Logger) ([]*SourceEntry, error) {
+func (s SourceExec) Load(ctx context.Context, logger kitlog.Logger, _ *http.Client) ([]*SourceEntry, error) {
 	var (
 		command = s.Command[0]
 		args    = s.Command[1:]

source/source_github.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"encoding/base64"
 	"fmt"
+	"net/http"
 	"regexp"
 	"strings"
 	"sync"
@@ -36,7 +37,7 @@ func (s SourceGitHub) String() string {
 	return fmt.Sprintf("github (repos=%s files=%s)", s.Repos, s.Files)
 }
 
-func (s SourceGitHub) Load(ctx context.Context, logger kitlog.Logger) ([]*SourceEntry, error) {
+func (s SourceGitHub) Load(ctx context.Context, logger kitlog.Logger, _ *http.Client) ([]*SourceEntry, error) {
 	client := github.NewClient(oauth2.NewClient(ctx, oauth2.StaticTokenSource(
 		&oauth2.Token{AccessToken: string(s.Token)},
 	)))

source/source_graphql.go

@@ -4,13 +4,13 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"net/http"
 	"reflect"
 	"strings"
 
 	kitlog "github.com/go-kit/kit/log"
 	"github.com/go-ozzo/ozzo-validation/is"
 	validation "github.com/go-ozzo/ozzo-validation/v4"
-	"github.com/hashicorp/go-cleanhttp"
 	"github.com/machinebox/graphql"
 	"github.com/pkg/errors"
 	"github.com/samber/lo"
@@ -60,10 +60,9 @@ func (s SourceGraphQL) String() string {
 	return fmt.Sprintf("graphql (endpoint=%s)", s.Endpoint)
 }
 
-func (s SourceGraphQL) Load(ctx context.Context, logger kitlog.Logger) ([]*SourceEntry, error) {
-	client := graphql.NewClient(string(s.Endpoint),
-		graphql.WithHTTPClient(cleanhttp.DefaultClient()))
-	client.Log = func(msg string) {
+func (s SourceGraphQL) Load(ctx context.Context, logger kitlog.Logger, client *http.Client) ([]*SourceEntry, error) {
+	graphQLClient := graphql.NewClient(string(s.Endpoint), graphql.WithHTTPClient(client))
+	graphQLClient.Log = func(msg string) {
 		logger.Log("msg", msg)
 	}
 
@@ -102,7 +101,7 @@ func (s SourceGraphQL) Load(ctx context.Context, logger kitlog.Logger) ([]*Sourc
 
 		logger.Log("msg", "issuing GraphQL query",
 			"page", page, "offset", offset, "cursor", cursor)
-		err := client.Run(ctx, req, &data)
+		err := graphQLClient.Run(ctx, req, &data)
 		if err != nil {
 			return nil, errors.Wrap(err, "failed to execute GraphQL query")
 		}

source/source_inline.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"net/http"
 
 	kitlog "github.com/go-kit/kit/log"
 	validation "github.com/go-ozzo/ozzo-validation/v4"
@@ -22,7 +23,7 @@ func (s SourceInline) String() string {
 	return "inline" // we can't put all entries here, that would be too much
 }
 
-func (s SourceInline) Load(ctx context.Context, logger kitlog.Logger) ([]*SourceEntry, error) {
+func (s SourceInline) Load(ctx context.Context, logger kitlog.Logger, _ *http.Client) ([]*SourceEntry, error) {
 	entries := []*SourceEntry{}
 	for idx, entry := range s.Entries {
 		data, err := json.Marshal(entry)

source/source_local.go

@@ -3,6 +3,7 @@ package source
 import (
 	"context"
 	"fmt"
+	"net/http"
 	"os"
 	"strings"
 
@@ -27,7 +28,7 @@ func (s SourceLocal) String() string {
 	return fmt.Sprintf("local (files=%s)", strings.Join(s.Files, ","))
 }
 
-func (s SourceLocal) Load(ctx context.Context, logger kitlog.Logger) ([]*SourceEntry, error) {
+func (s SourceLocal) Load(ctx context.Context, logger kitlog.Logger, _ *http.Client) ([]*SourceEntry, error) {
 	results := map[string]*SourceEntry{}
 	for _, pattern := range s.Files {
 		matches, err := filepathx.Glob(pattern)