From 2ed4df1950a9f578ed9f15fb1429ea0917df8f86 Mon Sep 17 00:00:00 2001 From: Christian Mesh Date: Fri, 8 Nov 2024 10:26:43 -0500 Subject: [PATCH 1/3] Add namespacePrefix for more efficient population Signed-off-by: Christian Mesh --- backend/cmd/generate/main.go | 3 +++ backend/internal/backend.go | 19 ++++++++++++++++ backend/internal/moduleindex/generator.go | 23 ++++++++++++++++++++ backend/internal/providerindex/generator.go | 24 +++++++++++++++++++++ 4 files changed, 69 insertions(+) diff --git a/backend/cmd/generate/main.go b/backend/cmd/generate/main.go index d31a7306..380728c4 100644 --- a/backend/cmd/generate/main.go +++ b/backend/cmd/generate/main.go @@ -24,6 +24,7 @@ func main() { licensesFile := "" skipUpdateProviders := false skipUpdateModules := false + namespacePrefix := "" namespace := "" name := "" targetSystem := "" @@ -55,6 +56,7 @@ func main() { flag.StringVar(&licensesFile, "licenses-file", licensesFile, "JSON file containing a list of approved licenses to include when indexing. (required)") flag.BoolVar(&skipUpdateProviders, "skip-update-providers", skipUpdateProviders, "Skip updating provider indexes.") flag.BoolVar(&skipUpdateModules, "skip-update-modules", skipUpdateModules, "Skip updating module indexes.") + flag.StringVar(&namespacePrefix, "namespace-prefix", namespace, "Limit updates to a namespace.") flag.StringVar(&namespace, "namespace", namespace, "Limit updates to a namespace.") flag.StringVar(&name, "name", name, "Limit updates to a name. Only works in conjunction with -namespace. For providers, this will result in a single provider getting updated. For modules, this will update all target systems under a name.") flag.StringVar(&targetSystem, "target-system", targetSystem, "Limit updates to a target system for module updates only. Only works in conjunction with -namespace and -name.") @@ -124,6 +126,7 @@ func main() { if err := backendInstance.Generate( ctx, append(forceOpts, + backend.WithNamespacePrefix(namespacePrefix), backend.WithNamespace(namespace), backend.WithName(name), backend.WithTargetSystem(targetSystem), diff --git a/backend/internal/backend.go b/backend/internal/backend.go index e3d59134..7582c7a7 100644 --- a/backend/internal/backend.go +++ b/backend/internal/backend.go @@ -77,6 +77,7 @@ type GenerateOpt func(c *GenerateConfig) error type GenerateConfig struct { SkipUpdateProviders bool SkipUpdateModules bool + NamespacePrefix string Namespace string Name string TargetSystem string @@ -206,6 +207,16 @@ var namespaceRe = regexp.MustCompile("^[a-zA-Z0-9._-]*$") var nameRe = regexp.MustCompile("^[a-zA-Z0-9._-]*$") var targetSystemRe = regexp.MustCompile("^[a-zA-Z0-9._-]*$") +func WithNamespacePrefix(namespacePrefix string) GenerateOpt { + return func(c *GenerateConfig) error { + if !namespaceRe.MatchString(namespacePrefix) { + return fmt.Errorf("invalid namespace: %s", namespaceRe) + } + c.NamespacePrefix = namespacePrefix + return nil + } +} + func WithNamespace(namespace string) GenerateOpt { return func(c *GenerateConfig) error { if !namespaceRe.MatchString(namespace) { @@ -318,6 +329,10 @@ func (b backend) generate(ctx context.Context, cfg GenerateConfig) error { if err := b.moduleIndexGenerator.GenerateNamespace(ctx, cfg.Namespace, moduleindex.WithForce(cfg.ForceRegenerate)); err != nil { return fmt.Errorf("failed to generate modules (%w)", err) } + } else if cfg.NamespacePrefix != "" { + if err := b.moduleIndexGenerator.GenerateNamespacePrefix(ctx, cfg.NamespacePrefix, moduleindex.WithForce(cfg.ForceRegenerate)); err != nil { + return fmt.Errorf("failed to generate modules (%w)", err) + } } else { if err := b.moduleIndexGenerator.Generate(ctx, moduleindex.WithForce(cfg.ForceRegenerate)); err != nil { return fmt.Errorf("failed to generate modules (%w)", err) @@ -333,6 +348,10 @@ func (b backend) generate(ctx context.Context, cfg GenerateConfig) error { if err := b.providerIndexGenerator.GenerateNamespace(ctx, cfg.Namespace, providerindex.WithForce(cfg.ForceRegenerate)); err != nil { return fmt.Errorf("failed to index providers (%w)", err) } + } else if cfg.NamespacePrefix != "" { + if err := b.providerIndexGenerator.GenerateNamespacePrefix(ctx, cfg.NamespacePrefix, providerindex.WithForce(cfg.ForceRegenerate)); err != nil { + return fmt.Errorf("failed to index providers (%w)", err) + } } else { if err := b.providerIndexGenerator.Generate(ctx, providerindex.WithForce(cfg.ForceRegenerate)); err != nil { return fmt.Errorf("failed to index providers (%w)", err) diff --git a/backend/internal/moduleindex/generator.go b/backend/internal/moduleindex/generator.go index 027b170c..2f854414 100644 --- a/backend/internal/moduleindex/generator.go +++ b/backend/internal/moduleindex/generator.go @@ -49,6 +49,9 @@ type Generator interface { Generate(ctx context.Context, opts ...Opts) error // GenerateNamespace generates module index files incrementally for one namespace and removes items no longer in the // registry. + GenerateNamespacePrefix(ctx context.Context, namespacePrefix string, opts ...Opts) error + // GenerateNamespace generates module index files incrementally for one namespace and removes items no longer in the + // registry. GenerateNamespace(ctx context.Context, namespace string, opts ...Opts) error // GenerateNamespaceAndName generates module index files incrementally for one namespace and removes items no longer in the // registry. @@ -127,6 +130,26 @@ func (g generator) GenerateNamespaceAndName(ctx context.Context, namespace strin }, opts) } +func (g generator) GenerateNamespacePrefix(ctx context.Context, namespacePrefix string, opts ...Opts) error { + namespacePrefix = module.NormalizeNamespace(namespacePrefix) + g.log.Info(ctx, "Listing all modules...") + moduleListFull, err := g.metadataAPI.ListModules(ctx) + if err != nil { + return err + } + + var moduleList []module.Addr + for _, module := range moduleListFull { + if strings.HasPrefix(module.Namespace, namespacePrefix) { + moduleList = append(moduleList, module) + } + } + + return g.generate(ctx, moduleList, func(moduleAddr ModuleAddr) bool { + return !strings.HasPrefix(moduleAddr.Namespace, namespacePrefix) + }, opts) +} + func (g generator) GenerateNamespace(ctx context.Context, namespace string, opts ...Opts) error { namespace = module.NormalizeNamespace(namespace) g.log.Info(ctx, "Listing all modules...") diff --git a/backend/internal/providerindex/generator.go b/backend/internal/providerindex/generator.go index c7bf0cdf..0fb821f3 100644 --- a/backend/internal/providerindex/generator.go +++ b/backend/internal/providerindex/generator.go @@ -32,6 +32,10 @@ type DocumentationGenerator interface { // registry. GenerateNamespace(ctx context.Context, namespace string, opts ...Opts) error + // GenerateNamespacePrefix generates module index files incrementally for one namespace and removes items no longer in the + // registry. + GenerateNamespacePrefix(ctx context.Context, namespace string, opts ...Opts) error + // GenerateSingleProvider generates module index files for a single provider only. GenerateSingleProvider(ctx context.Context, addr provider.Addr, opts ...Opts) error } @@ -117,9 +121,29 @@ func (d *documentationGenerator) Generate(ctx context.Context, opts ...Opts) err func (d *documentationGenerator) GenerateNamespace(ctx context.Context, namespace string, opts ...Opts) error { d.log.Info(ctx, "Listing all providers in namespace %s...", namespace) providerList, err := d.metadataAPI.ListProvidersByNamespace(ctx, namespace, true) + + d.log.Info(ctx, "Loaded %d providers", len(providerList)) + + err = d.scrape(ctx, providerList, opts) if err != nil { return err } + + return nil +} + +func (d *documentationGenerator) GenerateNamespacePrefix(ctx context.Context, namespacePrefix string, opts ...Opts) error { + d.log.Info(ctx, "Listing all providers in namespace prefix %s...", namespacePrefix) + providerListFull, err := d.metadataAPI.ListProviders(ctx, true) + if err != nil { + return err + } + var providerList []provider.Addr + for _, provider := range providerListFull { + if strings.HasPrefix(provider.Namespace, namespacePrefix) { + providerList = append(providerList, provider) + } + } d.log.Info(ctx, "Loaded %d providers", len(providerList)) err = d.scrape(ctx, providerList, opts) From ead1a614de0e1ae96e3b8b6f8070884bfb6a2ff0 Mon Sep 17 00:00:00 2001 From: AbstractionFactory <179820029+abstractionfactory@users.noreply.github.com> Date: Fri, 8 Nov 2024 16:36:54 +0100 Subject: [PATCH 2/3] Minor usability improvements Signed-off-by: AbstractionFactory <179820029+abstractionfactory@users.noreply.github.com> --- backend/cmd/generate/main.go | 2 +- backend/internal/backend.go | 6 ++++++ backend/internal/moduleindex/generator.go | 4 ++-- backend/internal/providerindex/generator.go | 17 ++++++++--------- 4 files changed, 17 insertions(+), 12 deletions(-) diff --git a/backend/cmd/generate/main.go b/backend/cmd/generate/main.go index 380728c4..61d10d71 100644 --- a/backend/cmd/generate/main.go +++ b/backend/cmd/generate/main.go @@ -56,7 +56,7 @@ func main() { flag.StringVar(&licensesFile, "licenses-file", licensesFile, "JSON file containing a list of approved licenses to include when indexing. (required)") flag.BoolVar(&skipUpdateProviders, "skip-update-providers", skipUpdateProviders, "Skip updating provider indexes.") flag.BoolVar(&skipUpdateModules, "skip-update-modules", skipUpdateModules, "Skip updating module indexes.") - flag.StringVar(&namespacePrefix, "namespace-prefix", namespace, "Limit updates to a namespace.") + flag.StringVar(&namespacePrefix, "namespace-prefix", namespace, "Limit updates to a namespace prefix.") flag.StringVar(&namespace, "namespace", namespace, "Limit updates to a namespace.") flag.StringVar(&name, "name", name, "Limit updates to a name. Only works in conjunction with -namespace. For providers, this will result in a single provider getting updated. For modules, this will update all target systems under a name.") flag.StringVar(&targetSystem, "target-system", targetSystem, "Limit updates to a target system for module updates only. Only works in conjunction with -namespace and -name.") diff --git a/backend/internal/backend.go b/backend/internal/backend.go index 7582c7a7..7b51b369 100644 --- a/backend/internal/backend.go +++ b/backend/internal/backend.go @@ -209,6 +209,9 @@ var targetSystemRe = regexp.MustCompile("^[a-zA-Z0-9._-]*$") func WithNamespacePrefix(namespacePrefix string) GenerateOpt { return func(c *GenerateConfig) error { + if c.Namespace != "" { + return fmt.Errorf("filtering for both namespace and namespace prefix is not supported") + } if !namespaceRe.MatchString(namespacePrefix) { return fmt.Errorf("invalid namespace: %s", namespaceRe) } @@ -219,6 +222,9 @@ func WithNamespacePrefix(namespacePrefix string) GenerateOpt { func WithNamespace(namespace string) GenerateOpt { return func(c *GenerateConfig) error { + if c.NamespacePrefix != "" { + return fmt.Errorf("filtering for both namespace and namespace prefix is not supported") + } if !namespaceRe.MatchString(namespace) { return fmt.Errorf("invalid namespace: %s", namespaceRe) } diff --git a/backend/internal/moduleindex/generator.go b/backend/internal/moduleindex/generator.go index 2f854414..19cd822f 100644 --- a/backend/internal/moduleindex/generator.go +++ b/backend/internal/moduleindex/generator.go @@ -47,8 +47,8 @@ const indexPrefix = "modules" type Generator interface { // Generate generates all module index files incrementally and removes items no longer in the registry. Generate(ctx context.Context, opts ...Opts) error - // GenerateNamespace generates module index files incrementally for one namespace and removes items no longer in the - // registry. + // GenerateNamespacePrefix generates module index files incrementally for all namespaces matching a given + // prefix and removes modules from the index that no longer exist. GenerateNamespacePrefix(ctx context.Context, namespacePrefix string, opts ...Opts) error // GenerateNamespace generates module index files incrementally for one namespace and removes items no longer in the // registry. diff --git a/backend/internal/providerindex/generator.go b/backend/internal/providerindex/generator.go index 0fb821f3..c40f5de8 100644 --- a/backend/internal/providerindex/generator.go +++ b/backend/internal/providerindex/generator.go @@ -28,13 +28,12 @@ type DocumentationGenerator interface { // Generate generates all module index files incrementally and removes items no longer in the registry. Generate(ctx context.Context, opts ...Opts) error - // GenerateNamespace generates module index files incrementally for one namespace and removes items no longer in the - // registry. + // GenerateNamespace generates provider index files incrementally for one namespace. GenerateNamespace(ctx context.Context, namespace string, opts ...Opts) error - // GenerateNamespacePrefix generates module index files incrementally for one namespace and removes items no longer in the - // registry. - GenerateNamespacePrefix(ctx context.Context, namespace string, opts ...Opts) error + // GenerateNamespacePrefix generates provider index files incrementally for multiple namespaces matching the given + // prefix. + GenerateNamespacePrefix(ctx context.Context, namespacePrefix string, opts ...Opts) error // GenerateSingleProvider generates module index files for a single provider only. GenerateSingleProvider(ctx context.Context, addr provider.Addr, opts ...Opts) error @@ -133,15 +132,15 @@ func (d *documentationGenerator) GenerateNamespace(ctx context.Context, namespac } func (d *documentationGenerator) GenerateNamespacePrefix(ctx context.Context, namespacePrefix string, opts ...Opts) error { - d.log.Info(ctx, "Listing all providers in namespace prefix %s...", namespacePrefix) + d.log.Info(ctx, "Listing all providers with the namespace prefix %s...", namespacePrefix) providerListFull, err := d.metadataAPI.ListProviders(ctx, true) if err != nil { return err } var providerList []provider.Addr - for _, provider := range providerListFull { - if strings.HasPrefix(provider.Namespace, namespacePrefix) { - providerList = append(providerList, provider) + for _, providerAddr := range providerListFull { + if strings.HasPrefix(providerAddr.Namespace, namespacePrefix) { + providerList = append(providerList, providerAddr) } } d.log.Info(ctx, "Loaded %d providers", len(providerList)) From bae5c6c8ff2a866f005a24e88dfd3562694f6781 Mon Sep 17 00:00:00 2001 From: AbstractionFactory <179820029+abstractionfactory@users.noreply.github.com> Date: Fri, 8 Nov 2024 16:38:34 +0100 Subject: [PATCH 3/3] Fixing flag bug Signed-off-by: AbstractionFactory <179820029+abstractionfactory@users.noreply.github.com> --- backend/internal/backend.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/internal/backend.go b/backend/internal/backend.go index 7b51b369..0bb9f952 100644 --- a/backend/internal/backend.go +++ b/backend/internal/backend.go @@ -209,7 +209,7 @@ var targetSystemRe = regexp.MustCompile("^[a-zA-Z0-9._-]*$") func WithNamespacePrefix(namespacePrefix string) GenerateOpt { return func(c *GenerateConfig) error { - if c.Namespace != "" { + if namespacePrefix != "" && c.Namespace != "" { return fmt.Errorf("filtering for both namespace and namespace prefix is not supported") } if !namespaceRe.MatchString(namespacePrefix) { @@ -222,7 +222,7 @@ func WithNamespacePrefix(namespacePrefix string) GenerateOpt { func WithNamespace(namespace string) GenerateOpt { return func(c *GenerateConfig) error { - if c.NamespacePrefix != "" { + if namespace != "" && c.NamespacePrefix != "" { return fmt.Errorf("filtering for both namespace and namespace prefix is not supported") } if !namespaceRe.MatchString(namespace) {