Skip to content

Commit

Permalink
Merge branch 'main' into registry-docs
Browse files Browse the repository at this point in the history 
Signed-off-by: AbstractionFactory <[email protected]>
  • Loading branch information
@abstractionfactory
abstractionfactory authored Oct 11, 2024
2 parents b8c6fb9 + c1d6aaf commit 58d5891
Show file tree
Hide file tree
Showing 6 changed files with 222 additions and 191 deletions.
2 changes: 1 addition & 1 deletion backend/go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ require (
github.com/go-enry/go-license-detector/v4 v4.3.1
github.com/lib/pq v1.10.9
github.com/mitchellh/go-spdx v0.1.0
github.com/opentofu/libregistry v0.0.0-20240911130704-36ad231a7168
github.com/opentofu/libregistry v0.0.0-20241009085434-a0be829febc1
github.com/opentofu/tofudl v0.0.0-20240730151408-3bd8529dae09
github.com/opentofu/tofutestutils v0.0.0-20240821111804-5fcfb797e0a7
golang.org/x/sync v0.8.0
Expand Down
2 changes: 2 additions & 0 deletions backend/go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -235,6 +235,8 @@ github.com/opentofu/libregistry v0.0.0-20240911094842-bfb6ed3cfb33 h1:NWhgKmeTGQ
github.com/opentofu/libregistry v0.0.0-20240911094842-bfb6ed3cfb33/go.mod h1:irS/XyfQZdwi3Ggm105FfDyj3d9deFAi5GraORIeMsI=
github.com/opentofu/libregistry v0.0.0-20240911130704-36ad231a7168 h1:2bwgWOJ90nzGuDeQFY6EBA/T6R3C3rnXYw7VFMURQnM=
github.com/opentofu/libregistry v0.0.0-20240911130704-36ad231a7168/go.mod h1:irS/XyfQZdwi3Ggm105FfDyj3d9deFAi5GraORIeMsI=
github.com/opentofu/libregistry v0.0.0-20241009085434-a0be829febc1 h1:1wc2S/a7GvWGI8uQ+njrTdHNsnPRC76M+AXBqcIrdEY=
github.com/opentofu/libregistry v0.0.0-20241009085434-a0be829febc1/go.mod h1:irS/XyfQZdwi3Ggm105FfDyj3d9deFAi5GraORIeMsI=
github.com/opentofu/registry-address v0.0.0-20230922120653-901b9ae4061a h1:NyM/PPbc+kxxv2d4OKfE32C5fLtVTLceyg4YKKCYO9Y=
github.com/opentofu/registry-address v0.0.0-20230922120653-901b9ae4061a/go.mod h1:HzQhpVo/NJnGmN+7FPECCVCA5ijU7AUcvf39enBKYOc=
github.com/opentofu/tofudl v0.0.0-20240730151408-3bd8529dae09 h1:bA3Dy3Be9o896wdcQhK0Ky2Eco0ZwCWZgt73dSaYuDI=
Expand Down
60 changes: 42 additions & 18 deletions backend/internal/moduleindex/generator.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import (
"os"
"path"
"slices"
"strings"
"sync"
"text/template"
"time"
Expand Down Expand Up @@ -348,7 +349,12 @@ func (g generator) generate(ctx context.Context, moduleList []module.Addr, block
}
for _, m := range modules.Modules {
if err := m.Validate(); err != nil {
return fmt.Errorf("invalid module (%w)", err)
// We are ignoring invalid version numbers here because the dataset contains them, but when a module is
// refreshed, it should be excluded above.
var invalidVersionNumber *module.InvalidVersionNumber
if !errors.As(err, &invalidVersionNumber) {
return fmt.Errorf("invalid module (%w)", err)
}
}
}
marshalled, err := json.Marshal(modules)
Expand Down Expand Up @@ -433,7 +439,7 @@ func (g generator) generateModuleVersion(ctx context.Context, moduleAddr ModuleA
}

g.log.Info(ctx, "Updating module details for %s version %s...", moduleAddr, ver.ID)
if err := g.refreshModuleDetails(ctx, moduleAddr, ver, &result.Details, workingCopy, licenseOK, entry.IsBlocked, entry.BlockedReason, ""); err != nil {
if err := g.refreshModuleDetails(ctx, moduleAddr, ver, &result.Details, workingCopy, licenseOK, entry.IsBlocked, entry.BlockedReason, "", ""); err != nil {
return fmt.Errorf("failed to extract module defaults for %s version %s (%w)", moduleAddr, ver.ID, err)
}

Expand Down Expand Up @@ -471,9 +477,9 @@ func (g generator) refreshLicense(ctx context.Context, moduleAddr ModuleAddr, mo
return err
}

func (g generator) refreshModuleDetails(ctx context.Context, moduleAddr ModuleAddr, ver ModuleVersionDescriptor, d *Details, workingCopy vcs.WorkingCopy, licenseOK bool, blocked bool, blockedReason string, prefix string) error {
func (g generator) refreshModuleDetails(ctx context.Context, moduleAddr ModuleAddr, ver ModuleVersionDescriptor, d *Details, workingCopy vcs.WorkingCopy, licenseOK bool, blocked bool, blockedReason string, sourcePrefix string, dstPrefix string) error {
var err error
if d.Readme, d.EditLink, err = g.extractReadme(ctx, moduleAddr, ver, workingCopy, licenseOK, blocked, blockedReason, prefix); err != nil {
if d.Readme, d.EditLink, err = g.extractReadme(ctx, moduleAddr, ver, workingCopy, licenseOK, blocked, blockedReason, sourcePrefix, dstPrefix); err != nil {
return err
}

Expand All @@ -484,17 +490,17 @@ func (g generator) refreshModuleDetails(ctx context.Context, moduleAddr ModuleAd
return nil
}

dir := path.Join(rawDirectory, prefix)
dir := path.Join(rawDirectory, sourcePrefix)
if err := g.extractModuleSchema(ctx, dir, d, licenseOK, blocked); err != nil {
return err
}

return nil
}

func (g generator) refreshExampleDetails(ctx context.Context, moduleAddr ModuleAddr, ver ModuleVersionDescriptor, e *Example, workingCopy vcs.WorkingCopy, licenseOK bool, blocked bool, blockedReason string, prefix string) error {
func (g generator) refreshExampleDetails(ctx context.Context, moduleAddr ModuleAddr, ver ModuleVersionDescriptor, e *Example, workingCopy vcs.WorkingCopy, licenseOK bool, blocked bool, blockedReason string, sourcePrefix string, dstPrefix string) error {
var err error
if e.Readme, e.EditLink, err = g.extractReadme(ctx, moduleAddr, ver, workingCopy, licenseOK, blocked, blockedReason, prefix); err != nil {
if e.Readme, e.EditLink, err = g.extractReadme(ctx, moduleAddr, ver, workingCopy, licenseOK, blocked, blockedReason, sourcePrefix, dstPrefix); err != nil {
return err
}

Expand All @@ -505,18 +511,18 @@ func (g generator) refreshExampleDetails(ctx context.Context, moduleAddr ModuleA
return nil
}

dir := path.Join(rawDirectory, prefix)
dir := path.Join(rawDirectory, sourcePrefix)
if err := g.extractExampleSchema(ctx, dir, e, licenseOK, blocked); err != nil {
return err
}

return nil
}

func (g generator) extractReadme(ctx context.Context, moduleAddr ModuleAddr, ver ModuleVersionDescriptor, workingCopy vcs.WorkingCopy, licenseOK bool, blocked bool, blockedReason string, prefix string) (bool, string, error) {
func (g generator) extractReadme(ctx context.Context, moduleAddr ModuleAddr, ver ModuleVersionDescriptor, workingCopy vcs.WorkingCopy, licenseOK bool, blocked bool, blockedReason string, sourcePrefix string, dstPrefix string) (bool, string, error) {
hasReadme := false
var readme []byte
sourcePath := path.Join(prefix, "README.md")
sourcePath := path.Join(sourcePrefix, "README.md")
fh, err := workingCopy.Open(sourcePath)
if err != nil {
if os.IsNotExist(err) {
Expand Down Expand Up @@ -544,8 +550,8 @@ func (g generator) extractReadme(ctx context.Context, moduleAddr ModuleAddr, ver
_ = fh.Close()
}
readmePath := path.Join(moduleAddr.Namespace, moduleAddr.Name, moduleAddr.TargetSystem, string(ver.ID), "README.md")
if prefix != "" {
readmePath = path.Join(moduleAddr.Namespace, moduleAddr.Name, moduleAddr.TargetSystem, string(ver.ID), prefix, "README.md")
if dstPrefix != "" {
readmePath = path.Join(moduleAddr.Namespace, moduleAddr.Name, moduleAddr.TargetSystem, string(ver.ID), dstPrefix, "README.md")
}
if err := g.storage.WriteFile(ctx, indexstorage.Path(readmePath), readme); err != nil {
return hasReadme, "", fmt.Errorf("failed to write README.md at %s (%w)", readmePath, err)
Expand Down Expand Up @@ -593,9 +599,15 @@ func (g generator) extractSubmodules(ctx context.Context, addr ModuleAddr, ver M
Resources: []Resource{},
},
}
submodulePrefix := path.Join(directoryPrefix, name)
if err := g.refreshModuleDetails(ctx, addr, ver, &submodule.Details, workingCopy, licenseOK, blocked, blockedReason, submodulePrefix); err != nil {
return fmt.Errorf("failed to refresh details for submodule %s (%w)", submodulePrefix, err)
sourcePrefix := path.Join(directoryPrefix, name)
dstPrefix, err := g.sanitizePath(directoryPrefix, name)
if err != nil {
// Invalid path, ignore.
g.log.Warn(ctx, "Failed to index %s version %s submodule %s (%v)", addr, ver.ID, dstPrefix, err)
continue
}
if err := g.refreshModuleDetails(ctx, addr, ver, &submodule.Details, workingCopy, licenseOK, blocked, blockedReason, sourcePrefix, dstPrefix); err != nil {
return fmt.Errorf("failed to refresh details for submodule %s (%w)", sourcePrefix, err)
}

m.Submodules[name] = submodule
Expand Down Expand Up @@ -630,7 +642,13 @@ func (g generator) extractExamples(ctx context.Context, moduleAddr ModuleAddr, v
Outputs: map[string]Output{},
},
}
examplePrefix := path.Join(directoryPrefix, name)
srcPrefix := path.Join(directoryPrefix, name)
dstPrefix, err := g.sanitizePath(directoryPrefix, name)
if err != nil {
// Invalid path, don't include in the index
g.log.Warn(ctx, "Failed to index %s version %s example %s (%v)", moduleAddr, ver.ID, dstPrefix, err)
continue
}
if err := g.refreshExampleDetails(
ctx,
moduleAddr,
Expand All @@ -640,9 +658,10 @@ func (g generator) extractExamples(ctx context.Context, moduleAddr ModuleAddr, v
licenseOK,
blocked,
blockedReason,
examplePrefix,
srcPrefix,
dstPrefix,
); err != nil {
return fmt.Errorf("failed to refresh details for example %s (%w)", examplePrefix, err)
return fmt.Errorf("failed to refresh details for example %s (%w)", srcPrefix, err)
}

m.Examples[name] = example
Expand Down Expand Up @@ -806,3 +825,8 @@ func (g generator) fetchRepoInfo(ctx context.Context, entry *Module) {
entry.UpstreamPopularity = upstreamRepoInfo.Popularity
entry.UpstreamForkCount = upstreamRepoInfo.ForkCount
}

func (g generator) sanitizePath(prefix string, name string) (string, error) {
name = strings.ReplaceAll(name, " ", "")
return path.Join(prefix, name), indexstorage.Path(name).Validate()
}
2 changes: 1 addition & 1 deletion frontend/package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@
"unified": "^11.0.5",
"unplugin-macros": "^0.13.3",
"vfile-matter": "^5.0.0",
"vite": "^5.4.2",
"vite": "^5.4.6",
"vite-tsconfig-paths": "^5.0.1"
},
"devDependencies": {
Expand Down
Loading

0 comments on commit 58d5891

Please sign in to comment.