Fixes #285: Unnecessary file upload (#288)

* Fixes #285: Unnecessary file upload

This change attempts to fix #285 by only uploading the module/provider index when it has actually changed

Signed-off-by: AbstractionFactory <[email protected]>

* Review fix

Signed-off-by: AbstractionFactory <[email protected]>

* Added comments

Signed-off-by: AbstractionFactory <[email protected]>

---------

Signed-off-by: AbstractionFactory <[email protected]>

Author: AbstractionFactory

backend/internal/moduleindex/generator.go

@@ -228,7 +228,13 @@ func (g generator) generate(ctx context.Context, moduleList []module.Addr, block
 			blocked, blockedReason := g.blocklist.IsModuleBlocked(moduleAddr.Addr)
 
 			moduleIndexPath := path.Join(moduleAddr.Namespace, moduleAddr.Name, moduleAddr.TargetSystem, "index.json")
+			// We are fetching the module entry from the megaindex and storing it
+			// further down below as a separate index file so the frontend has an easier time
+			// fetching it.
 			entry := modules.GetModule(moduleAddr.Addr)
+			// originalEntry serves the purpose of being an original copy to compare to
+			// so we don't write the index if it hasn't actually been modified to save costs.
+			var originalEntry *Module
 			needsAdd := false
 			if entry == nil {
 				entry = &Module{
@@ -239,6 +245,8 @@ func (g generator) generate(ctx context.Context, moduleList []module.Addr, block
 					BlockedReason: blockedReason,
 				}
 				needsAdd = true
+			} else {
+				originalEntry = entry.DeepCopy()
 			}
 
 			if entry.IsBlocked != blocked {
@@ -361,12 +369,18 @@ func (g generator) generate(ctx context.Context, moduleList []module.Addr, block
 					modulesToAdd = append(modulesToAdd, entry)
 					lock.Unlock()
 				}
-				versionListing, err := json.Marshal(entry)
-				if err != nil {
-					return fmt.Errorf("failed to marshal module index for %s (%w)", entry.Addr, err)
-				}
-				if err := g.storage.WriteFile(ctx, indexstorage.Path(moduleIndexPath), versionListing); err != nil {
-					return fmt.Errorf("failed to write the module index for %s (%w)", entry.Addr, err)
+				// Here we compare the module entry to its original copy to make sure
+				// we are only writing this index if needed. This is needed because writes
+				// on R2 cost money, whereas reads don't and updating all the provider and
+				// module indexes on every run costs ~300$ per month.
+				if originalEntry == nil || !originalEntry.Equals(entry) {
+					versionListing, err := json.Marshal(entry)
+					if err != nil {
+						return fmt.Errorf("failed to marshal module index for %s (%w)", entry.Addr, err)
+					}
+					if err := g.storage.WriteFile(ctx, indexstorage.Path(moduleIndexPath), versionListing); err != nil {
+						return fmt.Errorf("failed to write the module index for %s (%w)", entry.Addr, err)
+					}
 				}
 			}
 

backend/internal/moduleindex/module.go

@@ -10,6 +10,8 @@ import (
 
 // swagger:model Module
 type Module struct {
+	// If you add a field here, update Equals() and DeepCopy() below.
+
 	// required: true
 	Addr ModuleAddr `json:"addr"`
 	// required: true
@@ -36,6 +38,43 @@ type Module struct {
 	UpstreamPopularity int `json:"upstream_popularity"`
 	// UpstreamForkCount contains the number of forks of the upstream repository.
 	UpstreamForkCount int `json:"upstream_fork_count"`
+
+	// If you add a field here, update Equals() and DeepCopy() below.
+}
+
+// Equals compares every parameter of the two modules and returns true if both are equal on a deep comparison.
+func (m *Module) Equals(other *Module) bool {
+	if m == other {
+		return true
+	}
+	if m == nil || other == nil {
+		return false
+	}
+	return m.Addr.Equals(other.Addr.Addr) && m.Description == other.Description &&
+		slices.Equal(m.Versions, other.Versions) && m.IsBlocked == other.IsBlocked &&
+		m.BlockedReason == other.BlockedReason && m.Popularity == other.Popularity && m.ForkCount == other.ForkCount &&
+		m.ForkOfLink == other.ForkOfLink && m.ForkOf.Equals(other.ForkOf.Addr) &&
+		m.UpstreamPopularity == other.UpstreamPopularity && m.UpstreamForkCount == other.UpstreamForkCount
+}
+
+// DeepCopy creates a deep copy of the module, ensuring that all new data structures are independent.
+func (m *Module) DeepCopy() *Module {
+	versions := make([]ModuleVersionDescriptor, len(m.Versions))
+	copy(versions, m.Versions)
+
+	return &Module{
+		Addr:               m.Addr,
+		Description:        m.Description,
+		Versions:           versions,
+		IsBlocked:          m.IsBlocked,
+		BlockedReason:      m.BlockedReason,
+		Popularity:         m.Popularity,
+		ForkCount:          m.ForkCount,
+		ForkOfLink:         m.ForkOfLink,
+		ForkOf:             m.ForkOf,
+		UpstreamPopularity: m.UpstreamPopularity,
+		UpstreamForkCount:  m.UpstreamForkCount,
+	}
 }
 
 func (m *Module) Validate() error {

backend/internal/providerindex/generator.go

@@ -195,7 +195,13 @@ func (d *documentationGenerator) scrape(ctx context.Context, providers []provide
 		eg.Go(func() error {
 			blocked, blockedReason := d.blocklist.IsProviderBlocked(addr)
 
+			// We are fetching the provider entry from the megaindex and storing it
+			// further down below as a separate index file so the frontend has an easier time
+			// fetching it.
 			providerEntry := existingProviders.GetProvider(addr)
+			// originalProviderEntry serves the purpose of being an original copy to compare to
+			// so we don't write the index if it hasn't actually been modified to save costs.
+			var originalProviderEntry *providertypes.Provider
 			needsAdd := false
 			if providerEntry == nil {
 				providerEntry = &providertypes.Provider{
@@ -207,6 +213,8 @@ func (d *documentationGenerator) scrape(ctx context.Context, providers []provide
 					BlockedReason: blockedReason,
 				}
 				needsAdd = true
+			} else {
+				originalProviderEntry = providerEntry.DeepCopy()
 			}
 
 			// scrape the docs into their own directory
@@ -223,8 +231,14 @@ func (d *documentationGenerator) scrape(ctx context.Context, providers []provide
 				return err
 			}
 
-			if err := d.destination.StoreProvider(ctx, *providerEntry); err != nil {
-				return fmt.Errorf("failed to store provider %s (%w)", addr, err)
+			// Here we compare the provider entry to its original copy to make sure
+			// we are only writing this index if needed. This is needed because writes
+			// on R2 cost money, whereas reads don't and updating all the provider and
+			// module indexes on every run costs ~300$ per month.
+			if originalProviderEntry == nil || !originalProviderEntry.Equals(providerEntry) {
+				if err := d.destination.StoreProvider(ctx, *providerEntry); err != nil {
+					return fmt.Errorf("failed to store provider %s (%w)", addr, err)
+				}
 			}
 
 			if needsAdd {

backend/internal/providerindex/providertypes/provider.go

@@ -10,6 +10,8 @@ import (
 //
 // swagger:model Provider
 type Provider struct {
+	// If you add something here, don't forget to update the Equals() and DeepCopy() functions below.
+
 	// Addr holds the address of a provider. It can be split by / to obtain a namespace and name.
 	//
 	// required: true
@@ -61,6 +63,73 @@ type Provider struct {
 	IsBlocked bool `json:"is_blocked"`
 	// required: false
 	BlockedReason string `json:"blocked_reason,omitempty"`
+
+	// If you add something here, don't forget to update the Equals() and DeepCopy() functions below.
+}
+
+// Equals returns true if and only of all parameters of the two providers are equal (with a deep comparison).
+func (p *Provider) Equals(other *Provider) bool {
+	if p == other {
+		return true
+	}
+	if p == nil || other == nil {
+		return false
+	}
+	if len(p.ReverseAliases) != len(other.ReverseAliases) {
+		return false
+	} else {
+		for i := range len(p.ReverseAliases) {
+			if !p.ReverseAliases[i].Equals(other.ReverseAliases[i].Addr) {
+				return false
+			}
+		}
+	}
+	return p.Addr.Equals(other.Addr.Addr) && slices.Equal(p.Warnings, other.Warnings) && p.Link == other.Link &&
+		(p.CanonicalAddr == other.CanonicalAddr || p.CanonicalAddr.Equals(other.CanonicalAddr.Addr)) &&
+		p.Description == other.Description && p.Popularity == other.Popularity && p.ForkCount == other.ForkCount &&
+		p.ForkOfLink == other.ForkOfLink && p.ForkOf == other.ForkOf &&
+		p.UpstreamPopularity == other.UpstreamPopularity && p.UpstreamForkCount == other.UpstreamForkCount &&
+		slices.Equal(p.Versions, other.Versions) && p.BlockedReason == other.BlockedReason
+}
+
+// DeepCopy creates a deep copy of the Provider.
+func (p *Provider) DeepCopy() *Provider {
+	warnings := make([]string, len(p.Warnings))
+	copy(warnings, p.Warnings)
+
+	var canonicalAddr *ProviderAddr
+	if p.CanonicalAddr != nil {
+		canonicalAddr = &ProviderAddr{
+			Addr:      p.CanonicalAddr.Addr,
+			Display:   p.CanonicalAddr.Display,
+			Namespace: p.CanonicalAddr.Namespace,
+			Name:      p.CanonicalAddr.Name,
+		}
+	}
+
+	reverseAliases := make([]ProviderAddr, len(p.ReverseAliases))
+	copy(reverseAliases, p.ReverseAliases)
+
+	versions := make([]ProviderVersionDescriptor, len(p.Versions))
+	copy(versions, p.Versions)
+
+	return &Provider{
+		Addr:               p.Addr,
+		Warnings:           warnings,
+		Link:               p.Link,
+		CanonicalAddr:      canonicalAddr,
+		ReverseAliases:     reverseAliases,
+		Description:        p.Description,
+		Popularity:         p.Popularity,
+		ForkCount:          p.ForkCount,
+		ForkOfLink:         p.ForkOfLink,
+		ForkOf:             p.ForkOf,
+		UpstreamPopularity: p.UpstreamPopularity,
+		UpstreamForkCount:  p.UpstreamForkCount,
+		Versions:           versions,
+		IsBlocked:          p.IsBlocked,
+		BlockedReason:      p.BlockedReason,
+	}
 }
 
 func (p *Provider) Compare(other Provider) int {