Fixing unnecessarily uploading unchanged files (#280)

* Fixing unnecessarily uploading unchanged files

Signed-off-by: AbstractionFactory <[email protected]>

* Tests for skipping uploads

Signed-off-by: AbstractionFactory <[email protected]>

---------

Signed-off-by: AbstractionFactory <[email protected]>

Author: AbstractionFactory

backend/internal/indexstorage/bufferedstorage/buffered.go

@@ -1,6 +1,7 @@
 package bufferedstorage
 
 import (
+	"bytes"
 	"context"
 	"fmt"
 	"io/fs"
@@ -129,6 +130,17 @@ func (b *buffered) WriteFile(ctx context.Context, filePath indexstorage.Path, co
 	}
 
 	localPath := path.Join(b.localDir, string(filePath))
+
+	status := b.index.FileStatus(ctx, filePath)
+	switch status {
+	case fileStatusPresent:
+		localContents, err := os.ReadFile(localPath)
+		if err == nil && bytes.Equal(localContents, contents) {
+			// Do not re-upload a file that has not changed.
+			return nil
+		}
+	}
+
 	if err := os.MkdirAll(path.Dir(localPath), 0755); err != nil {
 		return fmt.Errorf("failed to create directory for %s (%w)", localPath, err)
 	}

backend/internal/indexstorage/bufferedstorage/buffered_test.go

@@ -1,6 +1,7 @@
 package bufferedstorage_test
 
 import (
+	"bytes"
 	"context"
 	"os"
 	"path"
@@ -129,6 +130,41 @@ func TestSubdir(t *testing.T) {
 	assertFileExists(t, ctx, buffer, "test/test.txt")
 }
 
+func TestSameContent(t *testing.T) {
+	const testContent = "Hello world!"
+	backingDir := t.TempDir()
+	backingStorage := tofutestutils.Must2(filesystemstorage.New(backingDir))
+
+	ctx := tofutestutils.Context(t)
+
+	tofutestutils.Must(backingStorage.WriteFile(ctx, "test.txt", []byte(testContent)))
+
+	localDir := t.TempDir()
+	buffer := tofutestutils.Must2(bufferedstorage.New(logger.NewTestLogger(t), localDir, backingStorage, 25))
+	if _, err := os.Stat(path.Join(localDir, "test.txt")); err == nil {
+		t.Fatalf("Test file was present before fetched.")
+	}
+	readData, err := buffer.ReadFile(ctx, "test.txt")
+	if err != nil {
+		t.Fatalf("Could not read test file: %v", err)
+	}
+	if !bytes.Equal(readData, []byte(testContent)) {
+		t.Fatalf("The test file has incorrect contents: %s", readData)
+	}
+
+	if _, err = os.Stat(path.Join(localDir, "test.txt")); err != nil {
+		t.Fatalf("Test file was not present before after fetching: %v", err)
+	}
+	tofutestutils.Must(buffer.WriteFile(ctx, "test.txt", []byte(testContent)))
+	if buffer.UncommittedFiles() != 0 {
+		t.Fatalf("Uncommitted files despite same content.")
+	}
+	tofutestutils.Must(buffer.WriteFile(ctx, "test.txt", []byte(testContent+"!")))
+	if buffer.UncommittedFiles() != 1 {
+		t.Fatalf("No or multiple uncommitted files despite different content!")
+	}
+}
+
 func assertFileDoesNotExist(t *testing.T, ctx context.Context, storage indexstorage.API, file indexstorage.Path) {
 	t.Helper()
 	_, err := storage.ReadFile(ctx, file)