Merge pull request #3 from wandb/git-background-tracking

Git background tracking

Author: shawnlewis

docs/design/git_integration_rework.md

@@ -0,0 +1,29 @@
+# Design Document: Git Integration Rework for Programmer
+
+## Overview
+This document outlines the proposed changes to the Git integration feature of the `programmer` tool. The goal is to allow real-time edits in the user's working state while maintaining a separate commit history in a `programmer-<session>` branch, without affecting the user's visible working state.
+
+## Objectives
+- Allow users to see changes in real-time using tools like VSCode's Git view.
+- Maintain a separate commit history in the background in a `programmer-<session>` branch.
+- Avoid using stash or switching the HEAD, ensuring the user's working state remains unchanged.
+
+## Proposed Solution
+To achieve the above objectives, we propose the following solution:
+
+### Session Branch Management
+1. **Initialization**: At the start of a session, initialize a `programmer-<session>` branch based on the current state of the user's branch.
+
+2. **Change Tracking**: Monitor file changes in the working directory to reflect them in the session branch, keeping the user's current branch and working directory unchanged.
+
+3. **Commit History**: Maintain a separate commit history in the `programmer-<session>` branch to allow for session management and review without interfering with the user's workflow.
+
+## Benefits
+- Users can continue using their preferred development tools and see live changes.
+- Maintains a clean separation of session history, enabling better session management and review.
+
+## Challenges
+- Requires careful handling of Git's internal mechanisms to ensure seamless integration.
+
+## Conclusion
+By leveraging a separate session branch, we can achieve seamless integration of the `programmer` tool with the user's workflow, providing real-time feedback and maintaining a comprehensive session history without disrupting the user's development environment.

programmer/environment.py

@@ -3,10 +3,8 @@
 from contextvars import ContextVar
 from contextlib import contextmanager
 
-
 from .git import GitRepo
 
-
 @dataclass
 class EnvironmentSnapshotKey:
     env_id: str
@@ -49,15 +47,17 @@ def start_session(self, session_id: str):
         self.original_git_ref = self.repo.get_current_head()
         self.programmer_branch = f"programmer-{session_id}"
         print("programmer_branch:", self.programmer_branch)
-        self.repo.checkout_new(self.programmer_branch)
+        # Create the programmer branch based on the current state
+        self.repo.create_branch(self.programmer_branch)
 
     def finish_session(self):
         if self.original_git_ref is None or self.programmer_branch is None:
             raise ValueError("Session not started")
-        self.repo.checkout_and_copy(self.original_git_ref)
+        # No need to checkout back as we never changed the branch
 
     def make_snapshot(self, message: str) -> EnvironmentSnapshotKey:
-        commit_hash = self.repo.add_all_and_commit(message)
+        # Commit directly to the programmer branch using new method
+        commit_hash = self.repo.commit_directly_to_branch(self.programmer_branch, message)
         return EnvironmentSnapshotKey(
             "git", {"origin": self.repo.get_origin_url(), "commit": commit_hash}
         )

programmer/git.py

@@ -1,8 +1,7 @@
-from git import Repo, InvalidGitRepositoryError, GitCommandError
 import os
+from git import Repo, InvalidGitRepositoryError, GitCommandError
 from typing import Optional
 import tempfile
-import shutil
 
 
 class GitRepo:
@@ -11,101 +10,70 @@ def __init__(self, repo: Repo):
 
     @classmethod
     def from_current_dir(cls) -> Optional["GitRepo"]:
-        """
-        Create a GitRepo instance from the current working directory or its parent directories.
-
-        Returns:
-            GitRepo instance if in a Git repository, None otherwise.
-        """
         try:
             repo = Repo(os.getcwd(), search_parent_directories=True)
             return cls(repo)
         except InvalidGitRepositoryError:
             return None
 
     def get_origin_url(self) -> Optional[str]:
-        """
-        Get the remote URL (e.g., GitHub URL) for this repository.
-
-        Returns:
-            The remote URL as a string if it exists, None otherwise.
-        """
         try:
             remote_url = self.repo.remotes.origin.url
             return remote_url if remote_url else None
         except AttributeError:
-            # No remote named 'origin' exists
             return None
 
-    def checkout_existing(self, ref: str) -> None:
-        self.repo.git.checkout(ref)
+    def create_branch(self, branch_name: str) -> None:
+        if branch_name not in self.repo.heads:
+            # Create a new branch from the current HEAD
+            self.repo.git.branch(branch_name, self.repo.head.commit.hexsha)
 
-    def checkout_new(self, branch_name: str) -> None:
-        self.repo.git.checkout(b=branch_name)
+    def commit_directly_to_branch(self, branch_name: str, message: str) -> str:
+        # Ensure the branch is initialized
+        self.create_branch(branch_name)
 
-    def get_current_head(self) -> str:
-        """
-        Get the current HEAD of the repository, which is either the branch name or the commit SHA.
+        # Use a temporary index file to stage files without affecting the actual index.
+        with tempfile.TemporaryDirectory() as temp_dir:
+            temp_index_file = os.path.join(temp_dir, "index")
+            env = os.environ.copy()
+            env["GIT_INDEX_FILE"] = temp_index_file
+
+            # Add all files from the working directory to the temporary index
+            self.repo.git.add(A=True, env=env)
+
+            # Determine the parent commit
+            parent_commit = self.repo.commit(branch_name)
+
+            # Check for changes between parent_commit and the temporary index
+            diff_output = self.repo.git.diff(parent_commit.hexsha, "--cached", env=env)
+
+            if not diff_output.strip():
+                # No changes to commit
+                return parent_commit.hexsha
+
+            # Write the tree from the temporary index
+            tree = self.repo.git.write_tree(env=env)
 
-        Returns:
-            The branch name if on a branch, otherwise the commit SHA.
-        """
+            # print(
+            #     f"Committing to branch {branch_name}, parent commit: {parent_commit.hexsha}"
+            # )
+
+            # Set author information using environment variables
+            env["GIT_AUTHOR_NAME"] = "programmer"
+            env["GIT_AUTHOR_EMAIL"] = "[email protected]"
+
+            # Use the Repo's git command interface to create a commit-tree
+            commit_hash = self.repo.git.commit_tree(
+                tree, "-p", parent_commit.hexsha, "-m", message, env=env
+            )
+
+            # Update the branch reference to point to the new commit
+            self.repo.git.update_ref(f"refs/heads/{branch_name}", commit_hash)
+
+            return commit_hash
+
+    def get_current_head(self) -> str:
         if self.repo.head.is_detached:
             return str(self.repo.head.commit.hexsha)
         else:
             return str(self.repo.active_branch.name)
-
-    def checkout_and_copy(self, to_ref: str) -> None:
-        current_branch = self.get_current_head()
-
-        with tempfile.TemporaryDirectory() as temp_dir:
-            # Get all changed files between current branch and to_ref
-            changed_files = self.repo.git.diff(
-                "--name-only", current_branch, to_ref
-            ).splitlines()
-
-            # Copy changed files to the temp directory
-            for file in changed_files:
-                src_path = os.path.join(self.repo.working_tree_dir, file)
-                dst_path = os.path.join(temp_dir, file)
-                os.makedirs(os.path.dirname(dst_path), exist_ok=True)
-                if os.path.exists(src_path):
-                    shutil.copy2(src_path, dst_path)
-
-            # Checkout the target branch
-            self.repo.git.checkout(to_ref)
-
-            # Copy files from temp directory to the working directory
-            for root, _, files in os.walk(temp_dir):
-                for file in files:
-                    src_path = os.path.join(root, file)
-                    rel_path = os.path.relpath(src_path, temp_dir)
-                    dst_path = os.path.join(self.repo.working_tree_dir, rel_path)
-                    os.makedirs(os.path.dirname(dst_path), exist_ok=True)
-                    shutil.copy2(src_path, dst_path)
-
-    def add_all_and_commit(self, message: str) -> str:
-        """
-        Add all files (including untracked ones) and commit them.
-
-        Args:
-            message: The commit message.
-
-        Returns:
-            The SHA of the new commit if changes were committed,
-            or the current HEAD's SHA if no changes were made.
-
-        Raises:
-            GitCommandError: If there are issues with Git operations.
-        """
-        # Add all files, including untracked ones
-        self.repo.git.add(A=True)
-
-        # Check if there are changes to commit
-        if self.repo.is_dirty(untracked_files=True):
-            # Commit the changes
-            commit = self.repo.index.commit(message)
-            return commit.hexsha
-        else:
-            # If no changes, return the current HEAD's SHA
-            return self.repo.head.commit.hexsha

programmer/programmer.py

@@ -99,7 +99,7 @@ def main():
     logging_mode = SettingsManager.get_setting("weave_logging")
     if logging_mode == "cloud":
         curdir = os.path.basename(os.path.abspath(os.curdir))
-        weave.init(f"programmerdev1-{curdir}")
+        weave.init(f"programmer-{curdir}")
     elif logging_mode == "local":
         weave.init_local_client()
 

programmer/tests/test_git_integration.py

@@ -0,0 +1,122 @@
+import pytest
+import tempfile
+import shutil
+import os
+from git import Repo
+from programmer.git import GitRepo
+
+
+@pytest.fixture
+def setup_repo():
+    # Create a temporary directory for the repository
+    test_dir = tempfile.mkdtemp()
+    repo = Repo.init(test_dir)
+    git_repo = GitRepo(repo)
+
+    # Set up user config for the test repo
+    with repo.config_writer() as config:
+        config.set_value("user", "name", "Test User")
+        config.set_value("user", "email", "[email protected]")
+
+    # Create an initial commit so HEAD exists
+    initial_file_path = os.path.join(test_dir, "initial.txt")
+    with open(initial_file_path, "w") as f:
+        f.write("Initial content")
+    repo.index.add([initial_file_path])
+    repo.index.commit("Initial commit")
+
+    # Create and checkout the main branch
+    main_branch = repo.create_head('main')
+    main_branch.checkout()
+
+    yield repo, git_repo, test_dir
+
+    # Remove the temporary directory after the test
+    shutil.rmtree(test_dir)
+
+
+def test_commit_directly_to_branch(setup_repo):
+    repo, git_repo, test_dir = setup_repo
+
+    # Create and commit a file in the main branch
+    file_path = os.path.join(test_dir, "test_file.py")
+    with open(file_path, "w") as f:
+        f.write("print('Hello, world!')\n")
+
+    repo.index.add([file_path])
+    repo.index.commit("Initial commit on main")
+
+    # Modify the file
+    with open(file_path, "a") as f:
+        f.write("print('Another line')\n")
+
+    # Commit changes to the programmer-<session> branch
+    session_branch_name = "programmer-session"
+    git_repo.create_branch(session_branch_name)
+    commit_message = "Commit from programmer session"
+    git_repo.commit_directly_to_branch(session_branch_name, commit_message)
+
+    # Verify the commit in the programmer-<session> branch
+    session_branch_commit = repo.commit(session_branch_name)
+    tree_files = session_branch_commit.tree.traverse()
+    file_names = [item.path for item in tree_files]
+
+    assert "test_file.py" in file_names
+
+    # Verify the content of the file in the commit
+    blob_data = (
+        session_branch_commit.tree["test_file.py"].data_stream.read().decode("utf-8")
+    )
+    assert "print('Another line')" in blob_data
+
+    # Verify that the main branch is unaffected
+    main_branch_commit = repo.commit("main")
+    assert main_branch_commit.hexsha != session_branch_commit.hexsha
+
+
+def test_no_empty_commit(setup_repo):
+    repo, git_repo, _ = setup_repo
+
+    # Create and checkout the programmer-<session> branch
+    session_branch_name = "programmer-session"
+    git_repo.create_branch(session_branch_name)
+
+    # Commit changes to the programmer-<session> branch
+    commit_message = "Commit from programmer session"
+    initial_commit_sha = repo.commit(session_branch_name).hexsha
+    commit_sha = git_repo.commit_directly_to_branch(session_branch_name, commit_message)
+
+    # Verify that the SHA returned is equal to the initial commit SHA
+    assert initial_commit_sha == commit_sha
+
+    # Verify no new commit is created
+    new_commit_sha = repo.commit(session_branch_name).hexsha
+    assert initial_commit_sha == new_commit_sha
+
+
+def test_multiple_commits_with_empty(setup_repo):
+    repo, git_repo, test_dir = setup_repo
+
+    # Create and checkout the programmer-<session> branch
+    session_branch_name = "programmer-session"
+    git_repo.create_branch(session_branch_name)
+    initial_commit_sha = repo.commit(session_branch_name).hexsha
+
+    # First commit (empty)
+    commit_message = "First empty commit"
+    commit_sha_1 = git_repo.commit_directly_to_branch(session_branch_name, commit_message)
+    assert commit_sha_1 == initial_commit_sha
+
+    # Second commit (non-empty)
+    file_path = os.path.join(test_dir, "test_file.py")
+    with open(file_path, "w") as f:
+        f.write("print('Hello, world!')\n")
+
+    git_repo.commit_directly_to_branch(session_branch_name, "Second commit with changes")
+    commit_sha_2 = repo.commit(session_branch_name).hexsha
+    assert commit_sha_2 != initial_commit_sha
+
+    # Third commit (empty)
+    commit_sha_3 = git_repo.commit_directly_to_branch(session_branch_name, "Third empty commit")
+    assert commit_sha_3 == commit_sha_2
+    assert commit_sha_3 != initial_commit_sha

programmer/tools.py

@@ -4,7 +4,7 @@
 import subprocess
 import weave
 
-LENGTH_LIMIT = 10000
+LENGTH_LIMIT = 30000
 
 
 def read_image_as_base64(path: str):